chenmingxuan
e1ea099592
Install and update artemis-use with task diagnostics and plugin guidance. Preserve cross-device file receipts, review canonical reports, guard stage transitions and reduce small history summaries. Improve exploration lifecycle, request recovery and browser live connections.
1675 lines
74 KiB
Python
1675 lines
74 KiB
Python
# Copyright 2026 Google LLC
|
||
#
|
||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
# you may not use this file except in compliance with the License.
|
||
# You may obtain a copy of the License at
|
||
#
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
#
|
||
# Unless required by applicable law or agreed to in writing, software
|
||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
# See the License for the specific language governing permissions and
|
||
# limitations under the License.
|
||
|
||
"""Universal Multi-Model FlashRunner for Artemis.
|
||
|
||
Executes autonomous reactive mobile workflows through OpenAI Responses.
|
||
|
||
Conversation shape (shared with the Pro operator, history redesign §3.2):
|
||
the prompt is built every turn from a session :class:`TranscriptLedger` —
|
||
a byte-stable system prefix, the committed earlier turns (append-only within
|
||
a context phase; old observations and long spans compacted together only at
|
||
pressure thresholds), and a fresh tail carrying the current
|
||
observation under a ``# CURRENT OBSERVATION [T+mm:ss]`` header. Actions run
|
||
synchronously and their tool messages already carry the outcome; only a turn
|
||
whose action failed ends with an ``--- Action Execution Result (T+mm:ss) ---``
|
||
message (it carries the error). Every timestamp the model sees is a
|
||
session-relative offset — the same clock the video analyzer uses for the
|
||
session recording.
|
||
|
||
Per-turn notices on the tail (the reasoning reminder after a silent turn, the
|
||
no-tool-call nudge, the final-turn warning, the user-guidance wrapper) are
|
||
marked ephemeral: the ledger's compaction pass deletes them when it rewrites
|
||
the recent window, so they never reach the frozen history or a chunk capsule.
|
||
The verbatim body of a user instruction is the exception: it stays in the
|
||
active window until its turn is chunked, whose user lines then carry it on.
|
||
"""
|
||
|
||
import asyncio
|
||
import base64
|
||
from dataclasses import dataclass, field
|
||
import json
|
||
from pathlib import Path
|
||
import re
|
||
import uuid
|
||
|
||
from artemis.config.settings import settings
|
||
from artemis.runtime.task_control import TaskControl, has_pending_instruction
|
||
|
||
from jinja2 import Template
|
||
from langchain_core.messages import (
|
||
BaseMessage,
|
||
HumanMessage,
|
||
SystemMessage,
|
||
ToolMessage,
|
||
)
|
||
|
||
from artemis.agents.flash.summarizer import VisualStepSummarizer, build_focus_context
|
||
from artemis.agents.flash.progress import PROGRESS_TOOL, TaskProgress
|
||
from artemis.agents.operator.prompts import REASONING_REMINDER, UserGuidance, render_user_guidance
|
||
from artemis.agents.validator.tool_declarations import (
|
||
ASK_EXPLORER_TOOL,
|
||
CLICK_SEQUENCE_TOOL,
|
||
LIST_NOTES_TOOL,
|
||
READ_NOTE_TOOL,
|
||
REPORT_TASK_STATUS_TOOL,
|
||
ToolExecutionResult,
|
||
VALIDATOR_TOOLS_DECLARATION,
|
||
capture_screenshot_and_parse_ui,
|
||
prune_intermediate_screenshots,
|
||
)
|
||
from artemis.config import (
|
||
MemoryRuntimeConfig,
|
||
MemoryTranscriptConfig,
|
||
StepSummarizerConfig,
|
||
load_agent_config,
|
||
)
|
||
from artemis.context import ArtemisContext
|
||
from artemis.controllers.unified_controller import UnifiedMobileController
|
||
from artemis.data_engine.trace import trace
|
||
from artemis.graph.perception import _check_injected_instruction_file
|
||
from artemis.graph.state import State
|
||
from artemis.llm.router import ModelEndpoint
|
||
from artemis.llm.structured import ParseFailure, parse_structured
|
||
from artemis.mcp.action_executor import McpActionExecutor
|
||
from artemis.mcp.observation import DeviceObservationError, require_observation
|
||
from artemis.memory.transcript import (
|
||
OBSERVATION_SCOPE_RULE,
|
||
PRO_UI_LIST_MARKER,
|
||
TranscriptLedger,
|
||
mark_ephemeral,
|
||
)
|
||
from artemis.services.llm import (
|
||
RobustChatModelWrapper,
|
||
acomplete,
|
||
get_openai_llm,
|
||
get_llm,
|
||
invoke_llm_with_timeout_message,
|
||
)
|
||
from artemis.tools.history import history_tool_declarations
|
||
from artemis.tools.android_files import android_file
|
||
from artemis.tools.tool_wrapper import tool_result_messages
|
||
from artemis.utils.coordinates import (
|
||
COORDINATE_SPACE_KEY,
|
||
COORDINATE_SPACE_NORMALIZED,
|
||
parse_swipe_parameters,
|
||
)
|
||
from artemis.utils.logger import get_logger
|
||
|
||
logger = get_logger(__name__)
|
||
|
||
_NO_TOOL_CALL_NOTICE = (
|
||
"You did not call any tools last turn. Please make progress by calling an"
|
||
" action tool or 'report_task_status'."
|
||
)
|
||
|
||
_FINAL_TURN_WARNING = "[WARNING] This is your final turn; only 'report_task_status' is available."
|
||
|
||
|
||
@dataclass
|
||
class _TurnRecord:
|
||
"""What one reactive turn produced, for committing it into the ledger.
|
||
|
||
``step_keys`` are the DataEngine step ids recorded this turn (one per
|
||
executed action; the tool_call_id stands in without a DataEngine) — the
|
||
first keys the observation screenshot to its visual-transition summary,
|
||
all of them feed the chunk ledger. ``actions`` collects the outcome of
|
||
every device action so the turn's execution result can be rendered.
|
||
"""
|
||
|
||
step_keys: list[str] = field(default_factory=list)
|
||
actions: list[tuple[str, str, str]] = field(default_factory=list)
|
||
|
||
def result(self) -> dict | None:
|
||
"""The turn's execution result in the validator-report shape.
|
||
|
||
``None`` unless an action failed: Flash executes synchronously and each
|
||
action's tool message already carries its outcome, so a result message
|
||
would only repeat it. A failure is worth its own message because it
|
||
carries the error the turn ended on.
|
||
"""
|
||
for name, status, text in self.actions:
|
||
if status != "success":
|
||
# Executor messages name the action themselves ("Error
|
||
# executing click: ..."); prefix only the ones that don't.
|
||
return {"status": "failed", "error": text if name in text else f"{name}: {text}"}
|
||
return None
|
||
|
||
|
||
class FlashRunner:
|
||
"""Reactive, ultra-fast agent execution loop supporting all VLM providers."""
|
||
|
||
def __init__(self, ctx: ArtemisContext, goal: str, max_turns: int | None = None):
|
||
self.ctx = ctx
|
||
self.goal = goal
|
||
self.final_report_received = False
|
||
self._android_observation_stale = False
|
||
try:
|
||
cfg = load_agent_config()
|
||
self.max_turns = max_turns if max_turns is not None else cfg.flash.max_turns
|
||
self.step_summarizer_cfg = cfg.flash.step_summarizer
|
||
self.memory_runtime_cfg = cfg.memory.runtime
|
||
self.transcript_cfg = cfg.memory.transcript
|
||
self.chunking_cfg = cfg.memory.chunking
|
||
except Exception:
|
||
self.max_turns = max_turns if max_turns is not None else 0
|
||
self.step_summarizer_cfg = StepSummarizerConfig()
|
||
self.memory_runtime_cfg = MemoryRuntimeConfig()
|
||
self.transcript_cfg = MemoryTranscriptConfig()
|
||
self.chunking_cfg = None
|
||
|
||
self.controller = UnifiedMobileController(ctx)
|
||
# ``agent_name="flash"`` makes ask_explorer follow ``explorer.flash_mode``
|
||
# (the Flash profile knob) instead of the Pro profile's tier.
|
||
self.executor = McpActionExecutor(ctx, self.controller, agent_name="flash")
|
||
self.summarizer = (
|
||
VisualStepSummarizer(
|
||
ctx,
|
||
model_name=self.step_summarizer_cfg.model,
|
||
retry_limit=self.memory_runtime_cfg.retry_limit,
|
||
timeout_seconds=self.step_summarizer_cfg.timeout_seconds,
|
||
max_concurrency=self.memory_runtime_cfg.max_concurrency,
|
||
flush_timeout_s=self.memory_runtime_cfg.flush_timeout_s,
|
||
)
|
||
if self.step_summarizer_cfg.enabled
|
||
else None
|
||
)
|
||
# Publish the service on the composition root (ctx.step_memory) so
|
||
# any co-resident consumer shares this run's summary runtime.
|
||
if self.summarizer is not None and getattr(ctx, "step_memory", None) is None:
|
||
try:
|
||
ctx.step_memory = self.summarizer
|
||
except (AttributeError, TypeError, ValueError):
|
||
pass
|
||
|
||
@property
|
||
def turn_limit(self) -> int | None:
|
||
"""The reactive turn cap, or ``None`` when the loop is unbounded."""
|
||
try:
|
||
limit = int(self.max_turns) if self.max_turns is not None else 0
|
||
except (TypeError, ValueError):
|
||
return None
|
||
return limit if limit > 0 else None
|
||
|
||
def _video_tools_enabled(self) -> bool:
|
||
"""Same gate the Pro graph applies before binding ``video_analyzer``."""
|
||
setup = getattr(self.ctx, "execution_setup", None)
|
||
return getattr(setup, "video_recording_tools_enabled", False) is True
|
||
|
||
def _get_tools(self) -> list:
|
||
# Task-local deliverables use the same notes and history as Pro.
|
||
tools = [
|
||
t
|
||
for t in VALIDATOR_TOOLS_DECLARATION
|
||
if t is not READ_NOTE_TOOL and t is not LIST_NOTES_TOOL
|
||
]
|
||
tools.insert(1, CLICK_SEQUENCE_TOOL)
|
||
tools.append(ASK_EXPLORER_TOOL)
|
||
if self.ctx.data_engine is not None:
|
||
from artemis.core.tool_declaration import ToolDeclaration
|
||
from artemis.tools.scratchpad import SaveNoteArgs
|
||
|
||
tools.extend([READ_NOTE_TOOL, LIST_NOTES_TOOL])
|
||
tools.append(
|
||
ToolDeclaration(
|
||
"save_note",
|
||
"Save a requested text deliverable to this task's trace/notes/<key>.md. "
|
||
"Use a plain key such as report or reusable_case, never an absolute path. "
|
||
"Returns the actual path, size, SHA-256 and read-back verification. Existing "
|
||
"notes with this key are overwritten. The output key is reserved for the "
|
||
"automatically saved final response. Use read_note to inspect saved content. "
|
||
"Do not save/rewrite a plan every turn; only persist requested deliverables "
|
||
"or important findings not already retained in history.",
|
||
SaveNoteArgs.model_json_schema(),
|
||
)
|
||
)
|
||
# Helper tools shared with the Pro operator: the history tools are
|
||
# declared from the very same args schemas the LangChain tools bind
|
||
# (same availability gates: a DataEngine session, recall config).
|
||
tools.extend(history_tool_declarations(self.ctx))
|
||
if android_file.is_available(self.ctx):
|
||
tools.append(android_file.to_tool_declaration())
|
||
if self._video_tools_enabled():
|
||
from artemis.tools.video_tool import VIDEO_ANALYZER_TOOL
|
||
|
||
tools.append(VIDEO_ANALYZER_TOOL)
|
||
tools.append(REPORT_TASK_STATUS_TOOL)
|
||
if getattr(self.ctx, "execution_target", None) == "computer":
|
||
from artemis.mcp.action_manifest import DEVICE_ACTIONS
|
||
|
||
tools = [
|
||
t
|
||
for t in tools
|
||
if t.name not in DEVICE_ACTIONS
|
||
and t.name not in {"ask_explorer", "video_analyzer", "click_sequence"}
|
||
]
|
||
# With an actuator installed, drop declarations for device actions the
|
||
# backend does not implement (and append its extension tools); without
|
||
# one, the full historical declaration set is kept.
|
||
actuator = getattr(self.ctx, "actuator", None)
|
||
if actuator is not None:
|
||
from artemis.mcp.action_manifest import filter_declarations
|
||
|
||
tools = filter_declarations(tools, actuator, "flash")
|
||
if getattr(self.ctx, "plugins", None):
|
||
tools.extend(self.ctx.plugins.declarations())
|
||
tools.append(PROGRESS_TOOL)
|
||
return tools
|
||
|
||
def _prune_intermediate_screenshots(self, messages: list[BaseMessage]) -> None:
|
||
"""Prunes binary screenshot blocks from intermediate observation messages."""
|
||
prune_intermediate_screenshots(messages)
|
||
|
||
# ------------------------------------------------------------------
|
||
# run() setup helpers
|
||
# ------------------------------------------------------------------
|
||
|
||
def _build_ledger(self) -> TranscriptLedger:
|
||
"""Builds the session transcript ledger with the shared Pro history policy.
|
||
|
||
The ``T+mm:ss`` clock is anchored to the DataEngine session start so
|
||
the observation headers, the chunk ledger lines and the video
|
||
analyzer's action timeline share one origin. L2/L3 chunk compression
|
||
needs DataEngine step records; without an engine the ledger runs
|
||
scrub-edge-only.
|
||
"""
|
||
engine = getattr(self.ctx, "data_engine", None)
|
||
session_start = getattr(engine, "session_start_time", None) if engine else None
|
||
cfg = self.transcript_cfg
|
||
ledger = TranscriptLedger(
|
||
step_memory=self.summarizer,
|
||
prune_history_xml=self.step_summarizer_cfg.prune_history_xml,
|
||
image_scrub_depth=getattr(cfg, "image_scrub_depth", 3),
|
||
pending_grace_steps=getattr(cfg, "pending_grace_steps", 3),
|
||
xml_scrub_depth=getattr(cfg, "xml_scrub_depth", 1),
|
||
image_scrub_depth_relaxed=getattr(cfg, "image_scrub_depth_relaxed", None),
|
||
context_budget_tokens=getattr(cfg, "context_budget_tokens", None),
|
||
start_ratio=getattr(cfg, "start_ratio", None),
|
||
soft_ratio=getattr(cfg, "soft_ratio", 0.7),
|
||
hard_ratio=getattr(cfg, "hard_ratio", 0.9),
|
||
session_start=session_start if isinstance(session_start, (int, float)) else None,
|
||
)
|
||
if engine is not None:
|
||
try:
|
||
from artemis.memory import HistoryChunkManager
|
||
|
||
ledger.attach_chunker(
|
||
HistoryChunkManager(
|
||
engine=engine,
|
||
ctx=self.ctx,
|
||
chunking_config=self.chunking_cfg,
|
||
transcript_config=cfg,
|
||
goal=self.goal,
|
||
)
|
||
)
|
||
except Exception as e:
|
||
logger.error(f"History chunk manager unavailable for FlashRunner: {e}")
|
||
try:
|
||
self.ctx.transcript_ledger = ledger
|
||
except (AttributeError, TypeError, ValueError):
|
||
pass
|
||
return ledger
|
||
|
||
#: The model of the current run; decides how screenshot tool results travel.
|
||
_llm = None
|
||
|
||
def _init_llm(self):
|
||
"""Initializes the Universal LLM via the Service Layer."""
|
||
try:
|
||
return get_llm(self.ctx, name="operator")
|
||
except Exception as e:
|
||
logger.warning(f"Failed to get operator LLM from config, using default: {e}")
|
||
|
||
return RobustChatModelWrapper(
|
||
get_openai_llm(model_name=settings.OPENAI_MODEL),
|
||
self.ctx,
|
||
endpoint=ModelEndpoint(model_name=settings.OPENAI_MODEL),
|
||
)
|
||
|
||
def _render_system_prompt(self, tools_declaration: list) -> str:
|
||
"""Renders the system prompt from the flash_runner.md template.
|
||
|
||
Tool-teaching segments are gated on the available tool set so an absent
|
||
tool leaves no trace in the prompt; until an actuator is wired in, the
|
||
full manifest set reproduces the historical prompt.
|
||
"""
|
||
prompt_path = Path(__file__).parent / "flash_runner.md"
|
||
prompt_template = prompt_path.read_text(encoding="utf-8")
|
||
available_tools = frozenset(t.name for t in tools_declaration)
|
||
prompt = Template(prompt_template).render(goal=self.goal, available_tools=available_tools)
|
||
if getattr(self.ctx, "plugins", None):
|
||
prompt += self.ctx.plugins.prompt()
|
||
return prompt
|
||
|
||
# ------------------------------------------------------------------
|
||
# Per-turn helpers (observe / think)
|
||
# ------------------------------------------------------------------
|
||
|
||
async def _read_injected_instruction(
|
||
self, offset_label: str | None = None
|
||
) -> tuple[str | None, UserGuidance | None]:
|
||
"""Consumes this turn's injected instruction: ``(instruction, guidance)``.
|
||
|
||
``instruction`` is the user's text verbatim — what the step record,
|
||
the chunk ledger and the visual-transition lens receive (the same
|
||
shape Pro stamps). ``guidance`` is the shared user-guidance pair
|
||
(:func:`render_user_guidance`, the Pro rendering without a plan to
|
||
edit): the ephemeral ``--- User Guidance ---`` wrapper and the
|
||
persistent verbatim body, both riding on the observation tail.
|
||
``offset_label`` stamps the body with the session offset. Both are
|
||
None when nothing was injected.
|
||
"""
|
||
if not (self.ctx.data_engine and self.ctx.data_engine.base_dir):
|
||
return None, None
|
||
try:
|
||
injected_payload = await asyncio.to_thread(
|
||
_check_injected_instruction_file,
|
||
str(self.ctx.data_engine.base_dir),
|
||
)
|
||
except (OSError, ValueError, AttributeError) as e:
|
||
logger.warning(f"Failed to check injected instruction in FlashRunner: {e}")
|
||
return None, None
|
||
if not (injected_payload and injected_payload.get("instruction")):
|
||
return None, None
|
||
instruction = str(injected_payload["instruction"])
|
||
guidance = render_user_guidance(
|
||
instruction,
|
||
release_loop=bool(injected_payload.get("release_loop")),
|
||
has_plan=False,
|
||
offset_label=offset_label,
|
||
)
|
||
return instruction, guidance
|
||
|
||
def _build_tail(
|
||
self,
|
||
ledger: TranscriptLedger,
|
||
turns: int,
|
||
img_bytes,
|
||
xml_list,
|
||
*,
|
||
injected: UserGuidance | None = None,
|
||
notices: list[str] | None = None,
|
||
is_final: bool = False,
|
||
previous_turn_silent: bool = False,
|
||
) -> HumanMessage:
|
||
"""Builds this turn's observation tail (Pro observation shape).
|
||
|
||
Header, screenshot and UI list carry the same markers as the Pro
|
||
operator's tail so the ledger's scrub edge treats both alike. The
|
||
objective is stated once, in the system prompt; the tail never repeats
|
||
it. Every per-turn notice (nudges, the user-guidance wrapper, the
|
||
final-turn warning, the reasoning reminder after a silent turn) is
|
||
marked ephemeral so it leaves the transcript with the recent window.
|
||
The verbatim body of an injected instruction is the one exception: it
|
||
is a regular block, so a standing instruction stays in the active
|
||
window until the turn is chunked (the chunk's user lines carry it on).
|
||
"""
|
||
# ``turns`` is kept for call-site symmetry: the tail has the same shape
|
||
# on every turn.
|
||
blocks: list[dict] = [
|
||
{"type": "text", "text": f"# CURRENT OBSERVATION [{ledger.elapsed_label()}]"}
|
||
]
|
||
if img_bytes:
|
||
img_b64 = base64.b64encode(img_bytes).decode("utf-8")
|
||
blocks.append({"type": "text", "text": "--- Current Screenshot ---"})
|
||
blocks.append(
|
||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}}
|
||
)
|
||
blocks.append(
|
||
{
|
||
"type": "text",
|
||
"text": f"{PRO_UI_LIST_MARKER}\n"
|
||
+ (
|
||
xml_list
|
||
or "当前没有可用的元素索引。仅根据当前截图定位坐标，禁止使用历史画面的元素编号。"
|
||
),
|
||
}
|
||
)
|
||
ephemeral: list[int] = []
|
||
# (text, ephemeral) in tail order.
|
||
per_turn: list[tuple[str, bool]] = [(text, True) for text in (notices or [])]
|
||
if injected:
|
||
# The wrapper is per-turn; the verbatim body right after it is a
|
||
# regular block that stays until the turn is chunked.
|
||
per_turn.append((injected.wrapper, True))
|
||
per_turn.append((injected.body, False))
|
||
if is_final:
|
||
per_turn.append((_FINAL_TURN_WARNING, True))
|
||
if previous_turn_silent:
|
||
per_turn.append((REASONING_REMINDER, True))
|
||
for text, is_ephemeral in per_turn:
|
||
if is_ephemeral:
|
||
ephemeral.append(len(blocks))
|
||
blocks.append({"type": "text", "text": text})
|
||
tail = HumanMessage(content=blocks)
|
||
if ephemeral:
|
||
mark_ephemeral(tail, ephemeral)
|
||
return tail
|
||
|
||
def _extract_response_text(self, response) -> str:
|
||
"""Extracts the natural-language thought text from the model response."""
|
||
raw_text = response.content if isinstance(response.content, str) else ""
|
||
if isinstance(response.content, list):
|
||
raw_text = "".join(
|
||
b.get("text", "") for b in response.content if isinstance(b, dict) and "text" in b
|
||
)
|
||
return raw_text
|
||
|
||
def _extract_response_thinking(self, response) -> str:
|
||
"""Extracts the model's native thinking blocks (thought summaries), if any.
|
||
|
||
Providers running with ``include_thoughts`` return them as ``thinking``
|
||
content blocks next to the ``text`` blocks; they are recorded on the step
|
||
as ``operator_native_thinking`` and never fed back as reply text.
|
||
"""
|
||
if not isinstance(response.content, list):
|
||
return ""
|
||
parts = [
|
||
str(b.get("thinking") or "")
|
||
for b in response.content
|
||
if isinstance(b, dict) and b.get("type") == "thinking"
|
||
]
|
||
return "\n".join(p for p in parts if p.strip())
|
||
|
||
def _token_usage_from_response(self, response) -> dict | None:
|
||
"""Extracts token usage metadata from the model response, if present."""
|
||
step_token_usage = None
|
||
if hasattr(response, "usage_metadata") and response.usage_metadata:
|
||
u = response.usage_metadata
|
||
pr = u.get("input_tokens") or u.get("prompt_tokens") or 0
|
||
co = u.get("output_tokens") or u.get("completion_tokens") or 0
|
||
to = u.get("total_tokens") or (pr + co)
|
||
if to > 0:
|
||
step_token_usage = {
|
||
"prompt_tokens": int(pr),
|
||
"completion_tokens": int(co),
|
||
"total_tokens": int(to),
|
||
}
|
||
elif hasattr(response, "response_metadata") and isinstance(
|
||
response.response_metadata, dict
|
||
):
|
||
u = response.response_metadata.get("usage_metadata") or response.response_metadata.get(
|
||
"token_usage"
|
||
)
|
||
if isinstance(u, dict):
|
||
pr = (
|
||
u.get("input_tokens")
|
||
or u.get("prompt_tokens")
|
||
or u.get("prompt_token_count")
|
||
or 0
|
||
)
|
||
co = (
|
||
u.get("output_tokens")
|
||
or u.get("completion_tokens")
|
||
or u.get("candidates_token_count")
|
||
or 0
|
||
)
|
||
to = u.get("total_tokens") or u.get("total_token_count") or (pr + co)
|
||
if to > 0:
|
||
step_token_usage = {
|
||
"prompt_tokens": int(pr),
|
||
"completion_tokens": int(co),
|
||
"total_tokens": int(to),
|
||
}
|
||
return step_token_usage
|
||
|
||
def _estimate_token_usage(self, messages: list[BaseMessage], raw_text: str) -> dict:
|
||
"""Approximates token usage from message sizes when metadata is absent."""
|
||
calc_prompt_chars = 0
|
||
calc_images = 0
|
||
for msg in messages:
|
||
c = getattr(msg, "content", "")
|
||
if isinstance(c, str):
|
||
calc_prompt_chars += len(c)
|
||
elif isinstance(c, list):
|
||
for block in c:
|
||
if isinstance(block, dict):
|
||
if block.get("type") == "image_url" or "image_url" in block:
|
||
calc_images += 1
|
||
else:
|
||
calc_prompt_chars += len(str(block.get("text", "")))
|
||
else:
|
||
calc_prompt_chars += len(str(block))
|
||
prompt_tokens = (calc_prompt_chars // 4) + (calc_images * 258)
|
||
completion_tokens = len(raw_text) // 4
|
||
return {
|
||
"prompt_tokens": max(1, prompt_tokens),
|
||
"completion_tokens": max(1, completion_tokens),
|
||
"total_tokens": max(1, prompt_tokens + completion_tokens),
|
||
}
|
||
|
||
def _resolve_token_usage(
|
||
self, response, messages: list[BaseMessage], raw_text: str
|
||
) -> tuple[dict, bool]:
|
||
"""Token usage for the call and whether the provider measured it.
|
||
|
||
Falls back to an estimate when the response carries no usage
|
||
metadata; the flag lets the caller keep an estimate out of the
|
||
ledger's chars-per-token calibration (an estimate is itself derived
|
||
from character counts, so calibrating on it would be circular).
|
||
"""
|
||
step_token_usage = self._token_usage_from_response(response)
|
||
if step_token_usage and step_token_usage.get("total_tokens", 0) > 0:
|
||
return step_token_usage, True
|
||
return self._estimate_token_usage(messages, raw_text), False
|
||
|
||
def _record_llm_trace(self, step_token_usage: dict, raw_text: str) -> None:
|
||
"""Records the llm_call trace for this turn in the DataEngine."""
|
||
if self.ctx.data_engine:
|
||
current_step_id = getattr(self.ctx.data_engine, "current_step_id", None)
|
||
self.ctx.data_engine.record_trace(
|
||
type="llm_call",
|
||
name="FlashRunner",
|
||
payload={"token_usage": step_token_usage, "response": raw_text},
|
||
step_id=current_step_id,
|
||
status="success",
|
||
)
|
||
|
||
def _resolve_tool_calls(self, response, raw_text: str) -> list:
|
||
"""Returns native tool calls, falling back to text parsing if absent."""
|
||
tool_calls = response.tool_calls or []
|
||
|
||
# Fallback text parsing if tool_calls not parsed natively
|
||
if not tool_calls and "```json" in raw_text:
|
||
parsed = parse_structured(raw_text)
|
||
if isinstance(parsed, ParseFailure):
|
||
logger.warning(
|
||
"FlashRunner response contained a JSON block that could"
|
||
f" not be parsed: {parsed.error}"
|
||
)
|
||
elif isinstance(parsed, dict) and "name" in parsed:
|
||
tool_calls = [
|
||
{
|
||
"name": parsed["name"],
|
||
"args": parsed.get("args", {}),
|
||
"id": str(uuid.uuid4()),
|
||
}
|
||
]
|
||
return tool_calls
|
||
|
||
# ------------------------------------------------------------------
|
||
# Per-tool-call helpers (act / record / report)
|
||
# ------------------------------------------------------------------
|
||
|
||
async def _finalize_task_report(
|
||
self,
|
||
name: str,
|
||
args: dict,
|
||
tc_id: str,
|
||
raw_text: str,
|
||
step_token_usage: dict,
|
||
pre_screenshot_bytes,
|
||
xml_list,
|
||
messages: list[BaseMessage],
|
||
native_text: str | None = None,
|
||
) -> dict | None:
|
||
"""Records the final step, acknowledges the tool call, and flushes."""
|
||
final_report = dict(args)
|
||
if args.get("report_note_key"):
|
||
from artemis.utils.notes import read_note_content
|
||
|
||
try:
|
||
key = args["report_note_key"]
|
||
if not isinstance(key, str) or not re.fullmatch(r"[\w-]{1,128}", key):
|
||
raise ValueError("report_note_key 必须是已保存报告的普通笔记名称")
|
||
body = read_note_content(self.ctx.data_engine.base_dir, key)
|
||
if not body.strip():
|
||
raise ValueError("报告正文为空")
|
||
final_report["explanation"] = body
|
||
except (OSError, ValueError, AttributeError) as exc:
|
||
messages.append(
|
||
ToolMessage(
|
||
tool_call_id=tc_id,
|
||
name=name,
|
||
status="error",
|
||
content=f"最终报告未提交：{exc}。修正报告名称或直接提供完整 explanation。",
|
||
)
|
||
)
|
||
return None
|
||
args = final_report
|
||
self.final_report_received = True
|
||
if self.ctx.data_engine:
|
||
try:
|
||
if self.ctx.data_engine.current_step_id is None:
|
||
self.ctx.data_engine.allocate_step_id()
|
||
self.ctx.data_engine.record_step(
|
||
pre_screenshot_bytes=pre_screenshot_bytes,
|
||
ui_tree=xml_list,
|
||
action_taken={"action": "report_task_status", "args": args},
|
||
operator_raw_thinking=raw_text,
|
||
operator_native_thinking=native_text or None,
|
||
last_execution_result={"result": "Task completed with final report."},
|
||
extra_metadata={"token_usage": step_token_usage},
|
||
)
|
||
except Exception as step_err:
|
||
logger.warning(f"Error recording final step in FlashRunner: {step_err}")
|
||
messages.append(
|
||
ToolMessage(
|
||
tool_call_id=tc_id,
|
||
name=name,
|
||
content=json.dumps({"status": "acknowledged"}),
|
||
status="success",
|
||
)
|
||
)
|
||
if self.summarizer:
|
||
await self.summarizer.flush()
|
||
return final_report
|
||
|
||
async def _capture_post_screenshot(
|
||
self, exec_result, name: str, action_names, state: State | None = None
|
||
):
|
||
"""Returns the post-action screenshot, capturing a fallback if needed.
|
||
|
||
When a device action produced no observation (a failed action, an
|
||
unknown tool, ...) the screen is observed through the very same
|
||
pipeline the executor's success path uses (:func:`observe`), so the
|
||
next observation tail carries the indexed minimal UI list and
|
||
``state`` gets the same refreshed screenshot path and indexed
|
||
elements. Nothing is captured for non-action tools.
|
||
"""
|
||
post_img_bytes = exec_result.screenshot_bytes
|
||
if (
|
||
exec_result.metadata.get("plugin_id")
|
||
and getattr(self.ctx, "execution_target", "android") != "computer"
|
||
):
|
||
return post_img_bytes
|
||
# File operations have their own verified receipts. Do not add a UI
|
||
# observation/visual summary that could obscure that non-visual evidence.
|
||
if post_img_bytes or name not in action_names or name in {"android_file", "save_note"}:
|
||
return post_img_bytes
|
||
if state is not None:
|
||
state.indexed_points = []
|
||
state.indexed_elements = []
|
||
state.latest_screenshot = None
|
||
obs, screenshot_bytes = await require_observation(self.ctx, self.controller, settle_ms=0)
|
||
post_img_bytes = screenshot_bytes
|
||
exec_result.ui_elements_text = obs.elements_text
|
||
exec_result.screenshot_path = obs.screenshot_path
|
||
if state is not None:
|
||
if obs.hierarchy_ok:
|
||
state.indexed_points = [el["center"] for el in obs.elements]
|
||
state.indexed_elements = obs.elements
|
||
if obs.screenshot_path:
|
||
state.latest_screenshot = obs.screenshot_path
|
||
return post_img_bytes
|
||
|
||
def _extract_normalized_coordinates(self, name: str, args: dict, resolved=None):
|
||
"""Extracts and enriches coordinate metadata for the recorded action.
|
||
|
||
``resolved`` is the normalized point the executor sent a point action
|
||
to; it is the coordinate of record when the model addressed an element
|
||
by index (the arguments then carry no point at all).
|
||
"""
|
||
norm_coords = None
|
||
norm_start = None
|
||
norm_end = None
|
||
if name in ("click", "tap", "long_press", "input_text") and (
|
||
isinstance(resolved, (list, tuple)) and len(resolved) == 2
|
||
):
|
||
return [int(resolved[0]), int(resolved[1])], None, None
|
||
if name == "swipe":
|
||
kind, target_val, _ = parse_swipe_parameters(args)
|
||
if kind == "coords" and isinstance(target_val, list) and len(target_val) == 4:
|
||
norm_coords = target_val
|
||
norm_start = target_val[:2]
|
||
norm_end = target_val[2:]
|
||
elif kind == "direction" and isinstance(target_val, str):
|
||
g_lower = target_val.lower()
|
||
if "up" in g_lower:
|
||
norm_coords = [600, 700, 600, 300]
|
||
elif "down" in g_lower:
|
||
norm_coords = [600, 300, 600, 700]
|
||
elif "left" in g_lower:
|
||
norm_coords = [750, 500, 250, 500]
|
||
elif "right" in g_lower:
|
||
norm_coords = [250, 500, 750, 500]
|
||
if norm_coords:
|
||
norm_start = norm_coords[:2]
|
||
norm_end = norm_coords[2:]
|
||
elif name in ("click", "tap", "long_press", "input_text"):
|
||
target = args.get("target") or args.get("coordinates")
|
||
if isinstance(target, (list, tuple)) and len(target) == 2:
|
||
norm_coords = list(target)
|
||
elif isinstance(target, str):
|
||
nums = re.findall(r"-?\d+(?:\.\d+)?", target)
|
||
if len(nums) == 2:
|
||
norm_coords = [int(float(nums[0])), int(float(nums[1]))]
|
||
return norm_coords, norm_start, norm_end
|
||
|
||
def _build_action_record(self, name: str, args: dict, exec_result) -> dict:
|
||
"""The action as the step record and the visual lens see it.
|
||
|
||
Flash records the model's own 0–1000 coordinates verbatim (an index
|
||
target records the point the executor resolved it to); the explicit
|
||
space marker keeps every later normalization pass (agent-friendly
|
||
steps, MCP inspector) a no-op on this record. The tool arguments ride
|
||
along under ``args``, and the target semantics the executor validated
|
||
(observed element fields for an index, the model's own
|
||
``target_description`` for a point) land top level in the shape Pro
|
||
records.
|
||
"""
|
||
metadata = getattr(exec_result, "metadata", None) or {}
|
||
resolved = metadata.get("target_coordinates")
|
||
norm_coords, norm_start, norm_end = self._extract_normalized_coordinates(
|
||
name, args, resolved=resolved
|
||
)
|
||
given = args.get("target") or args.get("coordinates") or args.get("sequence")
|
||
if isinstance(given, (int, float)) and not isinstance(given, bool):
|
||
given = None # An element index is not a coordinate.
|
||
action_dict = {
|
||
"action": name,
|
||
"coordinates": given or norm_coords,
|
||
COORDINATE_SPACE_KEY: COORDINATE_SPACE_NORMALIZED,
|
||
"args": args,
|
||
}
|
||
if norm_coords:
|
||
action_dict["normalized_coordinates"] = norm_coords
|
||
if norm_start and norm_end:
|
||
action_dict["normalized_start_coordinates"] = norm_start
|
||
action_dict["normalized_end_coordinates"] = norm_end
|
||
action_dict.update(metadata.get("target_semantics") or {})
|
||
if metadata.get("plugin_id"):
|
||
action_dict.update(
|
||
{
|
||
key: metadata[key]
|
||
for key in (
|
||
"plugin_id",
|
||
"plugin_version",
|
||
"resource_ids",
|
||
"plugin_status",
|
||
"source_id",
|
||
"source",
|
||
"call_id",
|
||
"execution",
|
||
"side_effect",
|
||
"error_code",
|
||
"attachments",
|
||
"recording",
|
||
"pre_image_name",
|
||
"post_image_name",
|
||
"pre_image_valid",
|
||
)
|
||
if key in metadata
|
||
}
|
||
)
|
||
action_dict.pop("coordinates", None)
|
||
action_dict.pop(COORDINATE_SPACE_KEY, None)
|
||
if name in {"android_file", "save_note"}:
|
||
action_dict.pop("coordinates", None)
|
||
action_dict.pop(COORDINATE_SPACE_KEY, None)
|
||
return action_dict
|
||
|
||
def _record_action_step(
|
||
self,
|
||
name: str,
|
||
args: dict,
|
||
exec_result,
|
||
raw_text: str,
|
||
step_token_usage: dict,
|
||
pre_screenshot_bytes,
|
||
xml_list,
|
||
post_img_bytes,
|
||
injected: str | None = None,
|
||
native_text: str | None = None,
|
||
action_dict: dict | None = None,
|
||
):
|
||
"""Records telemetry / step in DataEngine; returns the step id or None."""
|
||
recorded_step_id = None
|
||
try:
|
||
if self.ctx.data_engine.current_step_id is None:
|
||
self.ctx.data_engine.allocate_step_id()
|
||
|
||
if action_dict is None:
|
||
action_dict = self._build_action_record(name, args, exec_result)
|
||
|
||
succeeded = exec_result.status == "success"
|
||
last_execution_result = {
|
||
"status": "dispatched" if succeeded else "failed",
|
||
"result": exec_result.text_summary,
|
||
}
|
||
if not succeeded:
|
||
last_execution_result["error"] = exec_result.text_summary
|
||
|
||
extra_metadata: dict = {"token_usage": step_token_usage}
|
||
if injected:
|
||
# Stamped verbatim on the step it reached: the chunk ledger
|
||
# keeps it as a never-evicted line at every compression level.
|
||
extra_metadata["injected_instruction"] = injected
|
||
|
||
recorded_step_id = self.ctx.data_engine.record_step(
|
||
pre_screenshot_bytes=pre_screenshot_bytes,
|
||
post_screenshot_bytes=post_img_bytes,
|
||
ui_tree=(exec_result.ui_elements_text or xml_list),
|
||
action_taken=action_dict,
|
||
operator_raw_thinking=raw_text,
|
||
operator_native_thinking=native_text or None,
|
||
last_execution_result=last_execution_result,
|
||
extra_metadata=extra_metadata,
|
||
)
|
||
self._notify_history_chunker(recorded_step_id)
|
||
except Exception as step_err:
|
||
logger.warning(f"Error recording step in FlashRunner: {step_err}")
|
||
return recorded_step_id
|
||
|
||
def _notify_history_chunker(self, step_id) -> None:
|
||
"""Stamps the recorded step for the chunk manager (single segment, no plan)."""
|
||
ledger = getattr(self.ctx, "transcript_ledger", None)
|
||
chunker = getattr(ledger, "chunker", None) if ledger is not None else None
|
||
if chunker is None or step_id is None:
|
||
return
|
||
try:
|
||
chunker.on_step_stamped(str(step_id), None)
|
||
except Exception as e:
|
||
logger.warning(f"History chunker step stamp failed: {e}")
|
||
|
||
async def _execute_and_record_action(
|
||
self,
|
||
name: str,
|
||
args: dict,
|
||
tc_id: str,
|
||
state: State,
|
||
messages: list[BaseMessage],
|
||
raw_text: str,
|
||
step_token_usage: dict,
|
||
pre_screenshot_bytes,
|
||
xml_list,
|
||
action_sequence: int,
|
||
turn: _TurnRecord,
|
||
injected: str | None = None,
|
||
native_text: str | None = None,
|
||
index_elements: list[dict] | None = None,
|
||
):
|
||
"""Executes one tool call, records it, and updates loop state.
|
||
|
||
The tool message carries the outcome text only; the post-action
|
||
screenshot and UI list become the next turn's observation tail.
|
||
``index_elements`` is the turn's element snapshot index targets
|
||
resolve against (see :meth:`_process_tool_calls`); ``None`` resolves
|
||
against the state's current list. Returns the (possibly updated)
|
||
pre_screenshot_bytes, xml_list, and action_sequence for the next
|
||
iteration.
|
||
"""
|
||
# Dynamic dispatch set: manifest device actions plus any backend
|
||
# extension tools, so extension steps are recorded like actions.
|
||
# File helpers perform real work too. Persist their receipts as steps,
|
||
# otherwise helper-only turns vanish from the final checker's evidence.
|
||
action_names = self.executor.action_tool_names | {"android_file", "save_note"}
|
||
plugin_action = False
|
||
try:
|
||
operation = self.executor.execute(
|
||
name, args, tc_id, state, index_elements=index_elements
|
||
)
|
||
from artemis.plugins.runtime import get_plugin_runtime
|
||
|
||
plugins = get_plugin_runtime(self.ctx)
|
||
if plugins and name in plugins.bindings:
|
||
plugin_action = True
|
||
# A phone observation cannot serve as the before-image for a desktop action.
|
||
pre_screenshot_bytes = plugins.previous_image(name, args)
|
||
xml_list = ""
|
||
if getattr(self.ctx, "execution_target", "android") != "computer":
|
||
# Invalidate before dispatch: even a failed/interrupted PC
|
||
# action may have changed the environment or the phone.
|
||
self._android_observation_stale = True
|
||
state.indexed_elements = []
|
||
state.indexed_points = []
|
||
state.latest_screenshot = None
|
||
control = getattr(self.ctx, "task_control", None)
|
||
if plugins and name in plugins.tool_names and isinstance(control, TaskControl):
|
||
interrupted, exec_result = await control.run_interruptibly(
|
||
operation,
|
||
on_pause=getattr(self.ctx, "on_task_pause", None),
|
||
on_resume=getattr(self.ctx, "on_task_resume", None),
|
||
)
|
||
if interrupted:
|
||
messages.extend(
|
||
tool_result_messages(
|
||
tc_id,
|
||
"操作被暂停或指导打断，结果不确定。先重新观察，不能重放旧动作。",
|
||
name=name,
|
||
status="error",
|
||
llm=self._llm,
|
||
)
|
||
)
|
||
return None, "操作已打断，必须重新观察。", action_sequence
|
||
else:
|
||
exec_result = await operation
|
||
|
||
post_img_bytes = await self._capture_post_screenshot(
|
||
exec_result, name, action_names, state
|
||
)
|
||
if (
|
||
plugin_action
|
||
and exec_result.metadata.get("source")
|
||
and not exec_result.metadata.get("pre_image_valid")
|
||
):
|
||
pre_screenshot_bytes = None
|
||
|
||
# One record shape for the DataEngine step and the visual lens.
|
||
action_dict = (
|
||
self._build_action_record(name, args, exec_result) if name in action_names else None
|
||
)
|
||
|
||
# Record telemetry / step in DataEngine
|
||
recorded_step_id = None
|
||
if self.ctx.data_engine and name in action_names:
|
||
recorded_step_id = self._record_action_step(
|
||
name,
|
||
args,
|
||
exec_result,
|
||
raw_text,
|
||
step_token_usage,
|
||
None if name == "save_note" else pre_screenshot_bytes,
|
||
"" if name == "save_note" else xml_list,
|
||
post_img_bytes,
|
||
injected=injected,
|
||
native_text=native_text,
|
||
action_dict=action_dict,
|
||
)
|
||
|
||
if name in action_names:
|
||
turn.step_keys.append(str(recorded_step_id) if recorded_step_id else str(tc_id))
|
||
turn.actions.append((name, exec_result.status, exec_result.text_summary))
|
||
|
||
# ⚡ Non-blocking dispatch of objective visual transition summarizer
|
||
if self.summarizer and name in action_names and post_img_bytes:
|
||
action_sequence += 1
|
||
self.summarizer.dispatch(
|
||
step_number=action_sequence,
|
||
action_name=name,
|
||
# The recorded action minus its verb: the same shape the Pro
|
||
# summarizer hands the lens, so the phrase renders alike.
|
||
action_args={k: v for k, v in action_dict.items() if k != "action"},
|
||
pre_img_bytes=pre_screenshot_bytes,
|
||
post_img_bytes=post_img_bytes,
|
||
exec_outcome=exec_result.text_summary,
|
||
post_ui_elements_text=exec_result.ui_elements_text,
|
||
observation_settled=(exec_result.metadata or {}).get("observation_settled"),
|
||
receipt_only=bool(
|
||
plugin_action
|
||
and exec_result.metadata.get("read_only") is True
|
||
and exec_result.status == "success"
|
||
),
|
||
action_key=str(tc_id),
|
||
data_engine_step_id=recorded_step_id,
|
||
# The model's own reasoning for this turn steers the lens's
|
||
# attention; the target it named rides on ``args``.
|
||
focus=build_focus_context(
|
||
intent=raw_text,
|
||
goal=self.goal,
|
||
injected_instruction=injected,
|
||
),
|
||
)
|
||
|
||
if post_img_bytes:
|
||
pre_screenshot_bytes = post_img_bytes
|
||
if exec_result.ui_elements_text is not None:
|
||
xml_list = exec_result.ui_elements_text
|
||
|
||
# Helper tools may return multimodal blocks (a step screenshot);
|
||
# device actions always report text — their post screenshot is the
|
||
# next observation tail, never a tool-message image. The image
|
||
# carrier follows the model's provider (tool_result_messages).
|
||
raw_blocks = (
|
||
exec_result.raw_result
|
||
if name not in action_names or exec_result.metadata.get("plugin_id")
|
||
else None
|
||
)
|
||
content = (
|
||
raw_blocks
|
||
if isinstance(raw_blocks, list) and raw_blocks
|
||
else exec_result.text_summary or f"Action '{name}' dispatched."
|
||
)
|
||
messages.extend(
|
||
tool_result_messages(
|
||
tc_id, content, name=name, status=exec_result.status, llm=self._llm
|
||
)
|
||
)
|
||
|
||
except DeviceObservationError:
|
||
raise
|
||
except Exception as e:
|
||
logger.error(f"Error executing tool {name}: {e}")
|
||
if name in action_names:
|
||
turn.step_keys.append(str(tc_id))
|
||
turn.actions.append((name, "error", f"Error executing tool {name}: {e}"))
|
||
messages.append(
|
||
ToolMessage(
|
||
tool_call_id=tc_id,
|
||
name=name,
|
||
content=f"Error executing tool {name}: {e}",
|
||
status="error",
|
||
)
|
||
)
|
||
if plugin_action and getattr(self.ctx, "execution_target", "android") != "computer":
|
||
# The labelled tool result already carries the PC evidence. Do not
|
||
# capture an idle phone or duplicate the PC bitmap in the phone tail.
|
||
return None, self._plugin_observation_notice(), action_sequence
|
||
return pre_screenshot_bytes, xml_list, action_sequence
|
||
|
||
@staticmethod
|
||
def _plugin_observation_notice() -> str:
|
||
return (
|
||
"当前使用插件观察；实际画面和控件见对应工具返回，不是安卓画面。"
|
||
"安卓旧截图、元素编号和坐标已失效。连续操作电脑无需重复观察手机；"
|
||
"等待电脑加载可用正时长 wait_for_delay，等待本身不会刷新画面，之后调用插件观察工具；"
|
||
"只有切回安卓时才单独调用 wait_for_delay(time_in_ms=0) 读取最新手机画面，"
|
||
"收到后再决定手机动作。不要将电脑控件编号用于手机。"
|
||
)
|
||
|
||
async def _refresh_observation(self, state: State):
|
||
shot_path, image, elements = await capture_screenshot_and_parse_ui(
|
||
self.ctx, state, self.controller, skip_settling=False
|
||
)
|
||
state.latest_screenshot = shot_path
|
||
self._android_observation_stale = False
|
||
return image, elements
|
||
|
||
@staticmethod
|
||
def _defer_tool_calls(
|
||
messages: list[BaseMessage], tool_calls: list, *, reason: str | None = None
|
||
) -> None:
|
||
"""Close every pending call without executing decisions made on stale evidence."""
|
||
for tc in tool_calls:
|
||
messages.append(
|
||
ToolMessage(
|
||
tool_call_id=tc.get("id") or str(uuid.uuid4()),
|
||
name=tc["name"],
|
||
content=reason
|
||
or "未执行此调用：已切回安卓并重新观察。请先读取最新手机画面和控件，再重新决定动作或结论。",
|
||
status="error",
|
||
)
|
||
)
|
||
|
||
async def _invoke_model(self, llm, current_tools: list, messages: list[BaseMessage]):
|
||
"""Binds the active tools and invokes the model through the LLM gateway."""
|
||
# Bind active tools
|
||
bound_llm = llm.bind_tools(current_tools)
|
||
|
||
# Invoke Model. Streaming, live-token UI deltas, classified
|
||
# retries, and pause/resume are all owned by the LLM gateway
|
||
# (acomplete); a typed LLMCallError propagates if it gives up.
|
||
return await invoke_llm_with_timeout_message(
|
||
acomplete(bound_llm, messages), timeout_seconds=10
|
||
)
|
||
|
||
async def _process_tool_calls(
|
||
self,
|
||
tool_calls: list,
|
||
state: State,
|
||
messages: list[BaseMessage],
|
||
raw_text: str,
|
||
step_token_usage: dict,
|
||
pre_screenshot_bytes,
|
||
xml_list,
|
||
action_sequence: int,
|
||
turn: _TurnRecord,
|
||
injected: str | None = None,
|
||
native_text: str | None = None,
|
||
):
|
||
"""Dispatches the turn's tool calls.
|
||
|
||
Every index target of the turn resolves against the element list the
|
||
model decided on: the list of the observation this turn was built
|
||
from, snapshotted here before the first action. Each action still
|
||
refreshes ``state.indexed_elements`` (that refreshed list is what the
|
||
next observation shows), so without the snapshot ``click(5)`` after
|
||
``click(3)`` would be resolved against the screen after the first
|
||
click.
|
||
|
||
The snapshot is the very list object the observation left on the
|
||
state (not a copy), and observations replace the state's list rather
|
||
than mutating it, so ``ask_explorer`` candidates registered before
|
||
any action land in the snapshot by construction. When a helper tool
|
||
appends candidates after an action has already replaced the state's
|
||
list, the explorer numbered them against that replaced list, so the
|
||
snapshot is re-taken from it: indices the explorer just handed out
|
||
resolve correctly, at the cost of the pre-action numbering (mixing
|
||
the two in one turn is the model's error, not a supported case).
|
||
|
||
Returns (final_report, pre_screenshot_bytes, xml_list, action_sequence);
|
||
final_report is non-None only when 'report_task_status' was called.
|
||
"""
|
||
action_names = self.executor.action_tool_names | {"android_file"}
|
||
from artemis.plugins.runtime import get_plugin_runtime
|
||
|
||
plugins = get_plugin_runtime(self.ctx)
|
||
if not hasattr(self, "_progress"):
|
||
self._progress = TaskProgress()
|
||
android_tools = action_names | {"ask_explorer"}
|
||
if plugins:
|
||
android_tools -= plugins.tool_names
|
||
index_elements = self._turn_index_snapshot(state)
|
||
for call_index, tc in enumerate(tool_calls):
|
||
control = getattr(self.ctx, "task_control", None)
|
||
if isinstance(control, TaskControl) and has_pending_instruction(control.directory):
|
||
return None, pre_screenshot_bytes, xml_list, action_sequence
|
||
name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
|
||
args = tc.get("args") or {}
|
||
tc_id = tc.get("id") or str(uuid.uuid4())
|
||
|
||
if name == "update_task_progress":
|
||
try:
|
||
resources = (
|
||
{"android"}
|
||
if getattr(self.ctx, "execution_target", None) != "computer"
|
||
else set()
|
||
)
|
||
resources.update(plugins.packages if plugins else [])
|
||
directory = getattr(self.ctx.data_engine, "base_dir", None)
|
||
text = self._progress.update(args, resources=resources, directory=directory)
|
||
status = "success"
|
||
except (ValueError, OSError) as exc:
|
||
text, status = str(exc), "error"
|
||
messages.append(
|
||
ToolMessage(tool_call_id=tc_id, name=name, status=status, content=text)
|
||
)
|
||
if self.ctx.data_engine:
|
||
result = ToolExecutionResult(
|
||
tool_name=name, tool_call_id=tc_id, status=status, text_summary=text
|
||
)
|
||
step_id = self._record_action_step(
|
||
name,
|
||
args,
|
||
result,
|
||
raw_text,
|
||
step_token_usage,
|
||
None,
|
||
"",
|
||
None,
|
||
injected=injected,
|
||
native_text=native_text,
|
||
)
|
||
turn.step_keys.append(str(step_id or tc_id))
|
||
if status == "error":
|
||
self._defer_tool_calls(
|
||
messages,
|
||
tool_calls[call_index + 1 :],
|
||
reason="进度更新失败，先修正后再执行关联动作。",
|
||
)
|
||
return None, pre_screenshot_bytes, xml_list, action_sequence
|
||
continue
|
||
|
||
resource = None
|
||
if plugins and name in plugins.bindings:
|
||
if not plugins.is_read_only(name):
|
||
resource = plugins.bindings[name][0]
|
||
elif name in android_tools and name not in {"wait_for_delay", "ask_explorer"}:
|
||
if name != "android_file" or args.get("action") in {"write_text", "push"}:
|
||
resource = "android"
|
||
blocked = self._progress.guard(
|
||
resource,
|
||
completed=name == "report_task_status" and args.get("status") == "completed",
|
||
)
|
||
if blocked:
|
||
self._defer_tool_calls(messages, tool_calls[call_index:], reason=blocked)
|
||
return None, pre_screenshot_bytes, xml_list, action_sequence
|
||
|
||
# A positive wait on the PC means elapsed time, not a phone refresh.
|
||
if (
|
||
getattr(self, "_android_observation_stale", False)
|
||
and name == "wait_for_delay"
|
||
and isinstance(args.get("time_in_ms"), (int, float))
|
||
and 0 < args["time_in_ms"] < float("inf")
|
||
):
|
||
operation = asyncio.sleep(args["time_in_ms"] / 1000)
|
||
if isinstance(control, TaskControl):
|
||
interrupted, _ = await control.run_interruptibly(
|
||
operation,
|
||
on_pause=getattr(self.ctx, "on_task_pause", None),
|
||
on_resume=getattr(self.ctx, "on_task_resume", None),
|
||
)
|
||
if interrupted:
|
||
self._defer_tool_calls(
|
||
messages,
|
||
tool_calls[call_index:],
|
||
reason="等待被用户打断，请重新观察当前目标再继续。",
|
||
)
|
||
return None, pre_screenshot_bytes, xml_list, action_sequence
|
||
else:
|
||
await operation
|
||
message = f"已等待 {args['time_in_ms']} 毫秒，保持当前插件操作端；未采集新画面，请用插件观察工具核对加载结果。"
|
||
result = ToolExecutionResult(
|
||
tool_name=name, tool_call_id=tc_id, status="success", text_summary=message
|
||
)
|
||
step_id = None
|
||
if self.ctx.data_engine:
|
||
step_id = self._record_action_step(
|
||
name,
|
||
args,
|
||
result,
|
||
raw_text,
|
||
step_token_usage,
|
||
None,
|
||
"",
|
||
None,
|
||
injected=injected,
|
||
native_text=native_text,
|
||
)
|
||
turn.step_keys.append(str(step_id or tc_id))
|
||
turn.actions.append((name, "success", message))
|
||
messages.append(ToolMessage(tool_call_id=tc_id, name=name, content=message))
|
||
continue
|
||
|
||
returning_to_android = (
|
||
getattr(self, "_android_observation_stale", False) and name in android_tools
|
||
)
|
||
if returning_to_android and name != "wait_for_delay":
|
||
logger.info(f"Flash 暂缓 {name}：切回安卓，先刷新画面并重新决定动作。")
|
||
pre_screenshot_bytes, xml_list = await self._refresh_observation(state)
|
||
self._defer_tool_calls(messages, tool_calls[call_index:])
|
||
return None, pre_screenshot_bytes, xml_list, action_sequence
|
||
|
||
logger.info(f"Executing Flash tool: {name}({args})")
|
||
if name == "report_task_status":
|
||
final_report = await self._finalize_task_report(
|
||
name,
|
||
args,
|
||
tc_id,
|
||
raw_text,
|
||
step_token_usage,
|
||
pre_screenshot_bytes,
|
||
xml_list,
|
||
messages,
|
||
native_text=native_text,
|
||
)
|
||
return final_report, pre_screenshot_bytes, xml_list, action_sequence
|
||
|
||
elements_before = getattr(state, "indexed_elements", None)
|
||
before_len = len(elements_before) if isinstance(elements_before, list) else 0
|
||
pre_screenshot_bytes, xml_list, action_sequence = await self._execute_and_record_action(
|
||
name,
|
||
args,
|
||
tc_id,
|
||
state,
|
||
messages,
|
||
raw_text,
|
||
step_token_usage,
|
||
pre_screenshot_bytes,
|
||
xml_list,
|
||
action_sequence,
|
||
turn,
|
||
injected=injected,
|
||
native_text=native_text,
|
||
index_elements=index_elements,
|
||
)
|
||
if resource:
|
||
self._progress.last_resource = resource
|
||
if returning_to_android:
|
||
# wait_for_delay has no stale target and its normal post-action
|
||
# observation is fresh. The model must see it before any further
|
||
# calls (including a report) from this pre-observation batch.
|
||
if pre_screenshot_bytes is None:
|
||
pre_screenshot_bytes, xml_list = await self._refresh_observation(state)
|
||
self._android_observation_stale = False
|
||
self._defer_tool_calls(messages, tool_calls[call_index + 1 :])
|
||
return None, pre_screenshot_bytes, xml_list, action_sequence
|
||
if name not in action_names:
|
||
index_elements = self._extend_index_snapshot(state, index_elements, before_len)
|
||
return None, pre_screenshot_bytes, xml_list, action_sequence
|
||
|
||
@staticmethod
|
||
def _turn_index_snapshot(state: State) -> list[dict]:
|
||
"""The element list this turn's index targets resolve against.
|
||
|
||
The state's list object itself, so in-place appends (explorer
|
||
candidates) before any action join the snapshot; an empty list is
|
||
installed on the state when there is none, for the same reason.
|
||
"""
|
||
elements = getattr(state, "indexed_elements", None)
|
||
if not isinstance(elements, list):
|
||
elements = []
|
||
try:
|
||
state.indexed_elements = elements
|
||
except (AttributeError, TypeError, ValueError):
|
||
pass
|
||
return elements
|
||
|
||
@staticmethod
|
||
def _extend_index_snapshot(state: State, snapshot: list[dict], before_len: int) -> list[dict]:
|
||
"""Re-takes the snapshot after a helper tool appended candidates to a
|
||
list that is no longer the snapshot (an action replaced it first).
|
||
|
||
Appends into the snapshot itself need nothing: the explorer numbered
|
||
them against the snapshot. Only growth of a different list object
|
||
moves the snapshot, since that is the list the new numbers refer to.
|
||
"""
|
||
current = getattr(state, "indexed_elements", None)
|
||
if not isinstance(current, list) or current is snapshot:
|
||
return snapshot
|
||
if len(current) > before_len:
|
||
return current
|
||
return snapshot
|
||
|
||
async def _prepare_conversation(self, state: State, tools_declaration: list):
|
||
"""Installs the static prefix and captures the initial device state."""
|
||
ledger = self._build_ledger()
|
||
ledger.set_static_prefix(
|
||
[
|
||
SystemMessage(
|
||
content=self._render_system_prompt(tools_declaration)
|
||
+ "\n"
|
||
+ OBSERVATION_SCOPE_RULE
|
||
)
|
||
]
|
||
)
|
||
|
||
# Capture Initial State (Screenshot + UI Tree)
|
||
img_bytes, xml_list = await self._refresh_observation(state)
|
||
return ledger, img_bytes, xml_list
|
||
|
||
@staticmethod
|
||
def _commit_turn(ledger: TranscriptLedger, turn: _TurnRecord | None) -> None:
|
||
"""Commits the previous turn once its step ids and outcomes are known."""
|
||
if turn is None:
|
||
return
|
||
ledger.commit_staged(
|
||
step_key=turn.step_keys[0] if turn.step_keys else None,
|
||
validator_result=turn.result(),
|
||
extra_step_keys=turn.step_keys[1:],
|
||
)
|
||
|
||
@trace(type="agent", name="FlashRunner")
|
||
async def run(self, state: State) -> dict:
|
||
try:
|
||
return await self._run(state)
|
||
except DeviceObservationError as exc:
|
||
state.indexed_points = []
|
||
state.indexed_elements = []
|
||
state.latest_screenshot = None
|
||
logger.error(f"Flash 因设备观测失败停止：{exc}")
|
||
return {
|
||
"status": "failed",
|
||
"explanation": str(exc),
|
||
"error": str(exc),
|
||
"code": "device_observation_failed",
|
||
}
|
||
|
||
async def _run(self, state: State) -> dict:
|
||
limit = self.turn_limit
|
||
logger.info(
|
||
f"Starting Artemis Flash reactive loop for goal: {self.goal}"
|
||
f" (turn limit: {limit if limit else 'unlimited'})"
|
||
)
|
||
|
||
# 1. Initialize Universal LLM via Service Layer
|
||
llm = self._init_llm()
|
||
self._llm = llm
|
||
|
||
tools_declaration = self._get_tools()
|
||
report_only_tools = [t for t in tools_declaration if t.name == "report_task_status"]
|
||
|
||
ledger, img_bytes, xml_list = await self._prepare_conversation(state, tools_declaration)
|
||
|
||
turns = 0
|
||
action_sequence = 0
|
||
final_report = None
|
||
current_pre_screenshot_bytes = img_bytes
|
||
current_xml_list = xml_list
|
||
previous_turn: _TurnRecord | None = None
|
||
pending_notices: list[str] = []
|
||
no_tool_attempts = 0
|
||
no_tool_limit = settings.ARTEMIS_FLASH_MAX_NO_PROGRESS_ATTEMPTS
|
||
fallback_llm = None
|
||
using_tool_fallback = False
|
||
action_failure_attempts = 0
|
||
refreshed_for_guidance = False
|
||
|
||
while limit is None or turns < limit:
|
||
control = getattr(self.ctx, "task_control", None)
|
||
if isinstance(control, TaskControl):
|
||
resumed = await control.checkpoint(
|
||
on_pause=getattr(self.ctx, "on_task_pause", None),
|
||
on_resume=getattr(self.ctx, "on_task_resume", None),
|
||
)
|
||
if resumed:
|
||
using_tool_fallback = False
|
||
no_tool_attempts = 0
|
||
action_failure_attempts = 0
|
||
self._llm = llm
|
||
(
|
||
current_pre_screenshot_bytes,
|
||
current_xml_list,
|
||
) = await self._refresh_observation(state)
|
||
refreshed_for_guidance = True
|
||
pending_notices.append(
|
||
"设备在暂停期间可能被其他任务操作过。已重新观察画面；"
|
||
"请根据当前状态和新指导重新判断操作，勿沿用旧坐标。"
|
||
)
|
||
turns += 1
|
||
logger.info(f"--- Artemis Flash Turn {turns}{f'/{limit}' if limit else ''} ---")
|
||
|
||
if self.ctx.data_engine:
|
||
self.ctx.data_engine.allocate_step_id()
|
||
|
||
# Commit the previous turn: its step ids and outcomes exist now.
|
||
self._commit_turn(ledger, previous_turn)
|
||
previous_turn = None
|
||
|
||
# Check for real-time injected instructions: the verbatim text
|
||
# is stamped on the step, the guidance pair goes to the tail.
|
||
injected, injected_guidance = await self._read_injected_instruction(
|
||
ledger.elapsed_label()
|
||
)
|
||
if injected and not refreshed_for_guidance:
|
||
current_pre_screenshot_bytes, current_xml_list = await self._refresh_observation(
|
||
state
|
||
)
|
||
refreshed_for_guidance = False
|
||
|
||
# Tool restriction on the final turn (bounded loops only)
|
||
is_final = limit is not None and turns == limit
|
||
tail = self._build_tail(
|
||
ledger,
|
||
turns,
|
||
current_pre_screenshot_bytes,
|
||
current_xml_list,
|
||
injected=injected_guidance,
|
||
notices=pending_notices,
|
||
is_final=is_final,
|
||
# The ledger judged the turn it just committed: a bare tool
|
||
# call (or thinking only) earns a one-line reminder, no bounce.
|
||
previous_turn_silent=bool(getattr(ledger, "last_turn_silent", False)),
|
||
)
|
||
pending_notices = []
|
||
|
||
# S + F + A + tail: history changes only at a compaction boundary.
|
||
messages = ledger.render([tail])
|
||
turn_base = len(messages) - 1
|
||
current_tools = report_only_tools if is_final else self._get_tools()
|
||
|
||
if isinstance(control, TaskControl):
|
||
interrupted, response = await control.run_interruptibly(
|
||
self._invoke_model(
|
||
fallback_llm if using_tool_fallback else llm, current_tools, messages
|
||
),
|
||
on_pause=getattr(self.ctx, "on_task_pause", None),
|
||
on_resume=getattr(self.ctx, "on_task_resume", None),
|
||
)
|
||
else:
|
||
interrupted, response = (
|
||
False,
|
||
await self._invoke_model(
|
||
fallback_llm if using_tool_fallback else llm, current_tools, messages
|
||
),
|
||
)
|
||
|
||
if interrupted or (
|
||
isinstance(control, TaskControl) and has_pending_instruction(control.directory)
|
||
):
|
||
# Do not dispatch a stale action or final answer from a prompt
|
||
# assembled before the instruction arrived.
|
||
using_tool_fallback = False
|
||
no_tool_attempts = 0
|
||
action_failure_attempts = 0
|
||
self._llm = llm
|
||
current_pre_screenshot_bytes, current_xml_list = await self._refresh_observation(
|
||
state
|
||
)
|
||
refreshed_for_guidance = True
|
||
pending_notices.append(
|
||
"新指导已打断旧模型步骤。请重新观察画面并按最新指导决定下一步。"
|
||
)
|
||
turns -= 1
|
||
continue
|
||
|
||
if response is None:
|
||
break
|
||
|
||
messages.append(response)
|
||
|
||
# Extract thought and tool calls
|
||
raw_text = self._extract_response_text(response)
|
||
native_text = self._extract_response_thinking(response)
|
||
|
||
# Extract token usage metadata from response
|
||
step_token_usage, measured = self._resolve_token_usage(response, messages, raw_text)
|
||
# The ledger's compaction thresholds read the runner's own prompt
|
||
# size, never the session-wide meter (which any agent overwrites).
|
||
# The prompt as sent (``messages`` minus the just-appended reply)
|
||
# rides along so the ledger can calibrate its chars-per-token
|
||
# ratio — only from a provider measurement, never from the
|
||
# estimate (same as the Pro operator).
|
||
ledger.record_prompt_tokens(
|
||
step_token_usage.get("prompt_tokens") if measured else None,
|
||
messages=messages[:-1] if measured else None,
|
||
)
|
||
|
||
self._record_llm_trace(step_token_usage, raw_text)
|
||
|
||
tool_calls = self._resolve_tool_calls(response, raw_text)
|
||
if isinstance(control, TaskControl) and has_pending_instruction(control.directory):
|
||
using_tool_fallback = False
|
||
no_tool_attempts = 0
|
||
self._llm = llm
|
||
current_pre_screenshot_bytes, current_xml_list = await self._refresh_observation(
|
||
state
|
||
)
|
||
refreshed_for_guidance = True
|
||
pending_notices.append("新指导已收到；忽略旧模型动作，重新观察画面。")
|
||
turns -= 1
|
||
continue
|
||
|
||
if not tool_calls:
|
||
logger.info(
|
||
f"FlashRunner received response without tool calls at turn {turns}:"
|
||
f" {raw_text[:100]}..."
|
||
)
|
||
if is_final:
|
||
final_report = {"status": "failed", "explanation": raw_text}
|
||
break
|
||
no_tool_attempts += 1
|
||
active_limit = (
|
||
settings.ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS
|
||
if using_tool_fallback
|
||
else no_tool_limit
|
||
)
|
||
if no_tool_attempts >= active_limit:
|
||
if using_tool_fallback:
|
||
final_report = {
|
||
"status": "failed",
|
||
"explanation": (
|
||
f"首选模型连续 {no_tool_limit} 次、兜底模型连续 "
|
||
f"{no_tool_attempts} 次未调用工具，已停止任务；"
|
||
"未执行新的设备动作。"
|
||
),
|
||
}
|
||
break
|
||
fallback_llm = get_llm(self.ctx, name="operator", use_fallback=True)
|
||
using_tool_fallback = True
|
||
no_tool_attempts = 0
|
||
action_failure_attempts = 0
|
||
logger.warning("Flash 首选模型未调用工具，当前步骤切换到兜底模型")
|
||
pending_notices.append("首选模型连续未调用工具；本步骤改用兜底模型。")
|
||
pending_notices.append(_NO_TOOL_CALL_NOTICE)
|
||
# Keep the observation and any injected user instruction, but
|
||
# do not feed the model its own unproductive prose on retry.
|
||
ledger.stage_turn(messages[turn_base:-1])
|
||
previous_turn = _TurnRecord()
|
||
continue
|
||
|
||
no_tool_attempts = 0
|
||
if using_tool_fallback:
|
||
# A tool-producing turn completes the fallback's scope. The next
|
||
# observation goes back to the configured primary model.
|
||
self._llm = fallback_llm
|
||
# Process tool calls
|
||
turn = _TurnRecord()
|
||
(
|
||
final_report_from_calls,
|
||
current_pre_screenshot_bytes,
|
||
current_xml_list,
|
||
action_sequence,
|
||
) = await self._process_tool_calls(
|
||
tool_calls,
|
||
state,
|
||
messages,
|
||
raw_text,
|
||
step_token_usage,
|
||
current_pre_screenshot_bytes,
|
||
current_xml_list,
|
||
action_sequence,
|
||
turn,
|
||
injected=injected,
|
||
native_text=native_text,
|
||
)
|
||
if using_tool_fallback and any(status == "success" for _, status, _ in turn.actions):
|
||
using_tool_fallback = False
|
||
self._llm = llm
|
||
pending_notices.append("兜底模型已完成本步骤；后续步骤恢复首选模型。")
|
||
for _, status, _ in turn.actions:
|
||
action_failure_attempts = 0 if status == "success" else action_failure_attempts + 1
|
||
failure_limit = (
|
||
settings.ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS
|
||
if using_tool_fallback
|
||
else settings.ARTEMIS_ACTION_FAILURE_ATTEMPTS
|
||
)
|
||
if action_failure_attempts >= failure_limit:
|
||
if using_tool_fallback:
|
||
final_report = {
|
||
"status": "failed",
|
||
"explanation": f"兜底模型连续 {action_failure_attempts} 次设备动作失败，已停止任务。最后原因：{turn.result()}",
|
||
}
|
||
break
|
||
# A fresh, valid observation is required before spending any
|
||
# fallback model calls. Hardware recovery is not model retry.
|
||
current_pre_screenshot_bytes, current_xml_list = await self._refresh_observation(
|
||
state
|
||
)
|
||
fallback_llm = get_llm(self.ctx, name="operator", use_fallback=True)
|
||
using_tool_fallback = True
|
||
self._llm = fallback_llm
|
||
action_failure_attempts = 0
|
||
no_tool_attempts = 0
|
||
pending_notices.append(
|
||
f"首选模型连续 {settings.ARTEMIS_ACTION_FAILURE_ATTEMPTS} 次设备动作失败。"
|
||
f"最后原因：{turn.result()}。已重新观察；本步骤由兜底模型重新定位，勿重复旧动作。"
|
||
)
|
||
ledger.stage_turn(messages[turn_base:])
|
||
previous_turn = turn
|
||
if final_report_from_calls is not None:
|
||
return final_report_from_calls
|
||
|
||
if self.summarizer:
|
||
await self.summarizer.flush()
|
||
|
||
if final_report is None:
|
||
explanation = (
|
||
"Max turns reached without final status report."
|
||
if limit is not None and turns >= limit
|
||
else "The model returned no response; the reactive loop stopped."
|
||
)
|
||
final_report = {"status": "failed", "explanation": explanation}
|
||
return final_report
|