somew1nd
a45daf23f8
Report device actions as dispatched and preserve full failure details. Route Flash through McpActionExecutor and use ToolFailure across helper tools and agent loops instead of inferring failure from response text. Require model-provided descriptions for coordinate targets. Keep observed element metadata separate from those descriptions in safety checks, visual summaries, history, and frontend labels. Add Gemini Interactions API video analysis with configurable processing mode, chunk size, and timeouts. Centralize Google provider capabilities and token usage normalization, and update google-genai to 2.22.0. Preserve system instructions on follow-up interactions, replan pending chunks when falling back to static processing, and reject unanswered native video and audio segments instead of caching placeholder results. Keep helper-tool failure status intact through history tools and committee calls. Validation: 1,762 Python unit tests passed (3 skipped), 40 frontend formatter tests passed, and the production UI build completed. Ruff lint, formatting, the quality ratchet, and git diff checks passed.
972 lines
40 KiB
Python
972 lines
40 KiB
Python
# Copyright 2026 Google LLC
|
||
#
|
||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
# you may not use this file except in compliance with the License.
|
||
# You may obtain a copy of the License at
|
||
#
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
#
|
||
# Unless required by applicable law or agreed to in writing, software
|
||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
# See the License for the specific language governing permissions and
|
||
# limitations under the License.
|
||
|
||
"""Universal Multi-Model FlashRunner for Artemis.
|
||
|
||
Executes autonomous reactive mobile workflows across Google Gemini,
|
||
OpenAI GPT-4o/o3, Anthropic Claude 3.5/3.7, and OpenRouter endpoints.
|
||
|
||
Conversation shape (shared with the Pro operator, history redesign §3.2):
|
||
the prompt is built every turn from a session :class:`TranscriptLedger` —
|
||
a byte-stable system prefix, the committed earlier turns (append-only; old UI
|
||
lists stripped at depth 1, screenshots resolved to visual summaries at depth
|
||
K, long spans chunk-compressed), and a fresh tail carrying the current
|
||
observation under a ``# CURRENT OBSERVATION [T+mm:ss]`` header. Each
|
||
committed turn ends with an ``--- Action Execution Result (T+mm:ss) ---``
|
||
message, so every timestamp the model sees is a session-relative offset — the
|
||
same clock the video analyzer uses for the session recording.
|
||
"""
|
||
|
||
import asyncio
|
||
import base64
|
||
from dataclasses import dataclass, field
|
||
import json
|
||
from pathlib import Path
|
||
import re
|
||
import uuid
|
||
|
||
from jinja2 import Template
|
||
from langchain_core.messages import (
|
||
BaseMessage,
|
||
HumanMessage,
|
||
SystemMessage,
|
||
ToolMessage,
|
||
)
|
||
|
||
from artemis.agents.flash.summarizer import VisualStepSummarizer, build_focus_context
|
||
from artemis.agents.validator.tool_declarations import (
|
||
ASK_EXPLORER_TOOL,
|
||
CLICK_SEQUENCE_TOOL,
|
||
REPORT_TASK_STATUS_TOOL,
|
||
VALIDATOR_TOOLS_DECLARATION,
|
||
capture_screenshot_and_parse_ui,
|
||
prune_intermediate_screenshots,
|
||
)
|
||
from artemis.config import (
|
||
MemoryRuntimeConfig,
|
||
MemoryTranscriptConfig,
|
||
StepSummarizerConfig,
|
||
load_agent_config,
|
||
)
|
||
from artemis.context import ArtemisContext
|
||
from artemis.controllers.unified_controller import UnifiedMobileController
|
||
from artemis.data_engine.trace import trace
|
||
from artemis.graph.perception import _check_injected_instruction_file
|
||
from artemis.graph.state import State
|
||
from artemis.llm.structured import ParseFailure, parse_structured
|
||
from artemis.mcp.action_executor import McpActionExecutor
|
||
from artemis.memory.transcript import PRO_UI_LIST_MARKER, TranscriptLedger
|
||
from artemis.services.llm import (
|
||
RobustChatModelWrapper,
|
||
acomplete,
|
||
get_google_llm,
|
||
get_llm,
|
||
invoke_llm_with_timeout_message,
|
||
)
|
||
from artemis.tools.history import history_tool_declarations
|
||
from artemis.tools.tool_wrapper import tool_result_messages
|
||
from artemis.utils.coordinates import (
|
||
COORDINATE_SPACE_KEY,
|
||
COORDINATE_SPACE_NORMALIZED,
|
||
parse_swipe_parameters,
|
||
)
|
||
from artemis.utils.logger import get_logger
|
||
|
||
logger = get_logger(__name__)
|
||
|
||
#: Reminder carried by the first observation only (the system prompt states the rule).
|
||
_REASONING_FIRST_REMINDER = (
|
||
"CRITICAL RULE: In every single turn, you MUST FIRST output a natural"
|
||
" language reasoning/explanation paragraph BEFORE invoking any tool call."
|
||
)
|
||
|
||
_NO_TOOL_CALL_NOTICE = (
|
||
"You did not call any tools last turn. Please make progress by calling an"
|
||
" action tool or 'report_task_status'."
|
||
)
|
||
|
||
_FINAL_TURN_WARNING = "[WARNING] This is your final turn; only 'report_task_status' is available."
|
||
|
||
|
||
@dataclass
|
||
class _TurnRecord:
|
||
"""What one reactive turn produced, for committing it into the ledger.
|
||
|
||
``step_keys`` are the DataEngine step ids recorded this turn (one per
|
||
executed action; the tool_call_id stands in without a DataEngine) — the
|
||
first keys the observation screenshot to its visual-transition summary,
|
||
all of them feed the chunk ledger. ``actions`` collects the outcome of
|
||
every device action so the turn's execution result can be rendered.
|
||
"""
|
||
|
||
step_keys: list[str] = field(default_factory=list)
|
||
actions: list[tuple[str, str, str]] = field(default_factory=list)
|
||
|
||
def result(self) -> dict | None:
|
||
"""The turn's execution result in the validator-report shape.
|
||
|
||
``None`` when no device action ran (helper-only turns have no result
|
||
message, as in Pro).
|
||
"""
|
||
if not self.actions:
|
||
return None
|
||
for name, status, text in self.actions:
|
||
if status != "success":
|
||
# Executor messages name the action themselves ("Error
|
||
# executing click: ..."); prefix only the ones that don't.
|
||
return {"status": "failed", "error": text if name in text else f"{name}: {text}"}
|
||
return {"status": "dispatched"}
|
||
|
||
|
||
class FlashRunner:
|
||
"""Reactive, ultra-fast agent execution loop supporting all VLM providers."""
|
||
|
||
def __init__(self, ctx: ArtemisContext, goal: str, max_turns: int | None = None):
|
||
self.ctx = ctx
|
||
self.goal = goal
|
||
try:
|
||
cfg = load_agent_config()
|
||
self.max_turns = max_turns if max_turns is not None else cfg.flash.max_turns
|
||
self.step_summarizer_cfg = cfg.flash.step_summarizer
|
||
self.memory_runtime_cfg = cfg.memory.runtime
|
||
self.transcript_cfg = cfg.memory.transcript
|
||
self.chunking_cfg = cfg.memory.chunking
|
||
except Exception:
|
||
self.max_turns = max_turns if max_turns is not None else 0
|
||
self.step_summarizer_cfg = StepSummarizerConfig()
|
||
self.memory_runtime_cfg = MemoryRuntimeConfig()
|
||
self.transcript_cfg = MemoryTranscriptConfig()
|
||
self.chunking_cfg = None
|
||
|
||
self.controller = UnifiedMobileController(ctx)
|
||
# ``agent_name="flash"`` makes ask_explorer follow ``explorer.flash_mode``
|
||
# (the Flash profile knob) instead of the Pro profile's tier.
|
||
self.executor = McpActionExecutor(ctx, self.controller, agent_name="flash")
|
||
self.summarizer = (
|
||
VisualStepSummarizer(
|
||
ctx,
|
||
model_name=self.step_summarizer_cfg.model,
|
||
retry_limit=self.memory_runtime_cfg.retry_limit,
|
||
max_concurrency=self.memory_runtime_cfg.max_concurrency,
|
||
flush_timeout_s=self.memory_runtime_cfg.flush_timeout_s,
|
||
)
|
||
if self.step_summarizer_cfg.enabled
|
||
else None
|
||
)
|
||
# Publish the service on the composition root (ctx.step_memory, M2) so
|
||
# any co-resident consumer shares this run's summary runtime.
|
||
if self.summarizer is not None and getattr(ctx, "step_memory", None) is None:
|
||
try:
|
||
ctx.step_memory = self.summarizer
|
||
except (AttributeError, TypeError, ValueError):
|
||
pass
|
||
|
||
@property
|
||
def turn_limit(self) -> int | None:
|
||
"""The reactive turn cap, or ``None`` when the loop is unbounded."""
|
||
try:
|
||
limit = int(self.max_turns) if self.max_turns is not None else 0
|
||
except (TypeError, ValueError):
|
||
return None
|
||
return limit if limit > 0 else None
|
||
|
||
def _video_tools_enabled(self) -> bool:
|
||
"""Same gate the Pro graph applies before binding ``video_analyzer``."""
|
||
setup = getattr(self.ctx, "execution_setup", None)
|
||
return getattr(setup, "video_recording_tools_enabled", False) is True
|
||
|
||
def _get_tools(self) -> list:
|
||
tools = list(VALIDATOR_TOOLS_DECLARATION)
|
||
tools.insert(1, CLICK_SEQUENCE_TOOL)
|
||
tools.append(ASK_EXPLORER_TOOL)
|
||
# Helper tools shared with the Pro operator: the history tools are
|
||
# declared from the very same args schemas the LangChain tools bind
|
||
# (same availability gates: a DataEngine session, recall config).
|
||
tools.extend(history_tool_declarations(self.ctx))
|
||
if self._video_tools_enabled():
|
||
from artemis.tools.video_tool import VIDEO_ANALYZER_TOOL
|
||
|
||
tools.append(VIDEO_ANALYZER_TOOL)
|
||
tools.append(REPORT_TASK_STATUS_TOOL)
|
||
# With an actuator installed, drop declarations for device actions the
|
||
# backend does not implement (and append its extension tools); without
|
||
# one, the full historical declaration set is kept.
|
||
actuator = getattr(self.ctx, "actuator", None)
|
||
if actuator is not None:
|
||
from artemis.mcp.action_manifest import filter_declarations
|
||
|
||
tools = filter_declarations(tools, actuator, "flash")
|
||
return tools
|
||
|
||
def _prune_intermediate_screenshots(self, messages: list[BaseMessage]) -> None:
|
||
"""Prunes binary screenshot blocks from intermediate observation messages."""
|
||
prune_intermediate_screenshots(messages)
|
||
|
||
# ------------------------------------------------------------------
|
||
# run() setup helpers
|
||
# ------------------------------------------------------------------
|
||
|
||
def _build_ledger(self) -> TranscriptLedger:
|
||
"""Builds the session transcript ledger with the shared Pro history policy.
|
||
|
||
The ``T+mm:ss`` clock is anchored to the DataEngine session start so
|
||
the observation headers, the chunk ledger lines and the video
|
||
analyzer's action timeline share one origin. L2/L3 chunk compression
|
||
needs DataEngine step records; without an engine the ledger runs
|
||
scrub-edge-only.
|
||
"""
|
||
engine = getattr(self.ctx, "data_engine", None)
|
||
session_start = getattr(engine, "session_start_time", None) if engine else None
|
||
cfg = self.transcript_cfg
|
||
ledger = TranscriptLedger(
|
||
step_memory=self.summarizer,
|
||
prune_history_xml=self.step_summarizer_cfg.prune_history_xml,
|
||
image_scrub_depth=getattr(cfg, "image_scrub_depth", 3),
|
||
pending_grace_steps=getattr(cfg, "pending_grace_steps", 3),
|
||
xml_scrub_depth=getattr(cfg, "xml_scrub_depth", 1),
|
||
session_start=session_start if isinstance(session_start, (int, float)) else None,
|
||
)
|
||
if engine is not None:
|
||
try:
|
||
from artemis.memory import HistoryChunkManager
|
||
|
||
ledger.attach_chunker(
|
||
HistoryChunkManager(
|
||
engine=engine,
|
||
ctx=self.ctx,
|
||
chunking_config=self.chunking_cfg,
|
||
transcript_config=cfg,
|
||
goal=self.goal,
|
||
)
|
||
)
|
||
except Exception as e:
|
||
logger.error(f"History chunk manager unavailable for FlashRunner: {e}")
|
||
try:
|
||
self.ctx.transcript_ledger = ledger
|
||
except (AttributeError, TypeError, ValueError):
|
||
pass
|
||
return ledger
|
||
|
||
#: The model of the current run; decides how screenshot tool results travel.
|
||
_llm = None
|
||
|
||
def _init_llm(self):
|
||
"""Initializes the Universal LLM via the Service Layer."""
|
||
try:
|
||
return get_llm(self.ctx, name="operator")
|
||
except Exception as e:
|
||
logger.warning(f"Failed to get operator LLM from config, using default: {e}")
|
||
|
||
return RobustChatModelWrapper(get_google_llm(model_name="gemini-2.5-flash"), self.ctx)
|
||
|
||
def _render_system_prompt(self, tools_declaration: list) -> str:
|
||
"""Renders the system prompt from the flash_runner.md template.
|
||
|
||
Tool-teaching segments are gated on the available tool set so an absent
|
||
tool leaves no trace in the prompt; until an actuator is wired in, the
|
||
full manifest set reproduces the historical prompt.
|
||
"""
|
||
prompt_path = Path(__file__).parent / "flash_runner.md"
|
||
prompt_template = prompt_path.read_text(encoding="utf-8")
|
||
available_tools = frozenset(t.name for t in tools_declaration)
|
||
return Template(prompt_template).render(goal=self.goal, available_tools=available_tools)
|
||
|
||
# ------------------------------------------------------------------
|
||
# Per-turn helpers (observe / think)
|
||
# ------------------------------------------------------------------
|
||
|
||
async def _read_injected_instruction(self) -> tuple[str | None, str | None]:
|
||
"""Consumes this turn's injected instruction: ``(instruction, notice)``.
|
||
|
||
``instruction`` is the user's text verbatim — what the step record,
|
||
the chunk ledger and the visual-transition lens receive (the same
|
||
shape Pro stamps). ``notice`` wraps it with the operator-facing
|
||
directive and rides only on the observation tail. Both are None
|
||
when nothing was injected.
|
||
"""
|
||
if not (self.ctx.data_engine and self.ctx.data_engine.base_dir):
|
||
return None, None
|
||
try:
|
||
injected_payload = await asyncio.to_thread(
|
||
_check_injected_instruction_file,
|
||
str(self.ctx.data_engine.base_dir),
|
||
)
|
||
except (OSError, ValueError, AttributeError) as e:
|
||
logger.warning(f"Failed to check injected instruction in FlashRunner: {e}")
|
||
return None, None
|
||
if not (injected_payload and injected_payload.get("instruction")):
|
||
return None, None
|
||
instruction = str(injected_payload["instruction"])
|
||
notice = (
|
||
"[REAL-TIME INJECTED INSTRUCTION from user]:"
|
||
f" {instruction}\nYou MUST immediately"
|
||
" follow this instruction and adjust your plan/actions."
|
||
)
|
||
if injected_payload.get("release_loop"):
|
||
notice += (
|
||
"\nThe user has explicitly authorized stopping any"
|
||
" ongoing monitoring loop; you may now wrap up and"
|
||
" complete the task."
|
||
)
|
||
return instruction, notice
|
||
|
||
def _build_tail(
|
||
self,
|
||
ledger: TranscriptLedger,
|
||
turns: int,
|
||
img_bytes,
|
||
xml_list,
|
||
*,
|
||
injected: str | None = None,
|
||
notices: list[str] | None = None,
|
||
is_final: bool = False,
|
||
) -> HumanMessage:
|
||
"""Builds this turn's observation tail (Pro observation shape).
|
||
|
||
Header, screenshot and UI list carry the same markers as the Pro
|
||
operator's tail so the ledger's scrub edge treats both alike.
|
||
"""
|
||
blocks: list[dict] = []
|
||
if turns == 1:
|
||
blocks.append({"type": "text", "text": f"Your objective is: {self.goal}"})
|
||
blocks.append({"type": "text", "text": f"# CURRENT OBSERVATION [{ledger.elapsed_label()}]"})
|
||
if img_bytes:
|
||
img_b64 = base64.b64encode(img_bytes).decode("utf-8")
|
||
blocks.append({"type": "text", "text": "--- Current Screenshot ---"})
|
||
blocks.append(
|
||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}}
|
||
)
|
||
if xml_list:
|
||
blocks.append({"type": "text", "text": f"{PRO_UI_LIST_MARKER}\n{xml_list}"})
|
||
for notice in notices or []:
|
||
blocks.append({"type": "text", "text": notice})
|
||
if injected:
|
||
blocks.append({"type": "text", "text": injected})
|
||
if is_final:
|
||
blocks.append({"type": "text", "text": _FINAL_TURN_WARNING})
|
||
if turns == 1:
|
||
blocks.append({"type": "text", "text": _REASONING_FIRST_REMINDER})
|
||
return HumanMessage(content=blocks)
|
||
|
||
def _extract_response_text(self, response) -> str:
|
||
"""Extracts the natural-language thought text from the model response."""
|
||
raw_text = response.content if isinstance(response.content, str) else ""
|
||
if isinstance(response.content, list):
|
||
raw_text = "".join(
|
||
b.get("text", "") for b in response.content if isinstance(b, dict) and "text" in b
|
||
)
|
||
return raw_text
|
||
|
||
def _token_usage_from_response(self, response) -> dict | None:
|
||
"""Extracts token usage metadata from the model response, if present."""
|
||
step_token_usage = None
|
||
if hasattr(response, "usage_metadata") and response.usage_metadata:
|
||
u = response.usage_metadata
|
||
pr = u.get("input_tokens") or u.get("prompt_tokens") or 0
|
||
co = u.get("output_tokens") or u.get("completion_tokens") or 0
|
||
to = u.get("total_tokens") or (pr + co)
|
||
if to > 0:
|
||
step_token_usage = {
|
||
"prompt_tokens": int(pr),
|
||
"completion_tokens": int(co),
|
||
"total_tokens": int(to),
|
||
}
|
||
elif hasattr(response, "response_metadata") and isinstance(
|
||
response.response_metadata, dict
|
||
):
|
||
u = response.response_metadata.get("usage_metadata") or response.response_metadata.get(
|
||
"token_usage"
|
||
)
|
||
if isinstance(u, dict):
|
||
pr = (
|
||
u.get("input_tokens")
|
||
or u.get("prompt_tokens")
|
||
or u.get("prompt_token_count")
|
||
or 0
|
||
)
|
||
co = (
|
||
u.get("output_tokens")
|
||
or u.get("completion_tokens")
|
||
or u.get("candidates_token_count")
|
||
or 0
|
||
)
|
||
to = u.get("total_tokens") or u.get("total_token_count") or (pr + co)
|
||
if to > 0:
|
||
step_token_usage = {
|
||
"prompt_tokens": int(pr),
|
||
"completion_tokens": int(co),
|
||
"total_tokens": int(to),
|
||
}
|
||
return step_token_usage
|
||
|
||
def _estimate_token_usage(self, messages: list[BaseMessage], raw_text: str) -> dict:
|
||
"""Approximates token usage from message sizes when metadata is absent."""
|
||
calc_prompt_chars = 0
|
||
calc_images = 0
|
||
for msg in messages:
|
||
c = getattr(msg, "content", "")
|
||
if isinstance(c, str):
|
||
calc_prompt_chars += len(c)
|
||
elif isinstance(c, list):
|
||
for block in c:
|
||
if isinstance(block, dict):
|
||
if block.get("type") == "image_url" or "image_url" in block:
|
||
calc_images += 1
|
||
else:
|
||
calc_prompt_chars += len(str(block.get("text", "")))
|
||
else:
|
||
calc_prompt_chars += len(str(block))
|
||
prompt_tokens = (calc_prompt_chars // 4) + (calc_images * 258)
|
||
completion_tokens = len(raw_text) // 4
|
||
return {
|
||
"prompt_tokens": max(1, prompt_tokens),
|
||
"completion_tokens": max(1, completion_tokens),
|
||
"total_tokens": max(1, prompt_tokens + completion_tokens),
|
||
}
|
||
|
||
def _resolve_token_usage(self, response, messages: list[BaseMessage], raw_text: str) -> dict:
|
||
"""Extracts token usage from response metadata, estimating as fallback."""
|
||
step_token_usage = self._token_usage_from_response(response)
|
||
if not step_token_usage or step_token_usage.get("total_tokens", 0) <= 0:
|
||
step_token_usage = self._estimate_token_usage(messages, raw_text)
|
||
return step_token_usage
|
||
|
||
def _record_llm_trace(self, step_token_usage: dict, raw_text: str) -> None:
|
||
"""Records the llm_call trace for this turn in the DataEngine."""
|
||
if self.ctx.data_engine:
|
||
current_step_id = getattr(self.ctx.data_engine, "current_step_id", None)
|
||
self.ctx.data_engine.record_trace(
|
||
type="llm_call",
|
||
name="FlashRunner",
|
||
payload={"token_usage": step_token_usage, "response": raw_text},
|
||
step_id=current_step_id,
|
||
status="success",
|
||
)
|
||
|
||
def _resolve_tool_calls(self, response, raw_text: str) -> list:
|
||
"""Returns native tool calls, falling back to text parsing if absent."""
|
||
tool_calls = response.tool_calls or []
|
||
|
||
# Fallback text parsing if tool_calls not parsed natively
|
||
if not tool_calls and "```json" in raw_text:
|
||
parsed = parse_structured(raw_text)
|
||
if isinstance(parsed, ParseFailure):
|
||
logger.warning(
|
||
"FlashRunner response contained a JSON block that could"
|
||
f" not be parsed: {parsed.error}"
|
||
)
|
||
elif isinstance(parsed, dict) and "name" in parsed:
|
||
tool_calls = [
|
||
{
|
||
"name": parsed["name"],
|
||
"args": parsed.get("args", {}),
|
||
"id": str(uuid.uuid4()),
|
||
}
|
||
]
|
||
return tool_calls
|
||
|
||
# ------------------------------------------------------------------
|
||
# Per-tool-call helpers (act / record / report)
|
||
# ------------------------------------------------------------------
|
||
|
||
async def _finalize_task_report(
|
||
self,
|
||
name: str,
|
||
args: dict,
|
||
tc_id: str,
|
||
raw_text: str,
|
||
step_token_usage: dict,
|
||
pre_screenshot_bytes,
|
||
xml_list,
|
||
messages: list[BaseMessage],
|
||
) -> dict:
|
||
"""Records the final step, acknowledges the tool call, and flushes."""
|
||
final_report = args
|
||
if self.ctx.data_engine:
|
||
try:
|
||
if self.ctx.data_engine.current_step_id is None:
|
||
self.ctx.data_engine.allocate_step_id()
|
||
self.ctx.data_engine.record_step(
|
||
pre_screenshot_bytes=pre_screenshot_bytes,
|
||
ui_tree=xml_list,
|
||
action_taken={"action": "report_task_status", "args": args},
|
||
operator_raw_thinking=raw_text,
|
||
last_execution_result={"result": "Task completed with final report."},
|
||
extra_metadata={"token_usage": step_token_usage},
|
||
)
|
||
except Exception as step_err:
|
||
logger.warning(f"Error recording final step in FlashRunner: {step_err}")
|
||
messages.append(
|
||
ToolMessage(
|
||
tool_call_id=tc_id,
|
||
name=name,
|
||
content=json.dumps({"status": "acknowledged"}),
|
||
status="success",
|
||
)
|
||
)
|
||
if self.summarizer:
|
||
await self.summarizer.flush()
|
||
return final_report
|
||
|
||
async def _capture_post_screenshot(self, exec_result, name: str, action_names):
|
||
"""Returns the post-action screenshot, capturing a fallback if needed."""
|
||
post_img_bytes = exec_result.screenshot_bytes
|
||
if not post_img_bytes and name in action_names:
|
||
try:
|
||
controller = UnifiedMobileController(self.ctx)
|
||
screen_data = await controller.get_screen_data()
|
||
post_img_bytes = base64.b64decode(screen_data.base64)
|
||
if not exec_result.ui_elements_text:
|
||
exec_result.ui_elements_text = screen_data.elements
|
||
except Exception as shot_err:
|
||
logger.warning(f"Failed to capture fallback screenshot in FlashRunner: {shot_err}")
|
||
return post_img_bytes
|
||
|
||
def _extract_normalized_coordinates(self, name: str, args: dict):
|
||
"""Extracts and enriches coordinate metadata for the recorded action."""
|
||
norm_coords = None
|
||
norm_start = None
|
||
norm_end = None
|
||
if name == "swipe":
|
||
kind, target_val, _ = parse_swipe_parameters(args)
|
||
if kind == "coords" and isinstance(target_val, list) and len(target_val) == 4:
|
||
norm_coords = target_val
|
||
norm_start = target_val[:2]
|
||
norm_end = target_val[2:]
|
||
elif kind == "direction" and isinstance(target_val, str):
|
||
g_lower = target_val.lower()
|
||
if "up" in g_lower:
|
||
norm_coords = [600, 700, 600, 300]
|
||
elif "down" in g_lower:
|
||
norm_coords = [600, 300, 600, 700]
|
||
elif "left" in g_lower:
|
||
norm_coords = [750, 500, 250, 500]
|
||
elif "right" in g_lower:
|
||
norm_coords = [250, 500, 750, 500]
|
||
if norm_coords:
|
||
norm_start = norm_coords[:2]
|
||
norm_end = norm_coords[2:]
|
||
elif name in ("click", "tap", "long_press", "input_text"):
|
||
target = args.get("target") or args.get("coordinates")
|
||
if isinstance(target, (list, tuple)) and len(target) == 2:
|
||
norm_coords = list(target)
|
||
elif isinstance(target, str):
|
||
nums = re.findall(r"-?\d+(?:\.\d+)?", target)
|
||
if len(nums) == 2:
|
||
norm_coords = [int(float(nums[0])), int(float(nums[1]))]
|
||
return norm_coords, norm_start, norm_end
|
||
|
||
def _record_action_step(
|
||
self,
|
||
name: str,
|
||
args: dict,
|
||
exec_result,
|
||
raw_text: str,
|
||
step_token_usage: dict,
|
||
pre_screenshot_bytes,
|
||
xml_list,
|
||
post_img_bytes,
|
||
injected: str | None = None,
|
||
):
|
||
"""Records telemetry / step in DataEngine; returns the step id or None."""
|
||
recorded_step_id = None
|
||
try:
|
||
if self.ctx.data_engine.current_step_id is None:
|
||
self.ctx.data_engine.allocate_step_id()
|
||
|
||
norm_coords, norm_start, norm_end = self._extract_normalized_coordinates(name, args)
|
||
|
||
# Flash records the model's own 0–1000 coordinates verbatim; the
|
||
# explicit space marker keeps every later normalization pass
|
||
# (agent-friendly steps, MCP inspector) a no-op on this record.
|
||
action_dict = {
|
||
"action": name,
|
||
"coordinates": (
|
||
args.get("target")
|
||
or args.get("coordinates")
|
||
or args.get("sequence")
|
||
or norm_coords
|
||
),
|
||
COORDINATE_SPACE_KEY: COORDINATE_SPACE_NORMALIZED,
|
||
"args": args,
|
||
}
|
||
if norm_coords:
|
||
action_dict["normalized_coordinates"] = norm_coords
|
||
if norm_start and norm_end:
|
||
action_dict["normalized_start_coordinates"] = norm_start
|
||
action_dict["normalized_end_coordinates"] = norm_end
|
||
|
||
# The model's own statement of what it aimed at (target_description),
|
||
# validated and shaped by the executor into the fields Pro records.
|
||
action_dict.update((exec_result.metadata or {}).get("target_semantics") or {})
|
||
|
||
succeeded = exec_result.status == "success"
|
||
last_execution_result = {
|
||
"status": "dispatched" if succeeded else "failed",
|
||
"result": exec_result.text_summary,
|
||
}
|
||
if not succeeded:
|
||
last_execution_result["error"] = exec_result.text_summary
|
||
|
||
extra_metadata: dict = {"token_usage": step_token_usage}
|
||
if injected:
|
||
# Stamped verbatim on the step it reached: the chunk ledger
|
||
# keeps it as a never-evicted line at every compression level.
|
||
extra_metadata["injected_instruction"] = injected
|
||
|
||
recorded_step_id = self.ctx.data_engine.record_step(
|
||
pre_screenshot_bytes=pre_screenshot_bytes,
|
||
post_screenshot_bytes=post_img_bytes,
|
||
ui_tree=(exec_result.ui_elements_text or xml_list),
|
||
action_taken=action_dict,
|
||
operator_raw_thinking=raw_text,
|
||
last_execution_result=last_execution_result,
|
||
extra_metadata=extra_metadata,
|
||
)
|
||
self._notify_history_chunker(recorded_step_id)
|
||
except Exception as step_err:
|
||
logger.warning(f"Error recording step in FlashRunner: {step_err}")
|
||
return recorded_step_id
|
||
|
||
def _notify_history_chunker(self, step_id) -> None:
|
||
"""Stamps the recorded step for the chunk manager (single segment, no plan)."""
|
||
ledger = getattr(self.ctx, "transcript_ledger", None)
|
||
chunker = getattr(ledger, "chunker", None) if ledger is not None else None
|
||
if chunker is None or step_id is None:
|
||
return
|
||
try:
|
||
chunker.on_step_stamped(str(step_id), None)
|
||
except Exception as e:
|
||
logger.warning(f"History chunker step stamp failed: {e}")
|
||
|
||
async def _execute_and_record_action(
|
||
self,
|
||
name: str,
|
||
args: dict,
|
||
tc_id: str,
|
||
state: State,
|
||
messages: list[BaseMessage],
|
||
raw_text: str,
|
||
step_token_usage: dict,
|
||
pre_screenshot_bytes,
|
||
xml_list,
|
||
action_sequence: int,
|
||
turn: _TurnRecord,
|
||
injected: str | None = None,
|
||
):
|
||
"""Executes one tool call, records it, and updates loop state.
|
||
|
||
The tool message carries the outcome text only; the post-action
|
||
screenshot and UI list become the next turn's observation tail.
|
||
Returns the (possibly updated) pre_screenshot_bytes, xml_list, and
|
||
action_sequence for the next iteration.
|
||
"""
|
||
# Dynamic dispatch set: manifest device actions plus any backend
|
||
# extension tools, so extension steps are recorded like actions.
|
||
action_names = self.executor.action_tool_names
|
||
try:
|
||
exec_result = await self.executor.execute(name, args, tc_id, state)
|
||
|
||
post_img_bytes = await self._capture_post_screenshot(exec_result, name, action_names)
|
||
|
||
# Record telemetry / step in DataEngine
|
||
recorded_step_id = None
|
||
if self.ctx.data_engine and name in action_names:
|
||
recorded_step_id = self._record_action_step(
|
||
name,
|
||
args,
|
||
exec_result,
|
||
raw_text,
|
||
step_token_usage,
|
||
pre_screenshot_bytes,
|
||
xml_list,
|
||
post_img_bytes,
|
||
injected=injected,
|
||
)
|
||
|
||
if name in action_names:
|
||
turn.step_keys.append(str(recorded_step_id) if recorded_step_id else str(tc_id))
|
||
turn.actions.append((name, exec_result.status, exec_result.text_summary))
|
||
|
||
# ⚡ Non-blocking dispatch of objective visual transition summarizer
|
||
if self.summarizer and name in action_names:
|
||
action_sequence += 1
|
||
self.summarizer.dispatch(
|
||
step_number=action_sequence,
|
||
action_name=name,
|
||
action_args=args,
|
||
pre_img_bytes=pre_screenshot_bytes,
|
||
post_img_bytes=post_img_bytes,
|
||
exec_outcome=exec_result.text_summary,
|
||
action_key=str(tc_id),
|
||
data_engine_step_id=recorded_step_id,
|
||
# The model's own reasoning for this turn steers the lens's
|
||
# attention; the target it named rides on ``args``.
|
||
focus=build_focus_context(
|
||
intent=raw_text,
|
||
goal=self.goal,
|
||
injected_instruction=injected,
|
||
),
|
||
)
|
||
|
||
if post_img_bytes:
|
||
pre_screenshot_bytes = post_img_bytes
|
||
if exec_result.ui_elements_text:
|
||
xml_list = exec_result.ui_elements_text
|
||
|
||
# Helper tools may return multimodal blocks (a step screenshot);
|
||
# device actions always report text — their post screenshot is the
|
||
# next observation tail, never a tool-message image. The image
|
||
# carrier follows the model's provider (tool_result_messages).
|
||
raw_blocks = exec_result.raw_result if name not in action_names else None
|
||
content = (
|
||
raw_blocks
|
||
if isinstance(raw_blocks, list) and raw_blocks
|
||
else exec_result.text_summary or f"Action '{name}' dispatched."
|
||
)
|
||
messages.extend(
|
||
tool_result_messages(
|
||
tc_id, content, name=name, status=exec_result.status, llm=self._llm
|
||
)
|
||
)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Error executing tool {name}: {e}")
|
||
if name in action_names:
|
||
turn.step_keys.append(str(tc_id))
|
||
turn.actions.append((name, "error", f"Error executing tool {name}: {e}"))
|
||
messages.append(
|
||
ToolMessage(
|
||
tool_call_id=tc_id,
|
||
name=name,
|
||
content=f"Error executing tool {name}: {e}",
|
||
status="error",
|
||
)
|
||
)
|
||
return pre_screenshot_bytes, xml_list, action_sequence
|
||
|
||
async def _invoke_model(self, llm, current_tools: list, messages: list[BaseMessage]):
|
||
"""Binds the active tools and invokes the model through the LLM gateway."""
|
||
# Bind active tools
|
||
bound_llm = llm.bind_tools(current_tools)
|
||
|
||
# Invoke Model. Streaming, live-token UI deltas, classified
|
||
# retries, and pause/resume are all owned by the LLM gateway
|
||
# (acomplete); a typed LLMCallError propagates if it gives up.
|
||
return await invoke_llm_with_timeout_message(
|
||
acomplete(bound_llm, messages), timeout_seconds=10, hard_timeout=180
|
||
)
|
||
|
||
async def _process_tool_calls(
|
||
self,
|
||
tool_calls: list,
|
||
state: State,
|
||
messages: list[BaseMessage],
|
||
raw_text: str,
|
||
step_token_usage: dict,
|
||
pre_screenshot_bytes,
|
||
xml_list,
|
||
action_sequence: int,
|
||
turn: _TurnRecord,
|
||
injected: str | None = None,
|
||
):
|
||
"""Dispatches the turn's tool calls.
|
||
|
||
Returns (final_report, pre_screenshot_bytes, xml_list, action_sequence);
|
||
final_report is non-None only when 'report_task_status' was called.
|
||
"""
|
||
for tc in tool_calls:
|
||
name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
|
||
args = tc.get("args") or {}
|
||
tc_id = tc.get("id") or str(uuid.uuid4())
|
||
logger.info(f"Executing Flash tool: {name}({args})")
|
||
|
||
if name == "report_task_status":
|
||
final_report = await self._finalize_task_report(
|
||
name,
|
||
args,
|
||
tc_id,
|
||
raw_text,
|
||
step_token_usage,
|
||
pre_screenshot_bytes,
|
||
xml_list,
|
||
messages,
|
||
)
|
||
return final_report, pre_screenshot_bytes, xml_list, action_sequence
|
||
|
||
pre_screenshot_bytes, xml_list, action_sequence = await self._execute_and_record_action(
|
||
name,
|
||
args,
|
||
tc_id,
|
||
state,
|
||
messages,
|
||
raw_text,
|
||
step_token_usage,
|
||
pre_screenshot_bytes,
|
||
xml_list,
|
||
action_sequence,
|
||
turn,
|
||
injected=injected,
|
||
)
|
||
return None, pre_screenshot_bytes, xml_list, action_sequence
|
||
|
||
async def _prepare_conversation(self, state: State, tools_declaration: list):
|
||
"""Installs the static prefix and captures the initial device state."""
|
||
ledger = self._build_ledger()
|
||
ledger.set_static_prefix(
|
||
[SystemMessage(content=self._render_system_prompt(tools_declaration))]
|
||
)
|
||
|
||
# Capture Initial State (Screenshot + UI Tree)
|
||
shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
|
||
self.ctx, state, self.controller, skip_settling=False
|
||
)
|
||
state.latest_screenshot = shot_path
|
||
return ledger, img_bytes, xml_list
|
||
|
||
@staticmethod
|
||
def _commit_turn(ledger: TranscriptLedger, turn: _TurnRecord | None) -> None:
|
||
"""Commits the previous turn once its step ids and outcomes are known."""
|
||
if turn is None:
|
||
return
|
||
ledger.commit_staged(
|
||
step_key=turn.step_keys[0] if turn.step_keys else None,
|
||
validator_result=turn.result(),
|
||
extra_step_keys=turn.step_keys[1:],
|
||
)
|
||
|
||
@trace(type="agent", name="FlashRunner")
|
||
async def run(self, state: State) -> dict:
|
||
limit = self.turn_limit
|
||
logger.info(
|
||
f"Starting Artemis Flash reactive loop for goal: {self.goal}"
|
||
f" (turn limit: {limit if limit else 'unlimited'})"
|
||
)
|
||
|
||
# 1. Initialize Universal LLM via Service Layer
|
||
llm = self._init_llm()
|
||
self._llm = llm
|
||
|
||
tools_declaration = self._get_tools()
|
||
report_only_tools = [t for t in tools_declaration if t.name == "report_task_status"]
|
||
|
||
ledger, img_bytes, xml_list = await self._prepare_conversation(state, tools_declaration)
|
||
|
||
turns = 0
|
||
action_sequence = 0
|
||
final_report = None
|
||
current_pre_screenshot_bytes = img_bytes
|
||
current_xml_list = xml_list
|
||
previous_turn: _TurnRecord | None = None
|
||
pending_notices: list[str] = []
|
||
|
||
while limit is None or turns < limit:
|
||
turns += 1
|
||
logger.info(f"--- Artemis Flash Turn {turns}{f'/{limit}' if limit else ''} ---")
|
||
|
||
if self.ctx.data_engine:
|
||
self.ctx.data_engine.allocate_step_id()
|
||
|
||
# Commit the previous turn: its step ids and outcomes exist now.
|
||
self._commit_turn(ledger, previous_turn)
|
||
previous_turn = None
|
||
|
||
# Check for real-time injected instructions: the verbatim text
|
||
# is stamped on the step, the wrapped notice goes to the tail.
|
||
injected, injected_notice = await self._read_injected_instruction()
|
||
|
||
# Tool restriction on the final turn (bounded loops only)
|
||
is_final = limit is not None and turns == limit
|
||
tail = self._build_tail(
|
||
ledger,
|
||
turns,
|
||
current_pre_screenshot_bytes,
|
||
current_xml_list,
|
||
injected=injected_notice,
|
||
notices=pending_notices,
|
||
is_final=is_final,
|
||
)
|
||
pending_notices = []
|
||
|
||
# S + F + A + tail: chunk compression and the scrub edge advance here.
|
||
messages = ledger.render([tail])
|
||
turn_base = len(messages) - 1
|
||
current_tools = report_only_tools if is_final else tools_declaration
|
||
|
||
response = await self._invoke_model(llm, current_tools, messages)
|
||
|
||
if response is None:
|
||
break
|
||
|
||
messages.append(response)
|
||
|
||
# Extract thought and tool calls
|
||
raw_text = self._extract_response_text(response)
|
||
|
||
# Extract token usage metadata from response
|
||
step_token_usage = self._resolve_token_usage(response, messages, raw_text)
|
||
|
||
self._record_llm_trace(step_token_usage, raw_text)
|
||
|
||
tool_calls = self._resolve_tool_calls(response, raw_text)
|
||
|
||
if not tool_calls:
|
||
logger.info(
|
||
f"FlashRunner received response without tool calls at turn {turns}:"
|
||
f" {raw_text[:100]}..."
|
||
)
|
||
if is_final:
|
||
final_report = {"status": "failed", "explanation": raw_text}
|
||
break
|
||
pending_notices.append(_NO_TOOL_CALL_NOTICE)
|
||
ledger.stage_turn(messages[turn_base:])
|
||
previous_turn = _TurnRecord()
|
||
continue
|
||
|
||
# Process tool calls
|
||
turn = _TurnRecord()
|
||
(
|
||
final_report_from_calls,
|
||
current_pre_screenshot_bytes,
|
||
current_xml_list,
|
||
action_sequence,
|
||
) = await self._process_tool_calls(
|
||
tool_calls,
|
||
state,
|
||
messages,
|
||
raw_text,
|
||
step_token_usage,
|
||
current_pre_screenshot_bytes,
|
||
current_xml_list,
|
||
action_sequence,
|
||
turn,
|
||
injected=injected,
|
||
)
|
||
ledger.stage_turn(messages[turn_base:])
|
||
previous_turn = turn
|
||
if final_report_from_calls is not None:
|
||
return final_report_from_calls
|
||
|
||
if self.summarizer:
|
||
await self.summarizer.flush()
|
||
|
||
if final_report is None:
|
||
explanation = (
|
||
"Max turns reached without final status report."
|
||
if limit is not None and turns >= limit
|
||
else "The model returned no response; the reactive loop stopped."
|
||
)
|
||
final_report = {"status": "failed", "explanation": explanation}
|
||
return final_report
|