feat: unify history tools and improve execution reliability

artemis/agents/checker/checker.py

@@ -32,9 +32,9 @@ The release decision is computed by the caller from the verdicts
 from __future__ import annotations
 
 import asyncio
-import base64
 import json
 import re
+import time
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Literal
@@ -46,12 +46,15 @@ from pydantic import BaseModel, Field, ValidationError
 
 from artemis.constants import CHECKER_MAX_ITERATIONS
 from artemis.context import ArtemisContext
+from artemis.data_engine.context_vars import CURRENT_TRACE_ID
 from artemis.data_engine.trace import trace, trace_langchain_tool
 from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message
+from artemis.tools.history import get_history_tools
 from artemis.tools.scratchpad import get_read_note_tool_pure
 from artemis.tools.tool_wrapper import (
-    get_tool_result_content,
     invoke_tool_with_injection,
+    split_multimodal_result,
+    tool_result_messages,
 )
 from artemis.utils.logger import get_logger
 from artemis.utils.ocr_api import is_ocr_configured, perform_ocr
@@ -233,77 +236,6 @@ def get_probe_tool(ctx: ArtemisContext) -> BaseTool:
     return probe_device
 
 
-# --- History / step evidence tools ---------------------------------------------------
-
-
-def get_step_detail_tool(ctx: ArtemisContext) -> BaseTool:
-    @tool
-    def get_step_detail(step_no: int) -> str:
-        """Returns the recorded details of one execution step (action taken,
-        summary, metadata) by its step number."""
-        if not ctx.data_engine:
-            return "Error: no execution history available."
-        record = ctx.data_engine.get_step_record(step_no)
-        if record is None:
-            return f"Error: step {step_no} not found."
-        payload = {
-            "step_number": record.step_number,
-            "timestamp": record.timestamp,
-            "summary": record.summary,
-            "action_taken": record.action_taken,
-            "last_execution_result": record.last_execution_result,
-            "extra_metadata": record.extra_metadata,
-            "has_pre_screenshot": bool(record.pre_image_name),
-            "has_post_screenshot": bool(record.post_image_name),
-        }
-        try:
-            return json.dumps(payload, ensure_ascii=False, default=str)
-        except Exception as e:
-            return f"Error serializing step: {e}"
-
-    return get_step_detail
-
-
-class _ScreenshotRequest:
-    """Sentinel result carrying an image to splice into the conversation."""
-
-    def __init__(self, description: str, image_b64: str | None):
-        self.description = description
-        self.image_b64 = image_b64
-
-
-def get_step_screenshot_tool(ctx: ArtemisContext) -> BaseTool:
-    @tool
-    def get_step_screenshot(step_no: int, which: str = "pre") -> str:
-        """Attaches the recorded screenshot of a step to the conversation.
-        `which` is 'pre' (state observed at the start of the step) or 'post'
-        (state after the step's action)."""
-        # The real work happens in the loop's interception; this body only
-        # documents the contract for schema generation.
-        return f"screenshot request: step {step_no} ({which})"
-
-    return get_step_screenshot
-
-
-def _load_step_screenshot(ctx: ArtemisContext, step_no: int, which: str) -> _ScreenshotRequest:
-    if which not in ("pre", "post"):
-        return _ScreenshotRequest(f"Error: 'which' must be 'pre' or 'post', got '{which}'.", None)
-    if not ctx.data_engine:
-        return _ScreenshotRequest("Error: no execution history available.", None)
-    path = ctx.data_engine.get_step_image_path(step_no, which)
-    if path is None:
-        return _ScreenshotRequest(
-            f"No {which}-action screenshot recorded for step {step_no}.", None
-        )
-    try:
-        image_b64 = base64.b64encode(Path(path).read_bytes()).decode("utf-8")
-    except Exception as e:
-        return _ScreenshotRequest(f"Error reading screenshot: {e}", None)
-    return _ScreenshotRequest(
-        f"Screenshot of step {step_no} ({which}-action) is attached below.", image_b64
-    )
-
-
 # --- Prompt assembly (segments x entry x content x config) ---------------------------
 
 
@@ -365,10 +297,12 @@ def build_checker_tools(
     the final screen through its prompt (not a tool), so the table is identical
     for both entries; the difference lives in what the prompt provides.
     Device actions, note writes, and sub-agents are registered for neither.
+
+    The history tools are the shared ``search_history`` / ``replay_steps`` /
+    ``get_step_screenshot`` definitions (read-only, deterministic).
     """
     tools: list[BaseTool] = [
-        get_step_detail_tool(ctx),
-        get_step_screenshot_tool(ctx),
+        *get_history_tools(ctx),
         get_read_note_tool_pure(ctx),
     ]
     if probes_enabled(ctx):
@@ -507,37 +441,6 @@ async def _run_check_loop(
         for tc in response.tool_calls:
             tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
             args = dict(tc["args"])
-            if tool_name == "get_step_screenshot":
-                shot = _load_step_screenshot(
-                    ctx, int(args.get("step_no", -1)), str(args.get("which", "pre"))
-                )
-                # The image is spliced in below; invoking the (traced) tool as
-                # well records the lookup as a tool trace so the timeline shows
-                # it in order with the rest of the check.
-                screenshot_tool = next((t for t in traced_tools if t.name == tool_name), None)
-                if screenshot_tool is not None:
-                    try:
-                        await invoke_tool_with_injection(
-                            tool=screenshot_tool, args=args, tool_call_id=tc["id"]
-                        )
-                    except Exception as e:
-                        logger.debug(f"Screenshot lookup trace not recorded: {e}")
-                messages.append(ToolMessage(tool_call_id=tc["id"], content=shot.description))
-                if shot.image_b64:
-                    messages.append(
-                        HumanMessage(
-                            content=[
-                                {
-                                    "type": "image_url",
-                                    "image_url": {
-                                        "url": f"data:image/jpeg;base64,{shot.image_b64}"
-                                    },
-                                }
-                            ]
-                        )
-                    )
-                continue
-
             tool_to_run = next((t for t in traced_tools if t.name == tool_name), None)
             if tool_to_run is None:
                 messages.append(
@@ -552,18 +455,136 @@ async def _run_check_loop(
                 result_obj = await invoke_tool_with_injection(
                     tool=tool_to_run, args=args, tool_call_id=tc["id"]
                 )
-                result_str = get_tool_result_content(result_obj)
-                status = "error" if str(result_str).startswith("Error") else "success"
+                text, _ = split_multimodal_result(result_obj)
+                status = "error" if text.startswith("Error") else "success"
             except Exception as e:
-                result_str = f"Error running tool {tool_name}: {e}"
+                result_obj = f"Error running tool {tool_name}: {e}"
                 status = "error"
-            messages.append(ToolMessage(tool_call_id=tc["id"], content=result_str, status=status))
+            # Screenshots (get_step_screenshot) enter the conversation in the
+            # carrier the model's provider accepts; text results stay a plain
+            # ToolMessage.
+            messages.extend(
+                tool_result_messages(tc["id"], result_obj, name=tool_name, status=status, llm=llm)
+            )
 
     if report is None:
         report = CheckReport(verdicts=[])
     return _normalize_report(report, check_items)
 
 
+# --- Streamed-reasoning transcript -----------------------------------------------------
+
+
+class CheckerStreamCapture:
+    """Collects the Checker's own streamed LLM output for replay.
+
+    The LLM gateway forwards every provider delta to the DataEngine bus as an
+    ``llm_stream`` event carrying the emitting agent's trace id
+    (``parent_trace_id``). Inside the Checker's traced scope that id is the
+    Checker's own, so filtering the bus by it yields exactly the text the live
+    UI rendered for this attempt — one segment per (LLM turn, stream kind),
+    stamped with the time its first chunk arrived. A mid-stream reset
+    (``llm_stream_reset`` / ``discard``) drops that turn's partial text, the
+    same way the live UI does.
+    """
+
+    def __init__(self, ctx: ArtemisContext):
+        self._engine = getattr(ctx, "data_engine", None)
+        try:
+            trace_id = CURRENT_TRACE_ID.get()
+        except LookupError:
+            trace_id = None
+        self.trace_id: str | None = str(trace_id) if trace_id else None
+        self._segments: dict[tuple[str, str], dict] = {}
+        self._subscribed = False
+
+    def __enter__(self) -> CheckerStreamCapture:
+        subscribe = getattr(self._engine, "subscribe", None)
+        if self.trace_id and callable(subscribe):
+            try:
+                subscribe(self._on_event)
+                self._subscribed = True
+            except Exception as e:
+                logger.debug(f"Checker stream capture not attached: {e}")
+        return self
+
+    def __exit__(self, *exc) -> None:
+        if not self._subscribed:
+            return
+        self._subscribed = False
+        try:
+            self._engine.unsubscribe(self._on_event)
+        except Exception as e:
+            logger.debug(f"Checker stream capture not detached: {e}")
+
+    def _on_event(self, event_type: str, data: Any) -> None:
+        if not isinstance(data, dict):
+            return
+        if event_type == "llm_stream":
+            if str(data.get("parent_trace_id") or "") != self.trace_id:
+                return
+            exec_id = str(data.get("execution_id") or "")
+            chunk = data.get("chunk")
+            if not exec_id or not isinstance(chunk, str) or not chunk:
+                return
+            role = "thought" if data.get("stream_type") == "thinking" else "answer"
+            segment = self._segments.get((exec_id, role))
+            if segment is None:
+                self._segments[(exec_id, role)] = {
+                    "execution_id": exec_id,
+                    "role": role,
+                    "when": time.time(),
+                    "text": chunk,
+                }
+            else:
+                segment["text"] += chunk
+        elif event_type == "llm_stream_reset" and data.get("action") == "discard":
+            exec_id = str(data.get("stream_exec_id") or data.get("stream_execution_id") or "")
+            if exec_id:
+                for key in [k for k in self._segments if k[0] == exec_id]:
+                    del self._segments[key]
+
+    @property
+    def segments(self) -> list[dict]:
+        """Segments in the order their first chunk arrived."""
+        return [dict(s) for s in sorted(self._segments.values(), key=lambda s: s["when"])]
+
+
+def _persist_stream(
+    ctx: ArtemisContext,
+    capture: CheckerStreamCapture,
+    *,
+    attempt_id: str | None,
+    phase: str,
+    checkpoint_id: str,
+) -> None:
+    """Writes the attempt's transcript next to the ledger (best-effort).
+
+    Runs when the attempt ends for any reason — verdict, exception, timeout or
+    supersession — so a cancelled attempt still leaves what it said. A
+    transcript is observability, not execution state: writing it never
+    influences booking or release (the ledger stays the source of truth).
+    """
+    segments = capture.segments
+    if not attempt_id or not segments:
+        return
+    try:
+        from artemis.graph.checkpoints import append_attempt_stream_record
+
+        append_attempt_stream_record(
+            ctx.data_engine.base_dir,
+            {
+                "attempt_id": attempt_id,
+                "checkpoint_id": checkpoint_id,
+                "phase": phase,
+                "trace_id": capture.trace_id,
+                "segments": segments,
+            },
+        )
+    except Exception as e:
+        logger.debug(f"Checker stream transcript for {attempt_id} not persisted: {e}")
+
+
 # --- Entry points --------------------------------------------------------------------
 
 
@@ -593,12 +614,13 @@ async def run_checkpoint_check(
     state) and persistent-state probes are the only valid sources.
     """
     items = list(check_items)
+    checkpoint_id = attempt_id.split("#", 1)[0] if attempt_id else ""
     logger.info(f"Starting checkpoint check for '{subgoal_text}' ({len(items)} item(s))")
     _announce(
         ctx,
         attempt_id=attempt_id,
         phase="checkpoint",
-        checkpoint_id=attempt_id.split("#", 1)[0] if attempt_id else "",
+        checkpoint_id=checkpoint_id,
         subgoal_text=subgoal_text,
         anchor_step_id=getattr(anchor, "anchor_step_id", None),
         check_items=items,
@@ -638,7 +660,17 @@ async def run_checkpoint_check(
         SystemMessage(content=system_content),
         HumanMessage(content=human_text),
     ]
-    return await _run_check_loop(ctx, messages, tools, items)
+    with CheckerStreamCapture(ctx) as capture:
+        try:
+            return await _run_check_loop(ctx, messages, tools, items)
+        finally:
+            _persist_stream(
+                ctx,
+                capture,
+                attempt_id=attempt_id,
+                phase="checkpoint",
+                checkpoint_id=checkpoint_id,
+            )
 
 
 async def _capture_final_screen(ctx: ArtemisContext) -> tuple[str | None, str]:
@@ -748,4 +780,10 @@ async def run_final_check(
         SystemMessage(content=system_content),
         HumanMessage(content=content),
     ]
-    return await _run_check_loop(ctx, messages, tools, items)
+    with CheckerStreamCapture(ctx) as capture:
+        try:
+            return await _run_check_loop(ctx, messages, tools, items)
+        finally:
+            _persist_stream(
+                ctx, capture, attempt_id=attempt_id, phase="final", checkpoint_id="final"
+            )

artemis/agents/diagnoser/diagnoser.py

@@ -33,6 +33,7 @@ from artemis.graph.state import State
 from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message
 from artemis.tools.command_tool import get_run_short_adb_command_tool
 from artemis.tools.diagnoser_submit_answer_tool import get_submit_answer_tool
+from artemis.tools.history import get_history_tools
 from artemis.tools.index import get_tool_by_name
 from artemis.tools.log_tool import get_analyze_logs_tool
 from artemis.tools.mobile.read_hierarchy import get_ui_hierarchy_tool
@@ -40,6 +41,8 @@ from artemis.tools.scratchpad import get_list_notes_tool, get_read_note_tool
 from artemis.tools.tool_wrapper import (
     get_tool_result_content,
     invoke_tool_with_injection,
+    split_multimodal_result,
+    tool_result_messages,
 )
 from artemis.tools.video_tool import get_video_analyzer_tool
 from artemis.tools.wait_tool import get_wait_tool
@@ -63,6 +66,8 @@ class Diagnoser:
     def __init__(self, ctx: ArtemisContext):
         self.ctx = ctx
         self.is_device_online = self._is_device_online()
+        #: The model of the current run; decides how screenshot results travel.
+        self._llm = None
 
     def _is_device_online(self) -> bool:
         try:
@@ -101,6 +106,7 @@ class Diagnoser:
             get_run_short_adb_command_tool(self.ctx),
             get_wait_tool(self.ctx),
             get_ui_hierarchy_tool(self.ctx),
+            *get_history_tools(self.ctx),
             get_submit_answer_tool(self.ctx),
         ]
         if not self.is_device_online:
@@ -452,8 +458,13 @@ class Diagnoser:
             status="success",
         )
 
-    async def _run_tool(self, tc: dict, traced_tools: list, state: State) -> ToolMessage:
-        """Executes one requested tool call and wraps its result as a tool message."""
+    async def _run_tool(self, tc: dict, traced_tools: list, state: State) -> list[BaseMessage]:
+        """Executes one requested tool call and wraps its result as messages.
+
+        Text results are one tool message; a step screenshot travels in the
+        carrier the model's provider accepts (``tool_result_messages``), so
+        base64 never lands in a text field.
+        """
         tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
         logger.info(f"Diagnoser requested tool: {tool_name}")
 
@@ -461,7 +472,7 @@ class Diagnoser:
         status = "success"
         try:
             if tool_name == "video_analyzer":
-                return self._start_video_job(tc, traced_tools, state)
+                return [self._start_video_job(tc, traced_tools, state)]
 
             if ":" in tool_name:
                 tool_to_run = get_tool_by_name(tool_name, traced_tools)
@@ -482,12 +493,7 @@ class Diagnoser:
                             state=state,
                         )
                         result = get_tool_result_content(result_obj)
-                        if isinstance(result, list):
-                            result_str = "\n".join(map(str, result))
-                        elif not isinstance(result, str):
-                            result_str = str(result)
-                        else:
-                            result_str = result
+                        result_str, _ = split_multimodal_result(result)
                         span.result = result
                         if result_str.startswith("Error"):
                             status = "error"
@@ -506,26 +512,21 @@ class Diagnoser:
             result = f"Error running tool {tool_name}: {e}"
             status = "error"
 
-        return ToolMessage(
-            tool_call_id=tc["id"],
-            content=result,
-            status=status,
-        )
+        return tool_result_messages(tc["id"], result, name=tool_name, status=status, llm=self._llm)
 
     async def _execute_tool_calls(
         self, tool_calls: list, traced_tools: list, state: State
-    ) -> list[ToolMessage]:
+    ) -> list[BaseMessage]:
         """Executes all requested tools in parallel, skipping ``google_search``."""
         active_tool_calls = [
             tc
             for tc in tool_calls
             if (tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]) != "google_search"
         ]
-        return list(
-            await asyncio.gather(
-                *(self._run_tool(tc, traced_tools, state) for tc in active_tool_calls)
-            )
+        batches = await asyncio.gather(
+            *(self._run_tool(tc, traced_tools, state) for tc in active_tool_calls)
         )
+        return [message for batch in batches for message in batch]
 
     @trace(type="agent", name="diagnoser")
     async def run(self, prompt: str, state: State) -> str:
@@ -534,6 +535,7 @@ class Diagnoser:
         system_prompt, checker_feedback_template = self._load_prompt_config()
 
         base_llm = get_llm(ctx=self.ctx, name="diagnoser")
+        self._llm = base_llm
         traced_tools = self._build_traced_tools()
 
         plan_and_history = self._build_plan_and_history()

artemis/agents/flash/context_compressor.py

@@ -12,19 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Universal Context Compressor for Artemis Flash profile.
+"""Context compression shared by Flash and Pro.
 
-:class:`ScrubEdgeCompressor` implements the scrub-edge (擦洗沿) discipline
-from the history-module redesign §3.2: each message is mutated a bounded
-number of times near the tail (XML stripped at depth 1, screenshot resolved
-at depth K), then frozen forever. A late summary never backfills a frozen
+:class:`ScrubEdgeCompressor` strips XML at depth 1 and resolves screenshots
+at depth K, then freezes the message. A late summary never backfills a frozen
 message. FlashRunner uses this implementation; the Pro transcript ledger
 reuses it over its active region (see the class docstring).
-
-The legacy per-turn full-rescan ``compress_flash_messages`` was removed in M5
-(2026-09-01) after serving as the M1/M2 semantic reference — its surviving
-semantics are asserted directly against ``ScrubEdgeCompressor`` in
-``tests/unit/agents/test_flash_scrub_edge.py``.
 """
 
 from typing import Any, Callable
@@ -36,7 +29,10 @@ from artemis.utils.logger import get_logger
 
 logger = get_logger(__name__)
 
-_HISTORY_SUMMARY_PREFIX = "--- Historical Visual Transition ---\n"
+#: Header written above a resolved visual summary (public: the chunk capsule
+#: lens checks it to avoid repeating a summary already present verbatim).
+HISTORY_SUMMARY_PREFIX = "--- Historical Visual Transition ---\n"
+_HISTORY_SUMMARY_PREFIX = HISTORY_SUMMARY_PREFIX
 _UI_LIST_MARKER = "--- UI Element List ---"
 
 
@@ -53,7 +49,7 @@ def _without_marked_suffix(text: str, markers: tuple[str, ...] = (_UI_LIST_MARKE
 
 
 class ScrubEdgeCompressor:
-    """Scrub-edge context compressor (history-module redesign §3.2, M1).
+    """Replace older observation details with summaries near the message tail.
 
     Mutation discipline — every message is touched at most twice, always near
     the tail, and is then frozen forever:
@@ -75,8 +71,8 @@ class ScrubEdgeCompressor:
     touches the few messages near the scrub edge instead of rescanning the
     whole history.
 
-    Profile generalization (M2): the Pro transcript ledger reuses this exact
-    discipline over its active region. Its observation messages carry no
+    The Pro transcript ledger uses this discipline over its active region.
+    Its observation messages carry no
     ``tool_call_id``, so ``summary_key_getter`` supplies the summary-job key
     (the DataEngine step id) per message; ``strip_markers`` extends the
     depth-1 strip to the Pro tail blocks (UI element list + plan recitation);

artemis/agents/flash/flash_runner.md

@@ -25,7 +25,9 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
 # 3. READING YOUR HISTORY & THE SESSION CLOCK
 - **Conversation = execution history**: The earlier turns of this conversation are your raw execution history. Each observation opens with `# CURRENT OBSERVATION [T+mm:ss]`, and each turn that executed a device action ends with an `--- Action Execution Result (T+mm:ss) ---` message (`Status: success` means the action was dispatched and observed; `Status: failed` carries the `Error:` line). Every timestamp is a session-relative `T+mm:ss` offset from the start of this session; it is the only clock you have, for pacing waits as well as for reasoning about elapsed time.
 - **Compression**: Screenshots older than a few turns are replaced by short `--- Historical Visual Transition ---` summaries and old `--- Visible UI Elements ---` lists are dropped. Long spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks (a merged synopsis, segment titles, and a per-step action ledger). Because of this, if you spot critical text that you will need later (e.g., verification codes, specific titles, search lists), **you must explicitly verbalize and write those details into your text reasoning** so they persist across turns.
-{% if "recall_history" in available_tools %}- **Cold-History Recall (`recall_history`)**: Compressed history keeps the step numbers and `T+` offsets but not every detail. When you need exactly what you did or saw at an earlier step (a value you read, the result of an action, which screen you were on), call `recall_history` with keywords and/or the `[start, end]` step range shown in a chunk block; it searches the full stored history (summaries, actions, results, reasoning, on-screen text). It is a pre-decision helper: read its answer, then act.{% endif %}
+{% if "search_history" in available_tools %}- **History Lookup (`search_history`)**: Compressed history keeps the step numbers and `T+` offsets but not every detail. When you need exactly what you did or saw at an earlier step (a value you read, the result of an action, which screen you were on), call `search_history` with keywords and/or the `[start, end]` step range shown in a chunk block; it searches the full stored history (screen descriptions, actions, results, reasoning, tool calls, notes, on-screen text) and every hit carries its step number. It is a pre-decision helper: read its answer, then act.{% endif %}
+{% if "replay_steps" in available_tools %}- **Step Replay (`replay_steps`)**: Replays one earlier step (or a short range) exactly as your context showed it — screen description, reasoning, tool calls with their results, the action and its execution result. Use the step numbers shown in the compressed history{% if "search_history" in available_tools %} or returned by `search_history`{% endif %}.{% endif %}
+{% if "get_step_screenshot" in available_tools %}- **Step Screenshot (`get_step_screenshot`)**: Attaches the stored screenshot of an earlier step (which: pre, post, or overlay — the pre screenshot with the action drawn on it) when you need to see that screen again rather than read about it.{% endif %}
 {% if "video_analyzer" in available_tools %}- **Video Recall (`video_analyzer`)**: The whole session is continuously screen-recorded on the same `T+` clock. When a static screenshot is not enough — a video or animation that played, something that changed between two observations, a transient toast, a fast-scrolling list — call `video_analyzer` with a time range expressed on that clock (e.g., `from T+01:20 to T+01:35`, or `from 80s to 95s`) and a precise question. Prefer narrow ranges; ask for audio explicitly when only audio matters. It is a pre-decision helper: read its answer, then act.{% endif %}
 
 ---

artemis/agents/flash/runner.py

@@ -74,7 +74,13 @@ from artemis.services.llm import (
     get_llm,
     invoke_llm_with_timeout_message,
 )
-from artemis.utils.coordinates import parse_swipe_parameters
+from artemis.tools.history import history_tool_declarations
+from artemis.tools.tool_wrapper import tool_result_messages
+from artemis.utils.coordinates import (
+    COORDINATE_SPACE_KEY,
+    COORDINATE_SPACE_NORMALIZED,
+    parse_swipe_parameters,
+)
 from artemis.utils.logger import get_logger
 
 logger = get_logger(__name__)
@@ -178,25 +184,14 @@ class FlashRunner:
         setup = getattr(self.ctx, "execution_setup", None)
         return getattr(setup, "video_recording_tools_enabled", False) is True
 
-    def _recall_enabled(self) -> bool:
-        """Same gate as the Pro operator: a DataEngine session and the recall config."""
-        from artemis.tools.history_recall import _recall_available
-
-        try:
-            return bool(_recall_available(self.ctx))
-        except Exception:
-            return False
-
     def _get_tools(self) -> list:
         tools = list(VALIDATOR_TOOLS_DECLARATION)
         tools.insert(1, CLICK_SEQUENCE_TOOL)
         tools.append(ASK_EXPLORER_TOOL)
-        # Helper tools shared with the Pro operator, declared next to their
-        # LangChain tools so each contract has one owner.
-        if self._recall_enabled():
-            from artemis.tools.history_recall import RECALL_HISTORY_TOOL
-
-            tools.append(RECALL_HISTORY_TOOL)
+        # Helper tools shared with the Pro operator: the history tools are
+        # declared from the very same args schemas the LangChain tools bind
+        # (same availability gates: a DataEngine session, recall config).
+        tools.extend(history_tool_declarations(self.ctx))
         if self._video_tools_enabled():
             from artemis.tools.video_tool import VIDEO_ANALYZER_TOOL
 
@@ -261,6 +256,9 @@ class FlashRunner:
             pass
         return ledger
 
+    #: The model of the current run; decides how screenshot tool results travel.
+    _llm = None
+
     def _init_llm(self):
         """Initializes the Universal LLM via the Service Layer."""
         try:
@@ -579,6 +577,9 @@ class FlashRunner:
 
             norm_coords, norm_start, norm_end = self._extract_normalized_coordinates(name, args)
 
+            # Flash records the model's own 0–1000 coordinates verbatim; the
+            # explicit space marker keeps every later normalization pass
+            # (agent-friendly steps, MCP inspector) a no-op on this record.
             action_dict = {
                 "action": name,
                 "coordinates": (
@@ -587,6 +588,7 @@ class FlashRunner:
                     or args.get("sequence")
                     or norm_coords
                 ),
+                COORDINATE_SPACE_KEY: COORDINATE_SPACE_NORMALIZED,
                 "args": args,
             }
             if norm_coords:
@@ -709,21 +711,19 @@ class FlashRunner:
             if exec_result.ui_elements_text:
                 xml_list = exec_result.ui_elements_text
 
-            # Helper tools may return multimodal blocks (recalled screenshots);
+            # Helper tools may return multimodal blocks (a step screenshot);
             # device actions always report text — their post screenshot is the
-            # next observation tail, never a tool-message image.
+            # next observation tail, never a tool-message image. The image
+            # carrier follows the model's provider (tool_result_messages).
             raw_blocks = exec_result.raw_result if name not in action_names else None
             content = (
                 raw_blocks
                 if isinstance(raw_blocks, list) and raw_blocks
                 else exec_result.text_summary or f"Action '{name}' completed."
             )
-            messages.append(
-                ToolMessage(
-                    tool_call_id=tc_id,
-                    name=name,
-                    content=content,
-                    status=exec_result.status,
+            messages.extend(
+                tool_result_messages(
+                    tc_id, content, name=name, status=exec_result.status, llm=self._llm
                 )
             )
 
@@ -842,6 +842,7 @@ class FlashRunner:
 
         # 1. Initialize Universal LLM via Service Layer
         llm = self._init_llm()
+        self._llm = llm
 
         tools_declaration = self._get_tools()
         report_only_tools = [t for t in tools_declaration if t.name == "report_task_status"]

artemis/agents/history_analyzer/history_analyzer.md

@@ -3,6 +3,6 @@ You are a **History Analyzer**, expert in analyzing multi-agent session executio
 
 # MANDATE
 1. You are provided with the **Task Plan and Execution History** summarizing the high-level plan and history steps.
-2. For queries about specific step details (such as precise actions, coordinates, `operator_raw_thinking`, or `last_execution_result`), you should call the `get_step_details` tool.
-3. Never assume, guess, or invent details not visible in the Task Plan and Execution History. Always query `get_step_details` for specifics. If details are unavailable, state it clearly.
+2. For queries about specific step details (such as precise actions, coordinates, the reasoning behind a step, or its execution result), call `replay_steps` for the step or range in question; use `search_history` to locate the right steps by keyword when the step numbers are not obvious, and `get_step_screenshot` when you need to see what the screen showed.
+3. Never assume, guess, or invent details not visible in the Task Plan and Execution History. Always query `replay_steps` for specifics. If details are unavailable, state it clearly.
 4. Make good use of note tools to see what notes are available. When analyzing history records, open and cross-reference relevant notes to ensure you have grasped all necessary information.

artemis/agents/history_analyzer/history_analyzer.py

@@ -12,83 +12,58 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
+"""History Analyzer: answers natural-language questions about the session history.
+
+Mounts the shared history tools (``search_history`` / ``replay_steps`` /
+``get_step_screenshot``) plus the read-only note tools; tool results enter the
+conversation through ``tool_result_messages`` so a fetched screenshot travels
+in the carrier the model's provider accepts.
+"""
+
 from pathlib import Path
-from typing import Any
+
+from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
+from langchain_core.tools import BaseTool
 
-from langchain_core.messages import (
-    BaseMessage,
-    HumanMessage,
-    SystemMessage,
-    ToolMessage,
-)
-from langchain_core.tools import BaseTool, tool
 from artemis.context import ArtemisContext
-from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace
+from artemis.data_engine.trace import TraceSpan, trace
+from artemis.memory.context_policy import build_history_for
 from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message
+from artemis.tools.history import get_history_tools
 from artemis.tools.scratchpad import (
     get_list_notes_tool_pure,
     get_read_note_tool_pure,
 )
-from artemis.utils.logger import get_logger
-from artemis.utils.notes import (
-    get_note_file_path,
-    get_notes_dir,
+from artemis.tools.tool_wrapper import (
+    invoke_tool_with_injection,
+    split_multimodal_result,
+    tool_result_messages,
 )
-from artemis.memory.context_policy import build_history_for
+from artemis.utils.logger import get_logger
+from artemis.utils.notes import get_note_file_path
 from artemis.utils.task_tree import get_active_subgoal_hashes
 
 logger = get_logger(__name__)
 
+_FALLBACK_SYSTEM_PROMPT = (
+    "You are a History Analyzer. Your role is to analyze the execution history"
+    " of a session and answer user queries in natural language.\nIf you need"
+    " the full record of specific steps, use the `replay_steps` tool; use"
+    " `search_history` to locate steps by keyword and `get_step_screenshot` to"
+    " look at a step's screen."
+)
+
 
 class HistoryAnalyzer:
     def __init__(self, ctx: ArtemisContext):
         self.ctx = ctx
 
-    def exec_get_step_details(self, start_step: int, end_step: int) -> str:
-        """Retrieve the detailed, full-granularity information for a range of steps (inclusive).
-
-        Use this when you need to inspect specific actions taken, operator
-        thinking, or execution results.
-        """
-        try:
-            s_step = int(start_step)
-            e_step = int(end_step)
-        except (ValueError, TypeError):
-            return (
-                f"Error: start_step and end_step must be integers, got {start_step} and {end_step}."
-            )
-
-        matched_steps = []
-        for s in getattr(self, "history_steps", []):
-            step_num = s.get("step_number")
-            if step_num is not None and s_step <= step_num <= e_step:
-                details = {
-                    "step_number": s.get("step_number"),
-                    "relative_time": s.get("relative_time"),
-                    "summary": s.get("summary"),
-                    "action_taken": s.get("action_taken"),
-                    "operator_raw_thinking": s.get("operator_raw_thinking"),
-                    "last_execution_result": s.get("last_execution_result"),
-                }
-                matched_steps.append(details)
-        if not matched_steps:
-            return f"No steps found in range [{s_step}, {e_step}]."
-        return json.dumps(matched_steps, indent=2, ensure_ascii=False)
-
-    def _get_step_details_tool(self, history_steps: list[dict[str, Any]]) -> BaseTool:
-        self.history_steps = history_steps
-
-        @tool
-        def get_step_details(start_step: int, end_step: int) -> str:
-            """Retrieve the detailed, full-granularity information for a range of steps (inclusive).
-
-            Use this when you need to inspect specific actions taken, operator
-            thinking, or execution results.
-            """
-            return self.exec_get_step_details(start_step, end_step)
-
-        return get_step_details
+    def _build_tools(self) -> list[BaseTool]:
+        return [
+            *get_history_tools(self.ctx),
+            get_list_notes_tool_pure(self.ctx),
+            get_read_note_tool_pure(self.ctx),
+        ]
 
     @trace(type="agent", name="history_analyzer")
     async def run(self, query: str) -> str:
@@ -96,14 +71,13 @@ class HistoryAnalyzer:
             return "Error: DataEngine is not available to retrieve history."
 
         # 1. Fetch all steps from Data Engine
-        self.history_steps = self.ctx.data_engine.get_agent_friendly_steps()
-        if not self.history_steps:
+        history_steps = self.ctx.data_engine.get_agent_friendly_steps()
+        if not history_steps:
             return "No history recorded for this session yet."
 
         # 2. Build the plan and history or step list
         plan_and_history = ""
         try:
-            notes_dir = get_notes_dir(self.ctx.data_engine.base_dir)
             current_plan = ""
             current_path = get_note_file_path(self.ctx.data_engine.base_dir, "task_plan")
             if current_path.exists():
@@ -114,7 +88,7 @@ class HistoryAnalyzer:
                 plan_and_history = build_history_for(
                     "history_analyzer",
                     current_plan,
-                    self.history_steps,
+                    history_steps,
                     active_subgoal_hash,
                     engine=self.ctx.data_engine,
                 )
@@ -127,7 +101,7 @@ class HistoryAnalyzer:
                 [
                     f"- Step {s.get('step_number')} ({s.get('relative_time')}):"
                     f" {s.get('summary') or 'No summary'}"
-                    for s in self.history_steps
+                    for s in history_steps
                 ]
             )
 
@@ -136,12 +110,7 @@ class HistoryAnalyzer:
         if prompt_path.exists():
             system_prompt_template = prompt_path.read_text(encoding="utf-8")
         else:
-            system_prompt_template = (
-                "You are a History Analyzer. Your role is to analyze the"
-                " execution history of a session and answer user queries in"
-                " natural language.\nIf you need specific details for a range"
-                " of steps, use the `get_step_details` tool."
-            )
+            system_prompt_template = _FALLBACK_SYSTEM_PROMPT
 
         system_message_content = (
             system_prompt_template + f"\n\n### Task Plan and Execution History:\n{plan_and_history}"
@@ -149,11 +118,8 @@ class HistoryAnalyzer:
 
         # 4. Prepare LLM and bind tools
         llm = get_llm(ctx=self.ctx, name="history_analyzer")
-        get_step_details = self._get_step_details_tool(self.history_steps)
-
-        list_notes_tool = get_list_notes_tool_pure(self.ctx)
-        read_note_tool = get_read_note_tool_pure(self.ctx)
-        llm = llm.bind_tools(tools=[get_step_details, list_notes_tool, read_note_tool])
+        tools = self._build_tools()
+        llm = llm.bind_tools(tools=tools)
 
         # 5. ReAct Loop
         messages: list[BaseMessage] = [
@@ -178,62 +144,38 @@ class HistoryAnalyzer:
             # Process tool calls
             for tc in response.tool_calls:
                 tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
-                args = tc["args"]
+                args = dict(tc.get("args") or {})
+                tool = next((t for t in tools if t.name == tool_name), None)
 
                 with TraceSpan(name=tool_name, ctx=self.ctx) as span:
                     span.payload = {"args": args}
-                    if tool_name == "get_step_details":
-                        logger.info(
-                            f"HistoryAnalyzer executing tool get_step_details with args: {args}"
-                        )
-                        try:
-                            start_step = int(args.get("start_step", 0))
-                            end_step = int(args.get("end_step", 0))
-                            result = get_step_details.invoke(
-                                {"start_step": start_step, "end_step": end_step}
-                            )
-                            span.result = result
-                            status = "success"
-                        except Exception as e:
-                            span.status = "failed"
-                            span.error = str(e)
-                            result = f"Error running get_step_details: {e}"
-                            status = "error"
-                    elif tool_name == "list_notes":
-                        logger.info("HistoryAnalyzer executing tool list_notes")
-                        try:
-                            result = list_notes_tool.invoke({})
-                            span.result = result
-                            status = "success"
-                        except Exception as e:
-                            span.status = "failed"
-                            span.error = str(e)
-                            result = f"Error running list_notes: {e}"
-                            status = "error"
-                    elif tool_name == "read_note":
-                        logger.info(f"HistoryAnalyzer executing tool read_note with args: {args}")
-                        try:
-                            key = args.get("key", "")
-                            result = read_note_tool.invoke({"key": key})
-                            span.result = result
-                            status = "success"
-                        except Exception as e:
-                            span.status = "failed"
-                            span.error = str(e)
-                            result = f"Error running read_note: {e}"
-                            status = "error"
-                    else:
+                    if tool is None:
                         result = f"Error: Tool {tool_name} is not supported."
                         status = "error"
                         span.status = "failed"
                         span.error = result
+                    else:
+                        logger.info(f"HistoryAnalyzer executing tool {tool_name} with args: {args}")
+                        try:
+                            result = await invoke_tool_with_injection(
+                                tool=tool, args=args, tool_call_id=tc["id"]
+                            )
+                            text, _ = split_multimodal_result(result)
+                            status = "error" if text.startswith("Error") else "success"
+                            span.result = text
+                            if status == "error":
+                                span.status = "failed"
+                                span.error = text
+                        except Exception as e:
+                            span.status = "failed"
+                            span.error = str(e)
+                            result = f"Error running {tool_name}: {e}"
+                            status = "error"
 
-                messages.append(
-                    ToolMessage(
-                        tool_call_id=tc["id"],
-                        content=result,
-                        status=status,
-                    )
+                # A step screenshot travels in the carrier the model's provider
+                # accepts; text results stay a single ToolMessage.
+                messages.extend(
+                    tool_result_messages(tc["id"], result, name=tool_name, status=status, llm=llm)
                 )
 
         return "Error: HistoryAnalyzer failed to resolve the query within maximum turns."

artemis/agents/operator/operator.json

@@ -1,3 +1,3 @@
 {
-  "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n  1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n  2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n  3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n  4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] <Goal> into note <TargetNote> (Exit: <ExitCondition>; Interval: <Interval>)` or `[Loop:continuous] <Goal> into note <TargetNote> (Interval: <Interval>)` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n  - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snaLine truncated
+  "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n  1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n  2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n  3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n  4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] <Goal> into note <TargetNote> (Exit: <ExitCondition>; Interval: <Interval>)` or `[Loop:continuous] <Goal> into note <TargetNote> (Interval: <Interval>)` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n  - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snaLine truncated
 }

artemis/agents/operator/operator.py

@@ -37,8 +37,11 @@ from artemis.tools.index import get_tool_by_name
 from artemis.tools.tool_wrapper import (
     get_tool_result_content,
     invoke_tool_with_injection,
+    tool_result_messages,
 )
 from artemis.utils.coordinates import (
+    COORDINATE_SPACE_KEY,
+    COORDINATE_SPACE_PIXEL,
     compute_smart_swipe_coordinates,
     parse_swipe_parameters,
 )
@@ -58,7 +61,9 @@ DEFERRING_TOOLS = {
     "video_analyzer",
     "read_note",
     "list_notes",
-    "recall_history",
+    "search_history",
+    "replay_steps",
+    "get_step_screenshot",
     "run_adb_command",
     "manage_task",
     "analyze_task_output",
@@ -639,11 +644,15 @@ class OperatorNode:
                         results.append((f"Error: Tool {tool_name} not supported", "error"))
 
                 for tc, (content, status) in zip(other_calls, results):
-                    tool_outputs.append(
-                        ToolMessage(
-                            tool_call_id=tc["id"],
-                            content=content,
+                    # A step screenshot (content blocks) travels in the carrier
+                    # this model's provider accepts; text stays a ToolMessage.
+                    tool_outputs.extend(
+                        tool_result_messages(
+                            tc["id"],
+                            content,
+                            name=normalize_name(tc["name"]),
                             status=status,
+                            llm=base_llm,
                         )
                     )
 
@@ -732,8 +741,8 @@ class OperatorNode:
                         )
                     action_calls_to_translate = []
                 elif len(action_calls) > burst_cap:
-                    # A multi-action turn is a fast-action burst that the Validator
-                    # fires without the safety net; cap its length before it runs.
+                    # A multi-action turn is a fast-action burst: the Validator vets its first
+                    # member, fires the rest unvetted; cap its length before it runs.
                     validation_errors = True
                     for tc in action_calls:
                         tool_outputs.append(
@@ -1168,6 +1177,7 @@ class OperatorNode:
                 {
                     "action": "tap",
                     "coordinates": el["center"],
+                    COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
                     "normalized_coordinates": norm_c,
                     "times": times,
                     "delay_ms": delay_ms,
@@ -1193,6 +1203,7 @@ class OperatorNode:
                 {
                     "action": "long_press_on",
                     "coordinates": el["center"],
+                    COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
                     "normalized_coordinates": norm_c,
                     "duration": duration,
                     "target_text": el.get("text"),
@@ -1229,6 +1240,7 @@ class OperatorNode:
                 {
                     "action": "focus_and_input_text",
                     "coordinates": el["center"],
+                    COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
                     "normalized_coordinates": norm_c,
                     "text": text,
                     "clear_before_input": clear_exist,
@@ -1310,6 +1322,7 @@ class OperatorNode:
                 {
                     "action": "swipe",
                     "coordinates": [x1, y1, x2, y2],
+                    COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
                     "normalized_coordinates": [nx1, ny1, nx2, ny2],
                     "normalized_start_coordinates": [nx1, ny1],
                     "normalized_end_coordinates": [nx2, ny2],

artemis/agents/operator/prompts.py

@@ -20,6 +20,8 @@ from pathlib import Path
 
 from jinja2 import Environment, StrictUndefined, Template
 from langchain_core.messages import HumanMessage, SystemMessage
+
+from artemis.tools.history import HISTORY_TOOL_NAMES
 from PIL import Image
 
 from artemis.context import ArtemisContext
@@ -62,7 +64,13 @@ def load_operator_prompts() -> dict[str, str]:
 
 _PRE_DECISION_HELPER_TOOLS = ("ask_explorer", "ask_diagnoser", "video_analyzer")
 _PRE_DECISION_ADB_TOOLS = ("run_adb_command", "manage_task")
-_PRE_DECISION_MEMORY_TOOLS = ("read_note", "list_notes", "recall_history")
+_PRE_DECISION_MEMORY_TOOLS = (
+    "read_note",
+    "list_notes",
+    "search_history",
+    "replay_steps",
+    "get_step_screenshot",
+)
 _PRE_DECISION_ALL_TOOLS = (
     _PRE_DECISION_HELPER_TOOLS + _PRE_DECISION_ADB_TOOLS + _PRE_DECISION_MEMORY_TOOLS
 )
@@ -188,10 +196,12 @@ def resolve_operator_prompt_tools(ctx: ArtemisContext) -> frozenset[str]:
     if not (setup and getattr(setup, "video_recording_tools_enabled", False)):
         available.discard("video_analyzer")
 
-    # recall_history (M4) needs a DataEngine to search and is config-gated;
-    # the prompt must not advertise it when the tool is not bound this run.
-    if getattr(ctx, "data_engine", None) is None or not _recall_enabled():
-        available.discard("recall_history")
+    # The history tools need a DataEngine to read, and search_history is also
+    # config-gated; the prompt must not advertise a tool that is not bound.
+    if getattr(ctx, "data_engine", None) is None:
+        available.difference_update(HISTORY_TOOL_NAMES)
+    elif not _recall_enabled():
+        available.discard("search_history")
 
     # Device actions the installed actuator backend does not implement disappear
     # from the prompt in lockstep with their tool declarations.
@@ -438,14 +448,13 @@ def _last_successful_action_before(steps: list, step_number: int | None) -> tupl
 
 def _incident_target_label(incident: dict) -> str:
     action = incident.get("action") or {}
+    # The Operator only ever sees 0–1000 coordinates: the recorded pixel
+    # target is the controller's business and is never rendered here.
     normalized = action.get("normalized_coordinates")
-    pixels = action.get("coordinates")
     text = action.get("target_text")
     parts = []
     if normalized:
         parts.append(f"normalized {list(normalized)}")
-    elif pixels:
-        parts.append(f"pixel {list(pixels)}")
     if text:
         parts.append(f'"{text}"')
     return " ".join(parts) if parts else "the recorded target"
@@ -844,7 +853,7 @@ class ScreenshotSimilarityPromptComponent(PromptComponent):
 
 
 def _recall_enabled() -> bool:
-    """Whether the recall_history tool is enabled by configuration (M4)."""
+    """Whether the search_history tool is enabled by configuration."""
     try:
         from artemis.config import load_agent_config
 
@@ -861,7 +870,7 @@ class HistoricalStateHintPromptComponent(PromptComponent):
     including steps already chunk-compressed out of the visible transcript) —
     an O(n) integer scan, no model call, no historical image bytes. On a
     close match to a step *older* than the recent window it injects a one-line
-    hint pointing at ``recall_history``.
+    hint pointing at ``search_history``.
 
     Division of labor with :class:`ScreenshotSimilarityPromptComponent`: that
     component's pixel-exact 3-step look-back covers "stuck on the same
@@ -936,5 +945,5 @@ class HistoricalStateHintPromptComponent(PromptComponent):
             builder.add_human_content(
                 f"Historical state hint: current screen closely resembles the"
                 f" post-action screen from Step {best_step_number}. Use"
-                " recall_history only if its details are needed."
+                " search_history / replay_steps only if its details are needed."
             )

artemis/agents/outputter/outputter.py

@@ -16,12 +16,7 @@ import json
 from pathlib import Path
 
 from jinja2 import Template
-from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, ToolMessage
-from artemis.agents.outputter.tools import (
-    get_search_history_tool,
-    get_step_details_tool,
-    get_step_screenshot_tool,
-)
+from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
 from artemis.config import OutputConfig
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import trace
@@ -36,7 +31,12 @@ from artemis.tools.scratchpad import (
     get_update_note_tool_pure,
 )
 from artemis.tools.video_tool import get_video_analyzer_tool_pure
-from artemis.tools.tool_wrapper import invoke_tool_with_injection
+from artemis.tools.history import get_history_tools
+from artemis.tools.tool_wrapper import (
+    invoke_tool_with_injection,
+    split_multimodal_result,
+    tool_result_messages,
+)
 from artemis.utils.logger import get_logger
 from artemis.memory.context_policy import build_history_for
 from artemis.utils.task_tree import get_active_subgoal_hashes
@@ -112,10 +112,11 @@ async def outputter(
         " LAZY VERIFICATION: If the execution history and the final screenshot"
         " already in your context provide undeniable proof of the outcome, do"
         " not call tools. Directly output your conclusion.\n- ACTIVE RETRIEVAL:"
-        " You should only call tools (e.g., `get_step_details`,"
-        " `get_step_screenshot`, `search_history_for_text`) if you need to"
-        " extract specific hidden data (like verification codes, tracking"
-        " numbers) or if the final state is ambiguous.\n- VIDEO ANALYSIS COST:"
+        " You should only call tools (e.g., `search_history` to locate the"
+        " step, `replay_steps` for its full record, `get_step_screenshot` for"
+        " its image) if you need to extract specific hidden data (like"
+        " verification codes, tracking numbers) or if the final state is"
+        " ambiguous.\n- VIDEO ANALYSIS COST:"
         " The `video_analyzer` tool is expensive. Use it only when necessary"
         " (e.g., to verify video playback or motion).\n- PERSISTENT WRITE: If"
         " you extract important information requested by the user (such as a"
@@ -196,14 +197,8 @@ async def outputter(
         HumanMessage(content=human_message_content),
     ]
 
-    # Setup tools
-    history_steps = []
-    if ctx.data_engine:
-        history_steps = ctx.data_engine.get_agent_friendly_steps()
-
-    details_tool = get_step_details_tool(history_steps)
-    screenshot_tool = get_step_screenshot_tool(ctx, history_steps)
-    search_tool = get_search_history_tool(ctx)
+    # Setup tools (the shared history tools plus notes and video)
+    search_tool, replay_tool, screenshot_tool = get_history_tools(ctx)
     list_notes_tool = get_list_notes_tool_pure(ctx)
     read_note_tool = get_read_note_tool_pure(ctx)
     save_note_tool = get_save_note_tool_pure(ctx)
@@ -212,9 +207,9 @@ async def outputter(
     video_analyzer_tool = get_video_analyzer_tool_pure(ctx)
 
     tools = [
-        details_tool,
-        screenshot_tool,
         search_tool,
+        replay_tool,
+        screenshot_tool,
         list_notes_tool,
         read_note_tool,
         save_note_tool,
@@ -254,9 +249,9 @@ async def outputter(
 
             logger.info(f"Outputter executing tool {tool_name} with args: {args}")
             tool_map = {
-                "get_step_details": details_tool,
+                "search_history": search_tool,
+                "replay_steps": replay_tool,
                 "get_step_screenshot": screenshot_tool,
-                "search_history_for_text": search_tool,
                 "list_notes": list_notes_tool,
                 "read_note": read_note_tool,
                 "save_note": save_note_tool,
@@ -277,22 +272,17 @@ async def outputter(
                     )
                 else:
                     result = f"Error: Tool {tool_name} is not supported."
-                status = (
-                    "success"
-                    if not (isinstance(result, str) and result.startswith("Error"))
-                    else "error"
-                )
+                text, _ = split_multimodal_result(result)
+                status = "error" if text.startswith("Error") else "success"
             except Exception as e:
                 logger.error(f"Error running tool {tool_name}: {e}")
                 result = f"Error running tool {tool_name}: {e}"
                 status = "error"
 
-            messages.append(
-                ToolMessage(
-                    tool_call_id=tc["id"],
-                    content=result if isinstance(result, (str, list)) else str(result),
-                    status=status,
-                )
+            # Step screenshots enter the conversation in the carrier the
+            # model's provider accepts; text results stay a plain ToolMessage.
+            messages.extend(
+                tool_result_messages(tc["id"], result, name=tool_name, status=status, llm=llm)
             )
 
     if raw_answer is None:

artemis/agents/outputter/tools.py

@@ -1,170 +0,0 @@
-# Copyright 2026 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import base64
-import json
-from pathlib import Path
-from typing import Any
-
-from langchain_core.tools import BaseTool, tool
-from artemis.context import ArtemisContext
-from artemis.utils.logger import get_logger
-
-logger = get_logger(__name__)
-
-
-def get_step_details_tool(history_steps: list[dict[str, Any]]) -> BaseTool:
-    @tool
-    def get_step_details(
-        start_step: int,
-        end_step: int,
-    ) -> str:
-        """Retrieve the detailed, full-granularity information for a range of steps (inclusive).
-
-        Use this when you need to inspect specific actions taken, operator
-        thinking, or execution results.
-        """
-        try:
-            s_step = int(start_step)
-            e_step = int(end_step)
-        except (ValueError, TypeError):
-            return (
-                f"Error: start_step and end_step must be integers, got {start_step} and {end_step}."
-            )
-
-        matched_steps = []
-        for s in history_steps:
-            step_num = s.get("step_number")
-            if step_num is not None and s_step <= step_num <= e_step:
-                details = {
-                    "step_number": s.get("step_number"),
-                    "relative_time": s.get("relative_time"),
-                    "summary": s.get("summary"),
-                    "action_taken": s.get("action_taken"),
-                    "operator_raw_thinking": s.get("operator_raw_thinking"),
-                    "operator_native_thinking": s.get("operator_native_thinking"),
-                    "last_execution_result": s.get("last_execution_result"),
-                    "interleaved_events": s.get("interleaved_events"),
-                }
-                matched_steps.append(details)
-        if not matched_steps:
-            return f"No steps found in range [{s_step}, {e_step}]."
-        return json.dumps(matched_steps, indent=2, ensure_ascii=False)
-
-    return get_step_details
-
-
-def get_step_screenshot_tool(ctx: ArtemisContext, history_steps: list[dict[str, Any]]) -> BaseTool:
-    @tool
-    def get_step_screenshot(step_number: int) -> list[dict[str, Any]] | str:
-        """Retrieve the screenshot for a specific step.
-
-        Returns a multimodal content block containing the image.
-        """
-        if not ctx.data_engine:
-            return "Error: DataEngine not available."
-
-        try:
-            step_num = int(step_number)
-        except (ValueError, TypeError):
-            return f"Error: step_number must be an integer, got {step_number}."
-
-        # Find the step to get the image name
-        step_record = next((s for s in history_steps if s.get("step_number") == step_num), None)
-        if not step_record:
-            return f"Error: Step {step_num} not found in history."
-
-        image_name = step_record.get("pre_image_name") or step_record.get("post_image_name")
-        if not image_name:
-            return f"Error: No screenshot recorded for step {step_num}."
-
-        images_dir = Path(ctx.data_engine.global_base_dir) / "images"
-        image_path = images_dir / f"{image_name}.jpg"
-
-        if not image_path.exists():
-            return f"Error: Screenshot file for step {step_num} does not exist on disk."
-
-        try:
-            with open(image_path, "rb") as f:
-                img_base64 = base64.b64encode(f.read()).decode("utf-8")
-        except Exception as e:
-            return f"Error reading screenshot file: {e}"
-
-        # Return a LangChain multimodal content structure
-        return [
-            {"type": "text", "text": f"Screenshot for step {step_num}:"},
-            {
-                "type": "image_url",
-                "image_url": {"url": f"data:image/jpeg;base64,{img_base64}"},
-            },
-        ]
-
-    return get_step_screenshot
-
-
-def get_search_history_tool(ctx: ArtemisContext) -> BaseTool:
-    @tool
-    def search_history_for_text(query: str) -> str:
-        """Search for a specific text query within the OCR and UI hierarchy of all historical steps."""
-        if not ctx.data_engine:
-            return "Error: DataEngine not available."
-
-        session_id = ctx.data_engine.current_session_id
-        if not session_id:
-            return "Error: No active session."
-
-        try:
-            steps = ctx.data_engine.storage.get_steps(session_id)
-        except Exception as e:
-            return f"Error retrieving steps from storage: {e}"
-
-        results = []
-        for step in steps:
-            step_matches = []
-            # Search pre-action image
-            if step.pre_image_name:
-                try:
-                    matches = ctx.data_engine.storage.search_ui_by_hash(step.pre_image_name, query)
-                    if matches:
-                        step_matches.extend(matches)
-                except Exception as e:
-                    logger.error(f"Failed to search UI by hash {step.pre_image_name}: {e}")
-
-            # Search post-action image
-            if step.post_image_name:
-                try:
-                    matches = ctx.data_engine.storage.search_ui_by_hash(step.post_image_name, query)
-                    if matches:
-                        step_matches.extend(matches)
-                except Exception as e:
-                    logger.error(f"Failed to search UI by hash {step.post_image_name}: {e}")
-
-            if step_matches:
-                # Clean up match details to not bloat context, keep top 3
-                clean_matches = []
-                for m in step_matches[:3]:
-                    clean_matches.append(
-                        {
-                            "type": m.get("type"),
-                            "matched_text": m.get("matched_text"),
-                            "score": m.get("score"),
-                        }
-                    )
-                results.append({"step_number": step.step_number, "matches": clean_matches})
-
-        if not results:
-            return f"No matches found for '{query}' in history."
-        return json.dumps(results, indent=2, ensure_ascii=False)
-
-    return search_history_for_text

artemis/agents/summarizer/summarizer.md

@@ -30,6 +30,6 @@ You will receive:
 - **CHRONOLOGICAL STEP TRACE (CURRENT STEP)**: The step-by-step chronological history of the current execution turn, including:
   - `[Operator Monologue] / [Operator Native Thought]`: The operator's planning, intentions, progress counting, reasoning, verified states, and explicit mentions of missing prerequisites.
   - `[Operator Tool Call] / [Operator Final Action]`: The actual action(s) planned and executed.
-  - `[Planned Fast-Action Burst]`: A multi-action turn executed back to back without the safety net, with each member's outcome.
+  - `[Planned Fast-Action Burst]`: A multi-action turn: the first member is vetted by the safety net, the rest fire back to back unvetted; each member's outcome is listed.
   - `[Pre-Execution Safety Net]`: Validation checks prior to action execution.
   - `[Result]`: The Validator's execution outcome; an "Error:" line here is an execution incident the Operator must resolve.

artemis/agents/summarizer/summarizer.py

@@ -80,7 +80,7 @@ class SummarizerNode:
         pre_bytes = self._read_step_image(step_number, "pre")
         post_bytes = self._read_step_image(step_number, "post")
 
-        action_name, action_args = self._extract_action(record, state)
+        action_name, action_args = self._extract_action(record, state, ctx=self.ctx)
         result = getattr(state, "last_execution_result", None)
         # format_result_clean reports only errors/repairs; a clean run keeps
         # the controller's bare status word.
@@ -116,7 +116,7 @@ class SummarizerNode:
             return None
 
     @staticmethod
-    def _extract_action(record: Any, state: State) -> tuple[str, dict]:
+    def _extract_action(record: Any, state: State, *, ctx: Any = None) -> tuple[str, dict]:
         actions = getattr(record, "action_taken", None) if record is not None else None
         if not actions:
             decisions = getattr(state, "structured_decisions", None)
@@ -129,6 +129,15 @@ class SummarizerNode:
             actions = [actions]
         if not isinstance(actions, list) or not actions or not isinstance(actions[0], dict):
             return "no_op", {}
+        # The visual-transition lens is a model: it sees 0–1000 coordinates
+        # only. Pro records pixels (space-marked), so convert once here; a
+        # record already marked normalized passes through untouched.
+        from artemis.utils.coordinates import normalize_any_structure
+
+        device = getattr(ctx, "device", None) if ctx is not None else None
+        width = getattr(device, "device_width", None) or 1080
+        height = getattr(device, "device_height", None) or 2400
+        actions = normalize_any_structure(actions, width, height)
         first = actions[0]
         action_name = first.get("action") or "unknown"
         action_args = {k: v for k, v in first.items() if k != "action"}

artemis/agents/validator/execution_loop.py

@@ -18,12 +18,14 @@ Two execution tiers, selected by how many turn-ending actions the Operator
 emitted in the turn:
 
 * **Vetted single action** (one action): the pre-execution Safety Net runs
-  first (XML-first with pixel fallback), then the action executes locally with
-  a short retry loop.
-* **Fast-action burst** (two or more actions): the actions fire back to back
-  with no safety net, no retries and no screenshots in between. The burst is
-  the Operator's tool against turn latency (transient menus, toasts, control
-  bars); the Operator owns the risk. The first failure aborts the rest.
+  first (XML-first; the pixel judge decides only when the XML verdict is
+  weak), then the action executes locally with a short retry loop.
+* **Fast-action burst** (two or more actions): the first member is vetted by
+  the same Safety Net (the screen it was decided on is still the live screen)
+  and a refusal aborts the whole burst; the remaining members fire back to
+  back with no safety net, no retries and no screenshots in between. The burst
+  is the Operator's tool against turn latency (transient menus, toasts,
+  control bars); the Operator owns the risk. The first failure aborts the rest.
 
 Either way, a failure does not spawn a repair agent. The loop opens an
 *execution incident* (:mod:`artemis.agents.validator.incidents`) that rides in
@@ -49,8 +51,10 @@ from artemis.agents.validator.incidents import (
     KIND_SAFETY_NET,
     open_incident,
 )
+from artemis.agents.validator.precondition_xml import XML_VERDICT_STRONG
 from artemis.data_engine.trace import CURRENT_TRACE_ID
 from artemis.graph.state import State
+from artemis.utils.coordinates import normalize_action_dict
 from artemis.utils.logger import get_logger
 from artemis.utils.task_tree import format_action_clean
 
@@ -72,10 +76,29 @@ class _ActionOutcome:
     post_screenshot_b64: str | None = None
 
 
+#: ``safety_net_evidence`` keys the incident carries through verbatim (how the
+#: verdict was reached), next to the normalized location/occupant facts.
+_VERDICT_EVIDENCE_KEYS = ("xml_verdict", "xml_element_count", "xml_weak_reason", "pixel_judge")
+
+
+def _note_pixel_judge(action_item: dict, note: str) -> None:
+    """Records on the action item how the pixel judge figured in the verdict."""
+    action_item.setdefault("safety_net_evidence", {})["pixel_judge"] = note
+
+
 async def _run_precondition_gate(
     node, session, action_item: dict, state: State, pre_screenshot_b64: str, original_coords: list
 ) -> tuple[bool, ValidationErrorCategory, str]:
-    """Runs the pre-execution Safety Net: XML-first with pixel fallback."""
+    """Runs the pre-execution Safety Net: XML-first, pixel judge for weak verdicts.
+
+    The pixel judge (a VLM call) is a fallback for when the hierarchy could not
+    see the target, never a second opinion when it could. A STRONG XML failure
+    (healthy, non-trivial tree; the target's identifiers were searched and not
+    found) is final and blocks the action without consulting the judge. A WEAK
+    one (XML bypassed/timed out, sparse tree, identifier-less target) hands the
+    decision to the judge. The path taken is recorded in ``safety_net_evidence``
+    and appended to the reason as one short clause.
+    """
     is_explorer_candidate = action_item.get("target_class") == "ExplorerCandidate"
     is_ocr_element = "[OCR]" in str(action_item.get("target_class") or "")
     has_index_metadata = bool(
@@ -88,11 +111,17 @@ async def _run_precondition_gate(
         and not is_ocr_element
     )
 
-    if not has_index_metadata:
-        # Fall back to Pixel-based validation for pure coordinate/non-index interactions
-        return await node._validate_action_precondition_pixel(
+    async def _pixel_judge(note: str) -> tuple[bool, ValidationErrorCategory, str]:
+        passed, category, error = await node._validate_action_precondition_pixel(
             session, action_item, pre_screenshot_b64, original_coords, state=state
         )
+        if not passed:
+            _note_pixel_judge(action_item, note)
+        return passed, category, error
+
+    if not has_index_metadata:
+        # Pure coordinate/non-index interactions: the pixel judge is the only gate.
+        return await _pixel_judge("consulted alone (target had no hierarchy metadata)")
 
     # Index-based validation is fastest and safest when metadata is present
     (
@@ -110,27 +139,33 @@ async def _run_precondition_gate(
             f" ({validation_error}). Falling back to"
             " Pixel-based validation..."
         )
-        return await node._validate_action_precondition_pixel(
-            session, action_item, pre_screenshot_b64, original_coords, state=state
-        )
+        return await _pixel_judge("consulted alone (XML bypassed)")
 
+    evidence = action_item.get("safety_net_evidence") or {}
+    if evidence.get("xml_verdict") == XML_VERDICT_STRONG:
+        clause = (
+            f"XML verdict final: {evidence.get('xml_element_count')} nodes searched for the"
+            " target's identifiers; pixel judge not consulted"
+        )
+        _note_pixel_judge(action_item, "not consulted (strong XML verdict)")
+        logger.info(f"Pre-execution XML-based validation failed: {validation_error}. {clause}.")
+        return False, validation_category, f"{validation_error} [{clause}]"
+
+    weak_reason = evidence.get("xml_weak_reason") or "no verdict strength recorded"
     logger.info(
-        "Pre-execution XML-based validation failed:"
-        f" {validation_error}. Attempting Pixel-based"
-        " validation fallback..."
+        f"Pre-execution XML-based validation failed: {validation_error}. XML evidence is"
+        f" weak ({weak_reason}); asking the pixel judge..."
     )
-    (
-        pixel_passed,
-        pixel_category,
-        pixel_error,
-    ) = await node._validate_action_precondition_pixel(
-        session, action_item, pre_screenshot_b64, original_coords, state=state
+    pixel_passed, pixel_category, _pixel_error = await _pixel_judge(
+        "consulted (weak XML verdict); did not confirm the target"
     )
     if pixel_passed and pixel_category == ValidationErrorCategory.NONE:
-        logger.success("Pixel-based validation fallback PASSED! Overriding XML validation failure.")
+        logger.success("Pixel judge confirmed the target is present; overriding weak XML verdict.")
+        action_item.pop("safety_net_evidence", None)
         return True, ValidationErrorCategory.NONE, ""
 
-    return validation_passed, validation_category, validation_error
+    clause = f"XML evidence weak ({weak_reason}); pixel judge did not confirm the target"
+    return validation_passed, validation_category, f"{validation_error} [{clause}]"
 
 
 async def _attempt_local_execution(
@@ -180,14 +215,19 @@ async def _process_action(
     pre_screenshot_b64: str,
     *,
     burst: bool,
+    vetted: bool = True,
 ) -> _ActionOutcome:
-    """Runs precondition checks (vetted tier only) and execution for one action item."""
+    """Runs precondition checks (``vetted`` items only) and execution for one action item.
+
+    ``vetted`` is True for a single action and for the first member of a burst;
+    later burst members fire unvetted. ``burst`` members never retry.
+    """
     action_item = dict(action_item)  # Make a copy to avoid mutating original state actions
     outcome = _ActionOutcome(action_item=action_item)
     attempts_log = []
 
     validation_passed = True
-    if not burst:
+    if vetted:
         # 1. Pre-execution validation (Safety Net)
         original_coords = list(action_item.get("coordinates") or [])
         (
@@ -324,6 +364,9 @@ def _incident_evidence(ctx, outcome: _ActionOutcome) -> dict:
         evidence["occupant"] = raw["occupant"]
         if raw.get("occupant_bounds") is not None:
             evidence["occupant_bounds"] = raw["occupant_bounds"]
+    for key in _VERDICT_EVIDENCE_KEYS:
+        if raw.get(key) is not None:
+            evidence[key] = raw[key]
     return evidence
 
 
@@ -337,6 +380,17 @@ def _current_step_number(ctx, state: State) -> int | None:
     return None
 
 
+def _model_facing_action(ctx, action_item: dict) -> dict:
+    """The action item in the Operator's own 0–1000 coordinate space.
+
+    The live item carries the controller's pixels; every phrase rendered for
+    a model is normalized first, and the space marker makes that idempotent.
+    """
+    width = getattr(getattr(ctx, "device", None), "device_width", None) or 1080
+    height = getattr(getattr(ctx, "device", None), "device_height", None) or 2400
+    return normalize_action_dict(action_item, width, height)
+
+
 def _build_incident(ctx, state: State, outcome: _ActionOutcome, *, index: int, total: int) -> dict:
     step_number = _current_step_number(ctx, state)
     return open_incident(
@@ -345,7 +399,7 @@ def _build_incident(ctx, state: State, outcome: _ActionOutcome, *, index: int, t
         category=outcome.error_category,
         reason=outcome.error_msg,
         action_item=outcome.action_item,
-        action_description=format_action_clean(outcome.action_item),
+        action_description=format_action_clean(_model_facing_action(ctx, outcome.action_item)),
         action_index=index,
         burst_size=total,
         step_number=step_number,
@@ -371,8 +425,8 @@ async def run_validation_loop(node, state: State) -> dict:
     burst = total_actions > 1
     if burst:
         logger.info(
-            f"Fast-action burst: {total_actions} actions will fire back to back"
-            " without the safety net."
+            f"Fast-action burst: {total_actions} actions; the first is vetted by the"
+            " safety net, the rest fire back to back unvetted."
         )
 
     session = await node._get_mcp_session()
@@ -407,7 +461,16 @@ async def run_validation_loop(node, state: State) -> dict:
 
         try:
             outcome = await _process_action(
-                node, state, session, action_item, action_name, last_screenshot_b64, burst=burst
+                node,
+                state,
+                session,
+                action_item,
+                action_name,
+                last_screenshot_b64,
+                burst=burst,
+                # The first burst member was decided on the live screen, so the
+                # safety net still applies to it; later members fire unvetted.
+                vetted=(not burst or index == 0),
             )
             success = outcome.success
             execution.append(outcome.action_item)

artemis/agents/validator/pixel_safety_net.md

@@ -4,7 +4,7 @@ UI State Verification Expert. The Operator decided to act at the red dot while l
 Read the `[Target]` block first to determine which rule applies.
 
 Rules
-1. `Kind: specific UI control` — the target has a label, resource id, class or bounds. Judge whether that exact control remains visible at the red dot in Image 2. Ignore video, map, animation or background changes beneath it. A control that has hidden, faded, collapsed, moved or become covered is not present, even if tapping the same spot could restore it.
+1. `Kind: specific UI control` — the target has a label, resource id, class or bounds. Judge whether that exact control remains visible at the red dot in Image 2. Ignore video, map, animation or background changes beneath it. A control that has hidden, faded, collapsed, moved or become covered is not present, even if tapping the same spot could restore it. Judge the red dot, not the screen: if a different control, or nothing, now sits under the dot, the target is not present even when the same control (or its label) survives elsewhere on the screen or has turned into a different state such as an active input field.
 2. `Kind: coordinates only` — no UI hierarchy element was found at the point. Identify what appears under the red dot in both images. If Image 1 shows a distinct button, icon, link or handle, apply Rule 1. Apply the surface rule only when Image 1 shows blank space, a backdrop, or the body of a video, animation, map or live stream.
 3. Surface rule. Typical intents: tap the video body to wake its hidden controls, tap the backdrop to dismiss a dialog. Surfaces are expected to change between frames: ignore natural frame changes and the appearance or fading of overlay controls. Output `{"is_present": true, "confidence": 1.0}` unless an unexpected blocking popup now covers the red dot.
 4. Use `[Planned Action & Original Thinking]` to resolve whether the target was a control or its underlying surface. Do not treat a missing control as present because another tap could restore it.

artemis/agents/validator/precondition_xml.py

@@ -51,6 +51,15 @@ W_TEXT = 0.4
 W_BOUNDS = 0.3
 W_COORD = 0.3
 
+#: Strength of a failed XML verdict, recorded in ``safety_net_evidence``.
+#: STRONG verdicts are final; WEAK ones hand the decision to the pixel judge.
+XML_VERDICT_STRONG = "strong"
+XML_VERDICT_WEAK = "weak"
+
+#: A live hierarchy with fewer nodes than this is treated as sparse (a WebView,
+#: game canvas or half-drawn screen): its "not found" is weak evidence.
+XML_STRONG_MIN_ELEMENTS = 8
+
 
 async def validate_action_precondition(
     node, session, action_item: dict, state: State | None = None
@@ -527,6 +536,45 @@ def _describe_occupation(
     return is_occupied, occupant_desc, occupant_bounds
 
 
+def _count_nodes(elements) -> int:
+    """Counts hierarchy nodes, descending into nested ``children`` lists."""
+    total = 0
+    for elem in elements or []:
+        if isinstance(elem, dict):
+            total += 1 + _count_nodes(elem.get("children") or [])
+    return total
+
+
+def classify_xml_verdict(element_count: int, target_text, target_resource_id) -> tuple[str, str]:
+    """Grades a failed XML verdict as strong or weak.
+
+    STRONG: the hierarchy is non-trivial AND the target carried identifiers
+    (resource-id / text / content-desc) that were searched and not matched.
+    Anything else (sparse tree, identifier-less target such as bare bounds from
+    a hit test) is WEAK, and the pixel judge gets the final say.
+
+    Returns ``(strength, weak_reason)``; ``weak_reason`` is empty for STRONG.
+    """
+    if element_count < XML_STRONG_MIN_ELEMENTS:
+        return XML_VERDICT_WEAK, f"sparse hierarchy ({element_count} nodes)"
+    if not (target_text or target_resource_id):
+        return XML_VERDICT_WEAK, "target had no resource-id/text to search for"
+    return XML_VERDICT_STRONG, ""
+
+
+def _annotate_verdict_strength(
+    action_item: dict, elements: list, target_text, target_resource_id
+) -> None:
+    """Records the verdict strength on the action item's ``safety_net_evidence``."""
+    count = _count_nodes(elements)
+    strength, weak_reason = classify_xml_verdict(count, target_text, target_resource_id)
+    evidence = action_item.setdefault("safety_net_evidence", {})
+    evidence["xml_verdict"] = strength
+    evidence["xml_element_count"] = count
+    if weak_reason:
+        evidence["xml_weak_reason"] = weak_reason
+
+
 def _classify_failure(
     elements: list,
     action_item: dict,
@@ -602,6 +650,9 @@ async def validate_action_precondition_single(
     target_bounds = action_item.get("target_bounds")  # [l, t, r, b]
     target_resource_id = action_item.get("target_resource_id")
 
+    # Evidence belongs to one attempt; a retry must not inherit a stale verdict.
+    action_item.pop("safety_net_evidence", None)
+
     width, height = _resolve_screen_dims(ctx, state)
 
     # Calculate diagonal-based scale factor for resolution-independent thresholds
@@ -659,7 +710,7 @@ async def validate_action_precondition_single(
     if best["score"] >= threshold:
         return _handle_match_success(action_item, best, target_text, scale_factor)
 
-    return _classify_failure(
+    verdict = _classify_failure(
         elements,
         action_item,
         best,
@@ -670,3 +721,5 @@ async def validate_action_precondition_single(
         height=height,
         scale_factor=scale_factor,
     )
+    _annotate_verdict_strength(action_item, elements, target_text, target_resource_id)
+    return verdict