feat: unify history tools and improve execution reliability

tests/unit/tools/history/test_declarations.py

@@ -0,0 +1,99 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""One args schema, two contracts: the Flash ``ToolDeclaration`` and the
+LangChain tool of every history tool are derived from the same Pydantic model."""
+
+from types import SimpleNamespace
+
+import pytest
+
+from artemis.core.tool_declaration import ToolDeclaration
+from artemis.tools.history import (
+    GET_STEP_SCREENSHOT_TOOL,
+    HISTORY_TOOL_DECLARATIONS,
+    HISTORY_TOOL_NAMES,
+    HISTORY_TOOL_WRAPPERS,
+    HISTORY_TOOLS,
+    REPLAY_STEPS_TOOL,
+    SEARCH_HISTORY_TOOL,
+    get_history_tools,
+    history_tool_by_name,
+    history_tool_declarations,
+)
+
+
+@pytest.mark.parametrize("tool", HISTORY_TOOLS, ids=lambda t: t.name)
+def test_tool_declaration_mirrors_the_args_schema(tool):
+    declaration = tool.to_tool_declaration()
+    assert isinstance(declaration, ToolDeclaration)
+    assert declaration.name == tool.name
+    assert declaration.description == tool.description
+
+    properties = declaration.parameters["properties"]
+    assert list(properties) == list(tool.args_schema.model_fields)
+    for field_name, info in tool.args_schema.model_fields.items():
+        assert properties[field_name]["description"] == info.description
+    assert declaration.parameters["required"] == [
+        name for name, info in tool.args_schema.model_fields.items() if info.is_required()
+    ]
+
+    # The LangChain export binds the identical schema.
+    lc_tool = tool.to_langchain_tool(SimpleNamespace(data_engine=None))
+    assert lc_tool.name == tool.name
+    assert lc_tool.args_schema is tool.args_schema
+
+    # And the GenAI export agrees on names and required fields.
+    genai = tool.to_genai_declaration()
+    assert set(genai.parameters.properties) == set(properties)
+    assert list(genai.parameters.required or []) == declaration.parameters["required"]
+
+
+def test_declared_types_and_enums():
+    search = SEARCH_HISTORY_TOOL.parameters["properties"]
+    assert search["query"]["type"] == "string"
+    assert search["step_range"] == {
+        "type": "array",
+        "items": {"type": "integer"},
+        "description": search["step_range"]["description"],
+    }
+    assert search["max_results"]["type"] == "integer"
+    assert search["max_results"]["default"] == 5
+    assert SEARCH_HISTORY_TOOL.parameters["required"] == []
+
+    replay = REPLAY_STEPS_TOOL.parameters["properties"]
+    assert replay["start_step"]["type"] == "integer"
+    assert replay["end_step"]["type"] == "integer"
+    assert REPLAY_STEPS_TOOL.parameters["required"] == ["start_step"]
+
+    shot = GET_STEP_SCREENSHOT_TOOL.parameters["properties"]
+    assert shot["step_number"]["type"] == "integer"
+    assert shot["which"]["type"] == "string"
+    assert shot["which"]["enum"] == ["pre", "post", "overlay"]
+    assert shot["which"]["default"] == "pre"
+    assert GET_STEP_SCREENSHOT_TOOL.parameters["required"] == ["step_number"]
+
+
+def test_exports_are_consistent():
+    assert HISTORY_TOOL_NAMES == {"search_history", "replay_steps", "get_step_screenshot"}
+    assert [d.name for d in HISTORY_TOOL_DECLARATIONS] == [t.name for t in HISTORY_TOOLS]
+    assert len(HISTORY_TOOL_WRAPPERS) == 3
+    assert history_tool_by_name("default_api:replay_steps").name == "replay_steps"
+    assert history_tool_by_name("nope") is None
+
+    ctx = SimpleNamespace(data_engine=object())
+    expected = [t.name for t in HISTORY_TOOLS]
+    assert [t.name for t in get_history_tools(ctx)] == expected
+    assert [d.name for d in history_tool_declarations(ctx)] == expected
+    assert history_tool_declarations(SimpleNamespace(data_engine=None)) == []

tests/unit/tools/history/test_replay.py

@@ -0,0 +1,150 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""replay_steps: range replay, per-call step cap, whole-step token budget,
+``[Screen]`` status lines, and the no-history degradation."""
+
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+
+from artemis.tools.history import ReplayStepsTool, replay_steps, replay_steps_text
+
+
+def _friendly_step(number, tool_result="The toggle is ON", **overrides):
+    step = {
+        "step_id": f"step_{number}",
+        "step_number": number,
+        "relative_time": f"{number * 3}.0s",
+        "summary": f"Step {number} summary.",
+        "action_taken": {"action": "click", "target": [0.5, 0.5], "target_text": "Save"},
+        "last_execution_result": {"status": "success"},
+        "interleaved_events": [
+            {"type": "thought", "content": f"Thinking at step {number}."},
+            {
+                "type": "tool_call",
+                "name": "ask_explorer",
+                "args": {"question": "toggle?"},
+                "result": tool_result,
+            },
+        ],
+    }
+    step.update(overrides)
+    return step
+
+
+def _reader(steps):
+    reader = MagicMock()
+    reader.get_agent_friendly_steps_in_range.side_effect = lambda s, e: [
+        st for st in steps if s <= st["step_number"] <= e
+    ]
+    return reader
+
+
+def test_replay_range_renders_every_step_in_full():
+    reader = _reader([_friendly_step(3), _friendly_step(4, tool_result="The toggle is OFF " * 100)])
+    out = replay_steps_text(reader, 3, 4)
+
+    reader.get_agent_friendly_steps_in_range.assert_called_once_with(3, 4)
+    assert "- **Step 3 (Start: 9.0s)**" in out
+    assert "- **Step 4 (Start: 12.0s)**" in out
+    assert "[Screen]: Step 3 summary." in out
+    assert "Thinking at step 4." in out
+    assert "`ask_explorer(" in out
+    # Loose clamp: a ~1.8k-char tool result is replayed in full.
+    assert "The toggle is OFF " * 100 in out
+    assert "[Planned Action]: Tapped 'Save' at [0.5, 0.5]" in out
+
+
+def test_replay_single_step_reversed_bounds_and_missing_step():
+    reader = _reader([_friendly_step(2)])
+    assert "- **Step 2 (Start: 6.0s)**" in replay_steps_text(reader, 2)
+    reader.get_agent_friendly_steps_in_range.assert_called_once_with(2, 2)
+
+    reader = _reader([_friendly_step(2), _friendly_step(3)])
+    out = replay_steps_text(reader, 3, 2)
+    reader.get_agent_friendly_steps_in_range.assert_called_once_with(2, 3)
+    assert out.index("**Step 2") < out.index("**Step 3")
+
+    assert replay_steps_text(_reader([]), 9) == "Error: step 9 not found."
+    assert replay_steps_text(_reader([]), 9, 12) == "Error: no recorded steps in range 9–12."
+    assert "must be integers" in replay_steps_text(_reader([]), "x")
+
+
+def test_replay_caps_the_range_per_call():
+    reader = _reader([_friendly_step(n) for n in range(1, 30)])
+    out = replay_steps_text(reader, 1, 20, max_steps=5)
+    reader.get_agent_friendly_steps_in_range.assert_called_once_with(1, 5)
+    assert "**Step 5" in out
+    assert "**Step 6" not in out
+    assert "Call again from Step 6" in out
+
+
+def test_replay_token_budget_drops_whole_trailing_steps():
+    big = "x" * 4000  # ~1k tokens per step
+    reader = _reader([_friendly_step(n, tool_result=big) for n in range(1, 6)])
+    out = replay_steps_text(reader, 1, 5, max_steps=5, max_tokens=2500)
+
+    # Two steps fit; the rest are dropped as whole steps, never cut mid-step.
+    assert "**Step 1" in out and "**Step 2" in out
+    assert "**Step 3" not in out
+    assert out.count(big) == 2
+    assert "Token budget reached: Steps 1–2 replayed; 3 more step(s)" in out
+    assert "Call again from Step 3" in out
+
+
+def test_replay_always_shows_at_least_the_first_step():
+    reader = _reader([_friendly_step(1, tool_result="y" * 4000), _friendly_step(2)])
+    out = replay_steps_text(reader, 1, 2, max_tokens=10)
+    assert "**Step 1" in out
+    assert "y" * 4000 in out
+    assert "**Step 2" not in out
+
+
+def test_replay_screen_line_states():
+    with_text = replay_steps_text(_reader([_friendly_step(1)]), 1)
+    assert "[Screen]: Step 1 summary." in with_text
+    assert "[Summary]" not in with_text
+
+    pending = replay_steps_text(
+        _reader([_friendly_step(1, summary=None, extra_metadata={"summary_status": "pending"})]),
+        1,
+    )
+    assert "[Screen]: (screen description pending)" in pending
+
+    failed = replay_steps_text(
+        _reader([_friendly_step(1, summary="", extra_metadata={"summary_status": "failed"})]),
+        1,
+    )
+    assert "[Screen]: (screen description unavailable)" in failed
+
+    no_status = replay_steps_text(_reader([_friendly_step(1, summary=None)]), 1)
+    assert "[Screen]" not in no_status
+
+
+def test_replay_without_history():
+    assert "no execution history" in replay_steps_text(None, 1)
+
+
+@pytest.mark.asyncio
+async def test_tool_execute_reads_the_context_engine_and_degrades_without_one():
+    reader = _reader([_friendly_step(2)])
+    out = await ReplayStepsTool().execute(ctx=SimpleNamespace(data_engine=reader), start_step=2)
+    assert "- **Step 2 (Start: 6.0s)**" in out
+
+    out = await replay_steps.execute(ctx=SimpleNamespace(data_engine=None), start_step=1)
+    assert "no execution history" in out
+    assert replay_steps.is_available(SimpleNamespace(data_engine=None)) is False
+    assert replay_steps.is_available(SimpleNamespace(data_engine=reader)) is True

tests/unit/tools/history/test_result_messages.py

@@ -0,0 +1,120 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""tool_result_messages: one helper decides how a tool result — with or
+without images — enters the conversation, per LLM provider."""
+
+from enum import StrEnum
+from types import SimpleNamespace
+
+import pytest
+from langchain_core.messages import HumanMessage, ToolMessage
+
+from artemis.tools.tool_wrapper import (
+    resolve_image_carrier,
+    split_multimodal_result,
+    tool_result_messages,
+)
+
+IMAGE = {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,AAAA"}}
+SHOT = [{"type": "text", "text": "Screenshot of step 7 (pre-action) is attached."}, IMAGE]
+
+
+class _Provider(StrEnum):
+    GOOGLE = "google"
+    VERTEX_AI = "vertexai"
+
+
+def _llm(provider):
+    return SimpleNamespace(endpoint=SimpleNamespace(provider=provider))
+
+
+@pytest.mark.parametrize(
+    ("provider", "carrier"),
+    [
+        ("google", "tool"),
+        ("anthropic", "tool"),
+        ("vertexai", "human"),
+        ("openai", "human"),
+        ("openrouter", "human"),
+        ("xai", "human"),
+        ("ollama", "human"),
+        (None, "human"),
+    ],
+)
+def test_resolve_image_carrier_by_provider_name(provider, carrier):
+    assert resolve_image_carrier(provider) == carrier
+    assert resolve_image_carrier(_llm(provider)) == carrier
+
+
+def test_resolve_image_carrier_accepts_enums_and_endpoint_less_llms():
+    assert resolve_image_carrier(_Provider.GOOGLE) == "tool"
+    assert resolve_image_carrier(_llm(_Provider.VERTEX_AI)) == "human"
+    assert resolve_image_carrier(SimpleNamespace()) == "human"
+
+
+def test_split_multimodal_result():
+    assert split_multimodal_result("plain") == ("plain", [])
+    assert split_multimodal_result(None) == ("", [])
+    assert split_multimodal_result(ToolMessage(content="x", tool_call_id="t")) == ("x", [])
+    text, images = split_multimodal_result(SHOT)
+    assert text == "Screenshot of step 7 (pre-action) is attached."
+    assert images == [IMAGE]
+
+
+def test_text_only_results_stay_a_single_tool_message():
+    msgs = tool_result_messages("tc-1", "Step 3 replay", name="replay_steps", status="error")
+    assert len(msgs) == 1
+    assert isinstance(msgs[0], ToolMessage)
+    assert msgs[0].content == "Step 3 replay"
+    assert msgs[0].tool_call_id == "tc-1"
+    assert msgs[0].name == "replay_steps"
+    assert msgs[0].status == "error"
+
+    # Text-block lists pass through unchanged; other objects are stringified.
+    blocks = [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]
+    assert tool_result_messages("tc", blocks)[0].content == blocks
+    assert tool_result_messages("tc", {"k": 1})[0].content == "{'k': 1}"
+
+
+def test_tool_carrier_puts_the_image_inside_the_tool_message():
+    msgs = tool_result_messages("tc-2", SHOT, name="get_step_screenshot", llm=_llm("google"))
+    assert len(msgs) == 1
+    assert isinstance(msgs[0], ToolMessage)
+    assert msgs[0].content == SHOT
+    assert msgs[0].name == "get_step_screenshot"
+
+
+def test_human_carrier_splits_text_and_image_with_a_caption():
+    msgs = tool_result_messages(
+        "tc-3", SHOT, name="get_step_screenshot", status="success", llm=_llm("openai")
+    )
+    assert len(msgs) == 2
+    tool_msg, human_msg = msgs
+    assert isinstance(tool_msg, ToolMessage)
+    assert tool_msg.content == "Screenshot of step 7 (pre-action) is attached."
+    assert tool_msg.tool_call_id == "tc-3"
+    assert isinstance(human_msg, HumanMessage)
+    assert human_msg.content[0] == {
+        "type": "text",
+        "text": "[Screenshot returned by get_step_screenshot for step 7]",
+    }
+    assert human_msg.content[1] == IMAGE
+
+
+def test_explicit_carrier_wins_over_the_llm():
+    msgs = tool_result_messages("tc", SHOT, image_carrier="human", llm=_llm("google"))
+    assert len(msgs) == 2
+    msgs = tool_result_messages("tc", SHOT, image_carrier="tool", llm=_llm("openai"))
+    assert len(msgs) == 1

tests/unit/tools/history/test_screenshot.py

@@ -0,0 +1,109 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""get_step_screenshot: pre / post / overlay, missing files, invalid variants."""
+
+import base64
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+from PIL import Image
+
+from artemis.tools.history import get_step_screenshot, load_step_screenshot
+
+
+def _jpeg_file(tmp_path, name="pre.jpg", color="white"):
+    path = tmp_path / name
+    Image.new("RGB", (200, 400), color).save(path, format="JPEG")
+    return path
+
+
+def _reader(pre=None, post=None, action=None, record=True):
+    reader = MagicMock()
+    reader.get_step_image_path.side_effect = lambda n, which="pre": pre if which == "pre" else post
+    reader.get_step_record.return_value = SimpleNamespace(action_taken=action) if record else None
+    return reader
+
+
+def test_pre_and_post_screenshots_are_attached_as_content_blocks(tmp_path):
+    pre = _jpeg_file(tmp_path, "pre.jpg", "white")
+    post = _jpeg_file(tmp_path, "post.jpg", "black")
+    reader = _reader(pre=pre, post=post)
+
+    shot = load_step_screenshot(reader, 2, "pre")
+    reader.get_step_image_path.assert_called_with(2, "pre")
+    assert shot.image_bytes == pre.read_bytes()
+    assert shot.description == "Screenshot of step 2 (pre-action) is attached."
+    blocks = shot.to_content_blocks()
+    assert blocks[0] == {"type": "text", "text": shot.description}
+    assert blocks[1]["type"] == "image_url"
+    url = blocks[1]["image_url"]["url"]
+    assert url.startswith("data:image/jpeg;base64,")
+    assert base64.b64decode(url.split(",", 1)[1]) == pre.read_bytes()
+
+    shot = load_step_screenshot(reader, 2, "post")
+    reader.get_step_image_path.assert_called_with(2, "post")
+    assert shot.image_bytes == post.read_bytes()
+    assert "post-action" in shot.description
+
+
+def test_overlay_draws_the_recorded_raw_action(tmp_path):
+    pre = _jpeg_file(tmp_path)
+    reader = _reader(pre=pre, action={"action": "click", "target": [100, 200]})
+
+    shot = load_step_screenshot(reader, 2, "overlay")
+    reader.get_step_image_path.assert_called_once_with(2, "pre")
+    assert shot.overlay_drawn is True
+    assert "action drawn on it" in shot.description
+    assert shot.image_bytes != pre.read_bytes()
+
+
+def test_overlay_falls_back_to_plain_pre_without_action(tmp_path):
+    pre = _jpeg_file(tmp_path)
+    shot = load_step_screenshot(_reader(pre=pre, action=None), 2, "overlay")
+    assert shot.overlay_drawn is False
+    assert shot.image_bytes == pre.read_bytes()
+    assert "plain screenshot" in shot.description
+
+
+def test_missing_file_and_missing_step_return_plain_text():
+    shot = load_step_screenshot(_reader(pre=None, post=None), 3, "post")
+    assert shot.image_bytes is None
+    assert shot.to_content_blocks() == "No post-action screenshot recorded for step 3."
+
+    shot = load_step_screenshot(_reader(pre=None, record=False), 9, "pre")
+    assert shot.to_content_blocks() == "Error: step 9 not found."
+
+
+def test_invalid_variant_and_step_number_are_rejected():
+    shot = load_step_screenshot(_reader(), 1, "bogus")
+    assert shot.image_bytes is None
+    assert "'pre', 'post' or 'overlay'" in shot.description
+    assert "must be an integer" in load_step_screenshot(_reader(), "one").description
+    assert "no execution history" in load_step_screenshot(None, 1).description
+
+
+@pytest.mark.asyncio
+async def test_tool_execute_returns_blocks_or_text(tmp_path):
+    pre = _jpeg_file(tmp_path)
+    ctx = SimpleNamespace(data_engine=_reader(pre=pre))
+    out = await get_step_screenshot.execute(ctx=ctx, step_number=1, which="pre")
+    assert isinstance(out, list) and out[1]["type"] == "image_url"
+
+    out = await get_step_screenshot.execute(ctx=ctx, step_number=1, which="post")
+    assert out == "No post-action screenshot recorded for step 1."
+
+    out = await get_step_screenshot.execute(ctx=SimpleNamespace(data_engine=None), step_number=1)
+    assert "no execution history" in out

tests/unit/tools/history/test_search.py

@@ -0,0 +1,334 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""search_history boundaries and search correctness.
+
+Hard boundaries under test: bounded result count, bounded response tokens,
+every result carrying a step number. Plus: keyword/step-range/notes/chunk
+filter correctness, the step-range ledger re-entry (full-width
+``build_action_ledger`` rows), and the screen-text surface: the step's own
+pre AND post screenshots plus screenshots its tool results embedded.
+"""
+
+import hashlib
+import io
+import re
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+from PIL import Image
+
+from artemis.context import ArtemisContext
+from artemis.data_engine.engine import DataEngine
+from artemis.tools.history import (
+    SearchHistoryTool,
+    search_history,
+    search_history_available,
+    search_history_text,
+)
+from artemis.tools.history.search import result_search_text
+from artemis.utils.notes import save_note_content
+
+
+def _jpeg(color: str) -> bytes:
+    buf = io.BytesIO()
+    Image.new("RGB", (64, 128), color).save(buf, format="JPEG")
+    return buf.getvalue()
+
+
+def _make_engine(tmp_path):
+    mock_ctx = MagicMock(spec=ArtemisContext)
+    mock_execution_setup = MagicMock()
+    mock_execution_setup.traces_path = str(tmp_path)
+    mock_ctx.execution_setup = mock_execution_setup
+    mock_ctx.device = None
+    engine = DataEngine(mock_ctx)
+    engine.start_session("search test session")
+    return engine
+
+
+def _flush(engine):
+    for t in list(engine._pending_threads):
+        t.join()
+
+
+def _cfg(**overrides):
+    values = {"enabled": True, "max_results": 5, "max_text_tokens": 2000, "screen_scan_steps": 150}
+    values.update(overrides)
+    return SimpleNamespace(**values)
+
+
+@pytest.fixture
+def engine(tmp_path):
+    engine = _make_engine(tmp_path)
+    engine.record_step(
+        pre_screenshot_bytes=_jpeg("red"),
+        summary="Opened the login page.",
+        action_taken={"action": "click", "target_text": "Login entry"},
+        last_execution_result={"status": "success"},
+    )
+    engine.record_step(
+        pre_screenshot_bytes=_jpeg("green"),
+        post_screenshot_bytes=_jpeg("blue"),
+        summary="Typed the verification code 4711 into the input.",
+        action_taken={"action": "input_text", "text": "4711"},
+        last_execution_result={"status": "success"},
+        operator_raw_thinking="The code arrived via SMS; entering 4711 now.",
+    )
+    engine.record_step(
+        summary="A promo popup appeared and was dismissed.",
+        action_taken={"action": "click", "target_text": "Close popup"},
+        last_execution_result={"status": "failed", "error": "element vanished"},
+    )
+    _flush(engine)
+    return engine
+
+
+def _result_blocks(text: str) -> list[str]:
+    return [b for b in text.split("\n") if b.startswith("[")]
+
+
+def test_keyword_search_finds_step_and_carries_step_number(engine):
+    out = search_history_text(engine, query="verification code", recall_config=_cfg())
+    assert isinstance(out, str)
+    assert "Step 2" in out
+    assert "4711" in out
+    # Every result header names a step / step range / step anchor.
+    for header in _result_blocks(out):
+        assert re.search(r"Step[s]? \d", header), header
+
+
+def test_step_range_filters_matches(engine):
+    out = search_history_text(engine, query="popup", step_range=[1, 2], recall_config=_cfg())
+    # Step 3 (the popup step) is outside the range: only the range ledger and
+    # a no-match line may appear.
+    assert "A promo popup appeared" not in out
+    assert "No matches" in out
+
+
+def test_step_range_returns_full_width_ledger_rows(engine):
+    """The compressed-history marker line's re-entry point: a step range
+    returns that range's full per-step action ledger."""
+    out = search_history_text(engine, query="", step_range=[3, 1], recall_config=_cfg())
+    assert "Action ledger for Steps 1–3" in out
+    # Full-width build_action_ledger rows: step number + T+mm:ss offset +
+    # semantic action + result phrase.
+    assert re.search(r"- Step 1 \(T\+\d{2}:\d{2}\): .*Login entry", out)
+    assert re.search(r"- Step 2 \(T\+\d{2}:\d{2}\): ", out)
+    assert re.search(r"- Step 3 \(T\+\d{2}:\d{2}\): ", out)
+
+
+def test_result_count_is_clamped_by_config(engine):
+    for i in range(8):
+        engine.record_step(
+            summary=f"Scrolled the anchorterm feed page {i}.",
+            action_taken={"action": "swipe"},
+            last_execution_result={"status": "success"},
+        )
+    _flush(engine)
+    out = search_history_text(
+        engine, query="anchorterm", max_results=50, recall_config=_cfg(max_results=3)
+    )
+    assert out.count("[Step ") == 3
+    assert "more not shown" in out
+
+
+def test_response_is_truncated_at_token_budget(engine):
+    for i in range(6):
+        engine.record_step(
+            summary=f"needleword page {i} " + "filler content " * 60,
+            action_taken={"action": "click"},
+            last_execution_result={"status": "success"},
+        )
+    _flush(engine)
+    out = search_history_text(
+        engine, query="needleword", max_results=6, recall_config=_cfg(max_text_tokens=150)
+    )
+    assert len(out) <= 150 * 4 + 200  # budget + truncation notice
+    assert "truncated at the search token budget" in out
+
+
+def test_notes_are_searchable_and_anchored_to_a_step(engine):
+    save_note_content(engine.base_dir, "login_flow", "The OTP entry lives behind the SMS tab.")
+    out = search_history_text(engine, query="OTP entry", recall_config=_cfg())
+    assert "[Note 'login_flow'" in out
+    assert re.search(r"as of Step \d|last written at Step \d", out)
+
+
+def test_chunk_rows_are_searchable(engine):
+    engine.record_history_chunk(
+        start_step_number=1,
+        end_step_number=2,
+        version=1,
+        status="ready",
+        band2="  - Steps 1–2: walked the chunkneedle flow",
+        band3="- Step 1 (T+00:01): click -> executed",
+        rendered_text=(
+            "[Chunk 1 | Steps 1–2]\n② Compressed step summary\n"
+            "  - Steps 1–2: walked the chunkneedle flow\n"
+            "③ Step action ledger\n- Step 1 (T+00:01): click -> executed"
+        ),
+    )
+    _flush(engine)
+    out = search_history_text(engine, query="chunkneedle", recall_config=_cfg())
+    assert "[History chunk | Steps 1–2" in out
+
+
+def test_foreground_app_stamp_joins_search_surface(tmp_path):
+    """A package-name query hits the record_step foreground_app stamp."""
+    engine = _make_engine(tmp_path)
+    engine.record_step(
+        summary="Opened the alarms list.",
+        action_taken={"action": "click", "target_text": "Alarm"},
+        ui_tree=[{"package": "com.google.android.deskclock", "bounds": "[0,0][9,9]"}],
+    )
+    _flush(engine)
+
+    out = search_history_text(engine, query="deskclock", recall_config=_cfg())
+    assert "Step 1" in out
+    assert "No matches" not in out
+
+
+def test_tool_calls_of_a_step_are_searchable(engine):
+    engine.record_step(
+        summary="Asked the explorer.",
+        action_taken={"action": "click", "target_text": "Save"},
+        last_execution_result={"status": "success"},
+    )
+    engine.record_trace(
+        type="tool",
+        name="ask_explorer",
+        payload={"args": {"question": "toolneedle?"}, "result": "The resultneedle is ON."},
+        step_id=engine.last_recorded_step_id,
+    )
+    _flush(engine)
+    assert "[Step 4 " in search_history_text(engine, query="toolneedle", recall_config=_cfg())
+    assert "[Step 4 " in search_history_text(engine, query="resultneedle", recall_config=_cfg())
+
+
+# --- Screen text: pre + post screenshots and referenced screenshots ----------------------
+
+
+def test_post_screenshot_text_joins_the_search_surface(engine):
+    """OCR text that only exists on the step's post-action screenshot is found."""
+    record = engine.get_step_record(2)
+    engine.storage.update_image_data(
+        record.post_image_name,
+        ocr_result=[{"text": "Welcome back postneedle", "bounds": [0, 0, 10, 10]}],
+        ui_tree=None,
+    )
+    out = search_history_text(engine, query="postneedle", recall_config=_cfg())
+    assert "[Step 2 " in out
+
+
+def test_screen_scan_cap_limits_the_screenshot_sweep(engine):
+    record = engine.get_step_record(1)
+    engine.storage.update_image_data(
+        record.pre_image_name,
+        ocr_result=[{"text": "capneedle", "bounds": [0, 0, 10, 10]}],
+        ui_tree=None,
+    )
+    # Only the most recent step is scanned: Step 1's OCR is out of reach ...
+    out = search_history_text(engine, query="capneedle", recall_config=_cfg(screen_scan_steps=1))
+    assert "No matches" in out
+    # ... unless the range narrows the sweep onto it.
+    out = search_history_text(
+        engine, query="capneedle", step_range=[1, 1], recall_config=_cfg(screen_scan_steps=1)
+    )
+    assert "[Step 1 " in out
+
+
+def test_embedded_screenshot_description_joins_the_search_surface(tmp_path):
+    """A step whose tool result embedded an earlier screenshot is found by
+    that screenshot's OCR text: the image is described, not stripped."""
+    engine = _make_engine(tmp_path)
+    pre_bytes = _jpeg("red")
+    engine.record_step(
+        pre_screenshot_bytes=pre_bytes,
+        summary="Opened the alarms list.",
+        action_taken={"action": "click", "target_text": "Alarm"},
+        ocr_result=[{"text": "Alarm 07:30 ocrneedle", "bounds": [0, 0, 10, 10]}],
+    )
+    engine.record_step(
+        summary="Looked back at the alarm screen.",
+        action_taken={"action": "click", "target_text": "Save"},
+    )
+    engine.record_trace(
+        type="tool",
+        name="get_step_screenshot",
+        payload={
+            "args": {"step_number": 1, "which": "pre"},
+            "result": [
+                {"type": "text", "text": "Screenshot of step 1 (pre-action) is attached."},
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"<ImageRef: sha256={hashlib.sha256(pre_bytes).hexdigest()} length=1>"
+                    },
+                },
+            ],
+        },
+        step_id=engine.last_recorded_step_id,
+    )
+    _flush(engine)
+
+    out = search_history_text(engine, query="ocrneedle", recall_config=_cfg())
+    assert "[Step 1 " in out
+    assert "[Step 2 " in out
+    assert "base64" not in out
+
+
+def test_result_search_text_strips_image_blocks():
+    blocks = [
+        {"type": "text", "text": "hello needle"},
+        {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,QUJDRA=="}},
+    ]
+    text = result_search_text(blocks)
+    assert "hello needle" in text
+    assert "base64" not in text and "QUJDRA" not in text
+    assert result_search_text({"answer": "yes"}) == '{"answer": "yes"}'
+    assert result_search_text(None) == ""
+
+
+# --- Tool surface ---------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_execute_requires_query_or_range(engine):
+    tool = SearchHistoryTool()
+    out = await tool.execute(ctx=SimpleNamespace(data_engine=engine), query="", step_range=None)
+    assert "needs a query and/or a step_range" in out
+
+
+@pytest.mark.asyncio
+async def test_execute_searches_and_never_returns_images(engine):
+    out = await search_history.execute(
+        ctx=SimpleNamespace(data_engine=engine), query="verification code"
+    )
+    assert isinstance(out, str)
+    assert "Step 2" in out
+
+
+@pytest.mark.asyncio
+async def test_execute_without_engine_degrades():
+    out = await search_history.execute(ctx=SimpleNamespace(data_engine=None), query="x")
+    assert "no active execution history" in out
+
+
+def test_availability_requires_engine_and_config():
+    assert search_history.is_available(None) is False
+    assert search_history.is_available(SimpleNamespace(data_engine=None)) is False
+    assert search_history.is_available(SimpleNamespace(data_engine=object())) is True
+    assert search_history_available(SimpleNamespace(data_engine=object())) is True

tests/unit/tools/test_committee_tool.py

@@ -89,7 +89,7 @@ class TestCommitteeTool(unittest.IsolatedAsyncioTestCase):
             mock_response_tool_call = MagicMock()
             mock_response_tool_call.tool_calls = [
                 {
-                    "name": "get_step_details",
+                    "name": "replay_steps",
                     "args": {"start_step": 1, "end_step": 1},
                     "id": "call_1",
                 }

tests/unit/tools/test_history_recall.py

@@ -1,268 +0,0 @@
-# Copyright 2026 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""recall_history boundaries and search correctness (M4, design §10).
-
-Hard boundaries under test: bounded result count, bounded response tokens,
-image recall limited to the real stored screenshots of a single step, and
-every result carrying a step number. Plus: keyword/step-range/notes filter
-correctness and the era-marker re-entry (a step_range returns the full-width
-``build_action_ledger`` rows for that range).
-"""
-
-import io
-import re
-from types import SimpleNamespace
-from unittest.mock import MagicMock
-
-import pytest
-from PIL import Image
-
-from artemis.context import ArtemisContext
-from artemis.data_engine.engine import DataEngine
-from artemis.tools.history_recall import (
-    RecallHistoryTool,
-    recall_history,
-    search_history,
-)
-from artemis.utils.notes import save_note_content
-
-
-def _jpeg(color: str) -> bytes:
-    buf = io.BytesIO()
-    Image.new("RGB", (64, 128), color).save(buf, format="JPEG")
-    return buf.getvalue()
-
-
-def _make_engine(tmp_path):
-    mock_ctx = MagicMock(spec=ArtemisContext)
-    mock_execution_setup = MagicMock()
-    mock_execution_setup.traces_path = str(tmp_path)
-    mock_ctx.execution_setup = mock_execution_setup
-    mock_ctx.device = None
-    engine = DataEngine(mock_ctx)
-    engine.start_session("recall test session")
-    return engine
-
-
-def _flush(engine):
-    for t in list(engine._pending_threads):
-        t.join()
-
-
-def _cfg(**overrides):
-    values = {"enabled": True, "max_results": 5, "max_text_tokens": 2000, "max_image_steps": 1}
-    values.update(overrides)
-    return SimpleNamespace(**values)
-
-
-@pytest.fixture
-def engine(tmp_path):
-    engine = _make_engine(tmp_path)
-    engine.record_step(
-        pre_screenshot_bytes=_jpeg("red"),
-        summary="Opened the login page.",
-        action_taken={"action": "click", "target_text": "Login entry"},
-        last_execution_result={"status": "success"},
-    )
-    engine.record_step(
-        pre_screenshot_bytes=_jpeg("green"),
-        post_screenshot_bytes=_jpeg("blue"),
-        summary="Typed the verification code 4711 into the input.",
-        action_taken={"action": "input_text", "text": "4711"},
-        last_execution_result={"status": "success"},
-        operator_raw_thinking="The code arrived via SMS; entering 4711 now.",
-    )
-    engine.record_step(
-        summary="A promo popup appeared and was dismissed.",
-        action_taken={"action": "click", "target_text": "Close popup"},
-        last_execution_result={"status": "failed", "error": "element vanished"},
-    )
-    _flush(engine)
-    return engine
-
-
-def _result_blocks(text: str) -> list[str]:
-    return [b for b in text.split("\n") if b.startswith("[")]
-
-
-def test_keyword_search_finds_step_and_carries_step_number(engine):
-    out = search_history(engine, query="verification code", recall_config=_cfg())
-    assert isinstance(out, str)
-    assert "Step 2" in out
-    assert "4711" in out
-    # Every result header names a step / step range / step anchor.
-    for header in _result_blocks(out):
-        assert re.search(r"Step[s]? \d", header), header
-
-
-def test_step_range_filters_matches(engine):
-    out = search_history(engine, query="popup", step_range=[1, 2], recall_config=_cfg())
-    # Step 3 (the popup step) is outside the range: only the range ledger and
-    # a no-match line may appear.
-    assert "A promo popup appeared" not in out
-    assert "No matches" in out
-
-
-def test_step_range_returns_full_width_ledger_rows(engine):
-    """The recall-only era period paragraph's '(Step-level ledger via
-    recall_history for steps a–b)' re-entry point."""
-    out = search_history(engine, query="", step_range=[1, 3], recall_config=_cfg())
-    assert "Action ledger for Steps 1–3" in out
-    # Full-width build_action_ledger rows: step number + T+mm:ss offset +
-    # semantic action + result phrase.
-    assert re.search(r"- Step 1 \(T\+\d{2}:\d{2}\): .*Login entry", out)
-    assert re.search(r"- Step 2 \(T\+\d{2}:\d{2}\): ", out)
-    assert re.search(r"- Step 3 \(T\+\d{2}:\d{2}\): ", out)
-
-
-def test_result_count_is_clamped_by_config(engine):
-    for i in range(8):
-        engine.record_step(
-            summary=f"Scrolled the anchorterm feed page {i}.",
-            action_taken={"action": "swipe"},
-            last_execution_result={"status": "success"},
-        )
-    _flush(engine)
-    out = search_history(
-        engine, query="anchorterm", max_results=50, recall_config=_cfg(max_results=3)
-    )
-    assert out.count("[Step ") == 3
-    assert "more not shown" in out
-
-
-def test_response_is_truncated_at_token_budget(engine):
-    engine.record_step(
-        summary="needleword " + "filler content " * 400,
-        action_taken={"action": "click"},
-        last_execution_result={"status": "success"},
-    )
-    _flush(engine)
-    out = search_history(
-        engine,
-        query="needleword",
-        include_details=True,
-        recall_config=_cfg(max_text_tokens=150),
-    )
-    assert len(out) <= 150 * 4 + 200  # budget + truncation notice
-    assert "truncated at the recall token budget" in out
-
-
-def test_notes_are_searchable_and_anchored_to_a_step(engine):
-    save_note_content(engine.base_dir, "login_flow", "The OTP entry lives behind the SMS tab.")
-    out = search_history(engine, query="OTP entry", recall_config=_cfg())
-    assert "[Note 'login_flow'" in out
-    assert re.search(r"as of Step \d|last written at Step \d", out)
-
-
-def test_chunk_rows_are_searchable(engine):
-    engine.record_history_chunk(
-        start_step_number=1,
-        end_step_number=2,
-        version=1,
-        status="ready",
-        band2="  - Steps 1–2: walked the chunkneedle flow",
-        band3="- Step 1 (T+00:01): click -> executed",
-        rendered_text=(
-            "[Chunk 1 | Steps 1–2]\n② Compressed step summary\n"
-            "  - Steps 1–2: walked the chunkneedle flow\n"
-            "③ Step action ledger\n- Step 1 (T+00:01): click -> executed"
-        ),
-    )
-    _flush(engine)
-    out = search_history(engine, query="chunkneedle", recall_config=_cfg())
-    assert "[History chunk | Steps 1–2" in out
-
-
-def test_images_only_for_a_single_step_and_only_real_files(engine):
-    out = search_history(
-        engine,
-        query="verification code",
-        include_images=True,
-        recall_config=_cfg(),
-    )
-    assert isinstance(out, list)
-    images = [b for b in out if b.get("type") == "image_url"]
-    # Step 2 has a real pre and post screenshot on disk — but never more than
-    # one step's worth of images.
-    assert 1 <= len(images) <= 2
-    labels = [
-        b["text"]
-        for b in out
-        if b.get("type") == "text" and "screenshot of Step" in b.get("text", "")
-    ]
-    assert all("Step 2" in label for label in labels)
-    for block in images:
-        assert block["image_url"]["url"].startswith("data:image/jpeg;base64,")
-
-
-def test_images_disabled_by_config_cap(engine):
-    out = search_history(
-        engine,
-        query="verification code",
-        include_images=True,
-        recall_config=_cfg(max_image_steps=0),
-    )
-    assert isinstance(out, str)
-
-
-def test_missing_image_files_yield_note_not_blocks(engine):
-    # Step 3 recorded no screenshots at all.
-    out = search_history(
-        engine,
-        query="promo popup",
-        include_images=True,
-        recall_config=_cfg(),
-    )
-    assert isinstance(out, list)
-    assert not [b for b in out if b.get("type") == "image_url"]
-    assert any(
-        "no stored screenshots exist" in b.get("text", "") for b in out if b.get("type") == "text"
-    )
-
-
-@pytest.mark.asyncio
-async def test_execute_requires_query_or_range(engine):
-    tool = RecallHistoryTool()
-    ctx = SimpleNamespace(data_engine=engine)
-    out = await tool.execute(ctx=ctx, query="", step_range=None)
-    assert "needs a query and/or a step_range" in out
-
-
-@pytest.mark.asyncio
-async def test_execute_without_engine_degrades():
-    tool = RecallHistoryTool()
-    out = await tool.execute(ctx=SimpleNamespace(data_engine=None), query="x")
-    assert "no active DataEngine session" in out
-
-
-def test_availability_requires_engine_and_config():
-    assert recall_history.is_available(None) is False
-    assert recall_history.is_available(SimpleNamespace(data_engine=None)) is False
-    assert recall_history.is_available(SimpleNamespace(data_engine=object())) is True
-
-
-def test_foreground_app_stamp_joins_search_surface(tmp_path):
-    """M5: a package-name query hits the record_step foreground_app stamp."""
-    engine = _make_engine(tmp_path)
-    engine.record_step(
-        summary="Opened the alarms list.",
-        action_taken={"action": "click", "target_text": "Alarm"},
-        ui_tree=[{"package": "com.google.android.deskclock", "bounds": "[0,0][9,9]"}],
-    )
-    _flush(engine)
-
-    out = search_history(engine, query="deskclock", recall_config=_cfg())
-    assert "Step 1" in out
-    assert "No matches" not in out