feat: unify history tools and improve execution reliability

tests/unit/utils/test_coordinate_space.py

@@ -0,0 +1,104 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Coordinate-space contract of recorded actions.
+
+Pro records pixels (``coordinate_space="pixel"``), Flash records the model's
+own 0–1000 values (``coordinate_space="normalized"``); every model-facing
+render normalizes, and the explicit marker — never a magnitude heuristic —
+is what makes that pass idempotent.
+"""
+
+from artemis.utils.coordinates import (
+    COORDINATE_SPACE_KEY,
+    COORDINATE_SPACE_NORMALIZED,
+    COORDINATE_SPACE_PIXEL,
+    normalize_action_dict,
+    normalize_any_structure,
+)
+
+W, H = 1080, 2400
+
+
+def test_pixel_record_is_normalized_once_and_stamped():
+    pixel = {
+        "action": "tap",
+        "coordinates": [540, 1200],
+        "normalized_coordinates": [500, 500],
+        COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
+    }
+    once = normalize_action_dict(pixel, W, H)
+    assert once["coordinates"] == [500, 500]
+    assert once[COORDINATE_SPACE_KEY] == COORDINATE_SPACE_NORMALIZED
+    assert pixel["coordinates"] == [540, 1200]  # input untouched
+
+    twice = normalize_action_dict(once, W, H)
+    assert twice == once
+
+
+def test_normalized_record_is_returned_verbatim_even_when_values_look_like_pixels():
+    """No magnitude guessing: a Flash record of [320, 399] stays [320, 399]
+    (the buggy double pass used to turn it into [296, 166])."""
+    flash = {
+        "action": "click",
+        "coordinates": [320, 399],
+        COORDINATE_SPACE_KEY: COORDINATE_SPACE_NORMALIZED,
+        "args": {"target": [320, 399]},
+    }
+    out = normalize_action_dict(flash, W, H)
+    assert out == flash
+    assert out is not flash
+
+    big = dict(flash, coordinates=[1080, 2400])
+    assert normalize_action_dict(big, W, H)["coordinates"] == [1080, 2400]
+
+
+def test_legacy_unstamped_record_is_converted_once_then_idempotent():
+    legacy = {"action": "swipe", "coordinates": [540, 1800, 540, 600]}
+    once = normalize_action_dict(legacy, W, H)
+    assert once["coordinates"] == [500, 750, 500, 250]
+    assert once[COORDINATE_SPACE_KEY] == COORDINATE_SPACE_NORMALIZED
+    assert normalize_action_dict(once, W, H) == once
+
+    # A dict without coordinate fields gets no marker (nothing was converted).
+    assert COORDINATE_SPACE_KEY not in normalize_action_dict({"action": "press_key"}, W, H)
+
+
+def test_normalize_any_structure_recurses_into_incident_containers():
+    """An execution incident holds an ``action`` *dict* — it is a container,
+    not an action item, so its nested action is normalized too."""
+    report = {
+        "status": "failed",
+        "execution": [
+            {
+                "action": "tap",
+                "coordinates": [540, 1200],
+                COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
+                "attempts": ["Error"],
+            }
+        ],
+        "incident": {
+            "kind": "exec_error",
+            "action": {
+                "action": "tap",
+                "coordinates": [540, 1200],
+                COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
+            },
+        },
+    }
+    out = normalize_any_structure(report, W, H)
+    assert out["execution"][0]["coordinates"] == [500, 500]
+    assert out["incident"]["action"]["coordinates"] == [500, 500]
+    assert out["incident"]["kind"] == "exec_error"
+    assert normalize_any_structure(out, W, H) == out

tests/unit/utils/test_task_tree.py

@@ -621,3 +621,133 @@ def test_get_recent_subgoal_hashes_robust_milestone_exclusion(tmp_path):
 
     # 5. Older failed subgoals from previously completed milestones (hash_failed_old)
     assert hash_failed_old not in keep_hashes
+
+
+# --- Step replay (shared renderer for replay_steps / MCP view_step_details) ----------
+
+
+def _replay_step(tool_result, **overrides):
+    step = {
+        "step_id": "step_4",
+        "step_number": 4,
+        "relative_time": "12.0s",
+        "summary": "Confirmed the Wi-Fi toggle state via the explorer.",
+        "action_taken": {"action": "click", "target": [0.5, 0.5], "target_text": "Save"},
+        "last_execution_result": {"status": "success"},
+        "interleaved_events": [
+            {"type": "thought", "content": "Need to confirm the toggle before saving."},
+            {
+                "type": "tool_call",
+                "name": "ask_explorer",
+                "args": {"question": "Is the Wi-Fi toggle on?"},
+                "result": tool_result,
+            },
+            {"type": "tool_call", "name": "click", "args": {"target": [0.5, 0.5]}, "result": "ok"},
+        ],
+    }
+    step.update(overrides)
+    return step
+
+
+def test_render_step_replay_shows_summary_thoughts_and_full_tool_results():
+    from artemis.utils.task_tree import render_step_replay
+
+    long_result = "The Wi-Fi toggle is ON and the SSID row reads 'HomeNet'. " * 60  # ~3.5k chars
+    out = render_step_replay(_replay_step(long_result))
+
+    assert "- **Step 4 (Start: 12.0s)**" in out
+    assert "[Screen]: Confirmed the Wi-Fi toggle state via the explorer." in out
+    assert "Need to confirm the toggle before saving." in out
+    # The tool call is replayed verbatim: name, args and the whole result.
+    assert "`ask_explorer(" in out
+    assert long_result in out
+    # Action tools are the planned action, not a tool-call line.
+    assert "[Planned Action]: Tapped 'Save' at [0.5, 0.5]" in out
+    assert "`click(" not in out
+
+
+def test_render_step_replay_clamps_only_pathological_results():
+    from artemis.utils.task_tree import REPLAY_RESULT_CHARS, render_step_replay
+
+    huge = "y" * (REPLAY_RESULT_CHARS + 500)
+    out = render_step_replay(_replay_step(huge))
+    assert huge not in out
+    assert ("y" * REPLAY_RESULT_CHARS) + "..." in out
+
+
+def test_live_window_keeps_the_tight_tool_result_clamp():
+    """The Operator's live context window still clamps tool results tightly;
+    only the replay renderer is loose."""
+    from artemis.utils.task_tree import LIVE_RESULT_CHARS
+
+    long_result = "z" * 3000
+    out = build_plan_and_history(
+        "- [ ] Check toggle", [_replay_step(long_result)], "default", last_n_detailed=1
+    )
+    assert long_result not in out
+    assert ("z" * LIVE_RESULT_CHARS) + "..." in out
+    # No screen-description line in the live detailed view.
+    assert "[Screen]:" not in out
+
+
+def test_render_step_replay_reports_missing_screen_description_status():
+    from artemis.utils.task_tree import render_step_replay
+
+    pending = render_step_replay(
+        _replay_step("ok", summary=None, extra_metadata={"summary_status": "pending"})
+    )
+    assert "[Screen]: (screen description pending)" in pending
+    failed = render_step_replay(
+        _replay_step("ok", summary="", extra_metadata={"summary_status": "failed"})
+    )
+    assert "[Screen]: (screen description unavailable)" in failed
+    assert "[Screen]" not in render_step_replay(_replay_step("ok", summary=None))
+
+
+def test_render_step_replay_shows_described_images_in_tool_results():
+    from artemis.utils.task_tree import render_step_replay
+
+    result = [
+        {"type": "text", "text": "--- pre-action screenshot of Step 1 ---"},
+        {"type": "text", "text": "[screenshot: pre-action of Step 1]", "image_name": "ab" * 32},
+    ]
+    out = render_step_replay(_replay_step(result))
+    assert "[screenshot: pre-action of Step 1]" in out
+
+
+def test_format_action_clean_reads_flash_arguments_and_maps_manage_app():
+    """Read Flash action arguments from ``args`` when formatting app launches."""
+    from artemis.utils.task_tree import format_action_clean
+
+    assert (
+        format_action_clean(
+            {
+                "action": "manage_app",
+                "coordinates": None,
+                "args": {"action": "launch", "app_name": "com.android.settings"},
+            }
+        )
+        == "Launched app 'com.android.settings'"
+    )
+    assert (
+        format_action_clean({"action": "manage_app", "args": {"action": "stop", "app_name": "x"}})
+        == "Stopped app 'x'"
+    )
+    # Pro's own shape keeps rendering as before.
+    assert format_action_clean({"action": "launch_app", "app_name": "x"}) == "Launched app 'x'"
+    assert (
+        format_action_clean({"action": "manage_app", "app_name": "x", "intent": "clear"})
+        == "Managed app 'x' (action: clear)"
+    )
+    # Top-level fields win over args; args only fill the gaps.
+    assert (
+        format_action_clean(
+            {
+                "action": "click",
+                "coordinates": [1, 2],
+                "target_text": "Save",
+                "args": {"target": [9, 9]},
+            }
+        )
+        == "Tapped 'Save' at [1, 2]"
+    )

tests/unit/utils/test_video_manifest.py

@@ -0,0 +1,132 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Recording playlist manifest: session-relative segment offsets."""
+
+import json
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from artemis.utils.video import RECORDING_MANIFEST_VERSION, write_recording_manifest
+
+
+def _probe(durations: dict[str, float]):
+    async def probe(path):
+        return {"duration": durations[path.name], "width": 1080, "height": 1920}
+
+    return AsyncMock(side_effect=probe)
+
+
+@pytest.mark.asyncio
+async def test_manifest_carries_session_offsets_and_restart_gaps(tmp_path):
+    seg0 = tmp_path / "recording.mp4"
+    seg1 = tmp_path / "recording_001.mp4"
+    seg2 = tmp_path / "recording_002.mp4"
+    for path in (seg0, seg1, seg2):
+        path.write_bytes(b"mp4")
+
+    # Recording started 1.2s after the session; scrcpy restart gaps of 1.5s and 0.8s.
+    offsets = {seg0: 1.2, seg1: 12.7, seg2: 18.5}
+
+    with patch(
+        "artemis.utils.video.probe_video_segment",
+        _probe({seg0.name: 10.0, seg1.name: 5.0, seg2.name: 4.0}),
+    ):
+        manifest_path = await write_recording_manifest(tmp_path, [seg0, seg1, seg2], offsets)
+
+    assert manifest_path == tmp_path / "recording.json"
+    payload = json.loads(manifest_path.read_text(encoding="utf-8"))
+    assert payload["version"] == RECORDING_MANIFEST_VERSION
+    # Legacy back-to-back timeline is preserved for the existing player.
+    assert payload["duration"] == 19.0
+    assert [s["start"] for s in payload["segments"]] == [0.0, 10.0, 15.0]
+    assert [s["duration"] for s in payload["segments"]] == [10.0, 5.0, 4.0]
+    # Real session-relative timeline, including the gaps between segments.
+    assert [s["offset_ms"] for s in payload["segments"]] == [1200, 12700, 18500]
+    assert [s["duration_ms"] for s in payload["segments"]] == [10000, 5000, 4000]
+    assert payload["session_offset_ms"] == 1200
+    assert payload["session_end_ms"] == 22500
+    assert [s["file"] for s in payload["segments"]] == [seg0.name, seg1.name, seg2.name]
+    assert all(s["width"] == 1080 and s["height"] == 1920 for s in payload["segments"])
+    assert not (tmp_path / "recording.part.json").exists()
+
+
+@pytest.mark.asyncio
+async def test_manifest_without_offsets_falls_back_to_back_to_back(tmp_path):
+    seg0 = tmp_path / "recording.mp4"
+    seg1 = tmp_path / "recording_001.mp4"
+    seg0.write_bytes(b"mp4")
+    seg1.write_bytes(b"mp4")
+
+    with patch(
+        "artemis.utils.video.probe_video_segment",
+        _probe({seg0.name: 3.25, seg1.name: 2.0}),
+    ):
+        manifest_path = await write_recording_manifest(tmp_path, [seg0, seg1])
+
+    payload = json.loads(manifest_path.read_text(encoding="utf-8"))
+    assert [s["offset_ms"] for s in payload["segments"]] == [0, 3250]
+    assert [s["duration_ms"] for s in payload["segments"]] == [3250, 2000]
+    assert payload["session_offset_ms"] == 0
+    assert payload["session_end_ms"] == 5250
+
+
+@pytest.mark.asyncio
+async def test_manifest_fills_missing_offset_after_known_segment(tmp_path):
+    seg0 = tmp_path / "recording.mp4"
+    seg1 = tmp_path / "recording_001.mp4"
+    seg0.write_bytes(b"mp4")
+    seg1.write_bytes(b"mp4")
+
+    with patch(
+        "artemis.utils.video.probe_video_segment",
+        _probe({seg0.name: 10.0, seg1.name: 5.0}),
+    ):
+        manifest_path = await write_recording_manifest(tmp_path, [seg0, seg1], {seg0: 2.0})
+
+    payload = json.loads(manifest_path.read_text(encoding="utf-8"))
+    # Unknown offsets continue right after the previous segment (no gap assumed).
+    assert [s["offset_ms"] for s in payload["segments"]] == [2000, 12000]
+
+
+@pytest.mark.asyncio
+async def test_manifest_skips_invalid_segments_and_clamps_negative_offsets(tmp_path):
+    good = tmp_path / "recording.mp4"
+    broken = tmp_path / "recording_001.mp4"
+    missing = tmp_path / "recording_002.mp4"
+    good.write_bytes(b"mp4")
+    broken.write_bytes(b"mp4")
+
+    with patch(
+        "artemis.utils.video.probe_video_segment",
+        _probe({good.name: 4.0, broken.name: 0.0}),
+    ):
+        manifest_path = await write_recording_manifest(
+            tmp_path, [good, broken, missing], {good: -0.4}
+        )
+
+    payload = json.loads(manifest_path.read_text(encoding="utf-8"))
+    assert [s["file"] for s in payload["segments"]] == [good.name]
+    assert payload["segments"][0]["offset_ms"] == 0
+
+
+@pytest.mark.asyncio
+async def test_manifest_returns_none_without_valid_segments(tmp_path):
+    with patch(
+        "artemis.utils.video.probe_video_segment",
+        AsyncMock(return_value={"duration": 0, "width": 0, "height": 0}),
+    ):
+        assert await write_recording_manifest(tmp_path, [tmp_path / "nope.mp4"]) is None
+    assert not (tmp_path / "recording.json").exists()