perf: preserve conversation prefixes across agent turns

tests/unit/memory/test_transcript_ledger.py

@@ -71,9 +71,19 @@ def _turn(i: int, tool_call_count: int = 1) -> list:
     return messages
 
 
-def _play_turn(ledger: TranscriptLedger, i: int, *, prev_key=None, prev_result=None, **turn_kwargs):
+def _play_turn(
+    ledger: TranscriptLedger,
+    i: int,
+    *,
+    prev_key=None,
+    prev_result=None,
+    compact=False,
+    **turn_kwargs,
+):
     """Mimic the operator's real ordering: commit previous, render, stage."""
     ledger.commit_staged(step_key=prev_key, validator_result=prev_result)
+    if compact:
+        ledger.compact_active()
     turn = _turn(i, **turn_kwargs)
     rendered = ledger.render([turn[0]])
     ledger.stage_turn(turn)
@@ -159,8 +169,8 @@ def test_recitation_and_ui_list_stripped_at_the_text_edge_and_screenshot_at_k():
     until depth K and is then resolved in place; only the live tail ever
     carries an indexed element list."""
     ledger = TranscriptLedger(step_memory=_service(), image_scrub_depth=3)
-    _play_turn(ledger, 1)
-    rendered = _play_turn(ledger, 2, prev_key="step-1")
+    _play_turn(ledger, 1, compact=True)
+    rendered = _play_turn(ledger, 2, compact=True, prev_key="step-1")
     obs1 = ledger.active_messages[0]
     # Turn 1 is now depth 2: text edge passed, screenshot kept.
     assert PLAN_RECITATION_MARKER not in str(obs1.content)
@@ -168,10 +178,10 @@ def test_recitation_and_ui_list_stripped_at_the_text_edge_and_screenshot_at_k():
     assert _has_image(obs1)
     original_obs1 = _fingerprint(obs1)
 
-    rendered = _play_turn(ledger, 3, prev_key="step-2")
+    rendered = _play_turn(ledger, 3, compact=True, prev_key="step-2")
     # Turn 1 reached depth 3: the screenshot is resolved.
-    assert not _has_image(obs1)
-    assert _fingerprint(obs1) != original_obs1
+    assert not _has_image(ledger.active_messages[0])
+    assert _fingerprint(ledger.active_messages[0]) != original_obs1
     # Turn 2 (depth 2): text edge passed, screenshot still there; the live tail keeps both.
     obs2 = next(m for m in ledger.active_messages if "OBSERVATION [T+00:02]" in str(m.content))
     assert PLAN_RECITATION_MARKER not in str(obs2.content)
@@ -189,7 +199,7 @@ def test_depth_k_image_resolved_to_ready_visual_summary():
 
     ledger = TranscriptLedger(step_memory=service, image_scrub_depth=3)
     for i in range(1, 5):
-        _play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
+        _play_turn(ledger, i, compact=True, prev_key=f"step-{i - 1}" if i > 1 else None)
 
     first_obs = ledger.active_messages[0]
     assert not _has_image(first_obs)
@@ -212,7 +222,7 @@ def test_pending_grace_then_placeholder_never_backfilled():
     ledger = TranscriptLedger(step_memory=service, image_scrub_depth=2, pending_grace_steps=1)
     # Enough turns to push turn 1 past K + grace.
     for i in range(1, 7):
-        _play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
+        _play_turn(ledger, i, compact=True, prev_key=f"step-{i - 1}" if i > 1 else None)
 
     first_obs = ledger.active_messages[0]
     assert not _has_image(first_obs)
@@ -221,7 +231,7 @@ def test_pending_grace_then_placeholder_never_backfilled():
 
     # A late summary must never mutate the frozen message.
     service._summaries["step-1"] = "Too late."
-    _play_turn(ledger, 7, prev_key="step-6")
+    _play_turn(ledger, 7, compact=True, prev_key="step-6")
     assert _fingerprint(ledger.active_messages[0]) == frozen
 
 
@@ -232,7 +242,7 @@ def test_failed_summary_becomes_unavailable_placeholder():
 
     ledger = TranscriptLedger(step_memory=service, image_scrub_depth=2, pending_grace_steps=5)
     for i in range(1, 4):
-        _play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
+        _play_turn(ledger, i, compact=True, prev_key=f"step-{i - 1}" if i > 1 else None)
 
     first_obs = ledger.active_messages[0]
     assert not _has_image(first_obs)
@@ -473,7 +483,7 @@ def test_relaxed_depth_below_start_gate_keeps_more_screenshots():
     assert ledger.occupancy is None
     assert ledger.effective_image_scrub_depth == 6
     for i in range(1, 10):
-        _play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
+        _play_turn(ledger, i, compact=True, prev_key=f"step-{i - 1}" if i > 1 else None)
     # At the render of turn 9 its observation is the live tail (depth 1);
     # committed turns 8..5 are depths 2..5 and turn 4 (depth 6) is scrubbed.
     ledger.record_prompt_tokens(20_000)  # 20% < 35%
@@ -482,34 +492,34 @@ def test_relaxed_depth_below_start_gate_keeps_more_screenshots():
 
     tight = TranscriptLedger(step_memory=_service(), image_scrub_depth=3)
     for i in range(1, 10):
-        _play_turn(tight, i, prev_key=f"step-{i - 1}" if i > 1 else None)
+        _play_turn(tight, i, compact=True, prev_key=f"step-{i - 1}" if i > 1 else None)
     assert _image_turns(tight) == [8]
 
 
 def test_crossing_the_start_gate_tightens_once_and_relaxing_never_backfills():
     ledger = _banded_ledger()
     for i in range(1, 10):
-        _play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
+        _play_turn(ledger, i, compact=True, prev_key=f"step-{i - 1}" if i > 1 else None)
     assert _image_turns(ledger) == [5, 6, 7, 8]
 
     # Past the gate: the next render scrubs the extra depths in one pass
     # (turns 5–8 were depths 3–6 at that render; only turn 9 at depth 2 stays).
     ledger.record_prompt_tokens(40_000)  # 40% >= 35%
     assert ledger.effective_image_scrub_depth == 3
-    _play_turn(ledger, 10, prev_key="step-9")
+    _play_turn(ledger, 10, compact=True, prev_key="step-9")
     assert _image_turns(ledger) == [9]
 
     # Back under the gate: nothing is restored; new turns accumulate again
     # until four historical screenshots are in view.
     ledger.record_prompt_tokens(20_000)
     assert ledger.effective_image_scrub_depth == 6
-    _play_turn(ledger, 11, prev_key="step-10")
+    _play_turn(ledger, 11, compact=True, prev_key="step-10")
     assert _image_turns(ledger) == [9, 10]
-    _play_turn(ledger, 12, prev_key="step-11")
+    _play_turn(ledger, 12, compact=True, prev_key="step-11")
     assert _image_turns(ledger) == [9, 10, 11]
-    _play_turn(ledger, 13, prev_key="step-12")
+    _play_turn(ledger, 13, compact=True, prev_key="step-12")
     assert _image_turns(ledger) == [9, 10, 11, 12]
-    _play_turn(ledger, 14, prev_key="step-13")  # turn 9 reaches depth 6
+    _play_turn(ledger, 14, compact=True, prev_key="step-13")  # turn 9 reaches depth 6
     assert _image_turns(ledger) == [10, 11, 12, 13]
 
 
@@ -523,10 +533,8 @@ def test_no_band_configuration_keeps_the_fixed_depth():
     assert relaxed_only.effective_image_scrub_depth == 3  # no gate → never relaxed
 
 
-def test_scrub_edge_runs_before_the_chunker_at_render():
-    """The chunker sees the active window after the scrub edge advanced, so a
-    chunk closing over turns older than the floor never captures a live
-    screenshot in its source transcript."""
+def test_chunker_sees_immutable_raw_history_before_a_phase_boundary():
+    """Preparing capsules must not scrub any previously sent observation."""
     ledger = _banded_ledger()
     seen: list[list[int]] = []
 
@@ -537,9 +545,9 @@ def test_scrub_edge_runs_before_the_chunker_at_render():
     ledger.attach_chunker(Probe())
     for i in range(1, 9):
         _play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
-    # At the render of turn 8 the scrub edge (depth 6) had already resolved
-    # turn 3 (depth 6); the probe saw turns 4–7 with images, never turn 3.
-    assert seen[-1] == [4, 5, 6, 7]
+    # A render below pressure preserves every historical image.
+    assert seen[-1] == list(range(1, 8))
+    assert ledger.context_phase == 0
 
 
 # ---------------------------------------------------------------------------
@@ -580,6 +588,7 @@ def test_ephemeral_blocks_vanish_at_the_text_edge_and_not_before():
     ledger.stage_turn(turn_with_notice(1))
     for i in range(2, 4):
         ledger.commit_staged(step_key=f"step-{i - 1}")
+        ledger.compact_active()
         ledger.render([_observation(i)])
         ledger.stage_turn(turn_with_notice(i))
 
@@ -600,15 +609,12 @@ def test_ephemeral_blocks_vanish_at_the_text_edge_and_not_before():
 
 
 def test_rendering_past_the_edge_is_byte_identical_and_prefix_stable():
-    """(iii) A render with nothing new is byte-identical to the previous one,
-    and across turns exactly one message — the observation reaching depth K —
-    differs from the previous render's prefix, so the request prefix before
-    it is what the provider's prompt cache already holds."""
+    """Every full previous prompt, including its live tail, survives unchanged."""
     ledger = TranscriptLedger(step_memory=_ready_service(12), image_scrub_depth=3)
     ledger.set_static_prefix([SystemMessage(content="STATIC")])
 
     def snapshot(rendered: list) -> list[str]:
-        return [_fingerprint(m) for m in rendered[:-1]]  # everything but the live tail
+        return [_fingerprint(m) for m in rendered]  # includes the previously live observation
 
     previous: list[str] = []
     for i in range(1, 11):
@@ -620,24 +626,15 @@ def test_rendering_past_the_edge_is_byte_identical_and_prefix_stable():
         )
         current = snapshot(rendered)
         changed = [idx for idx, fp in enumerate(previous) if current[idx] != fp]
-        if i >= 3:
-            # The live tail is depth 1 and committed turn i-1 depth 2, so the
-            # turn reaching depth 3 is i-2; its observation heads its span.
-            expected = next(
-                idx
-                for idx, m in enumerate(rendered[:-1])
-                if f"OBSERVATION [T+00:0{(i - 2) % 10}]" in str(m.content)
-            )
-            assert changed == [expected], (i, changed)
-        else:
-            assert changed == [], (i, changed)
+        assert changed == [], (i, changed)
         previous = current
 
     # Same state, rendered again: byte-identical, including the tail.
+    ledger.commit_staged(step_key="step-10", validator_result={"status": "dispatched"})
     tail = [_observation(11)]
     again = [_fingerprint(m) for m in ledger.render(tail)]
     assert again == [_fingerprint(m) for m in ledger.render(tail)]
-    assert again[:-1] == previous
+    assert again[: len(previous)] == previous
 
 
 # ---------------------------------------------------------------------------