Artemis-Eapil/tests/unit/agents/test_flash_scrub_edge.py

# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Unit tests for the Flash scrub-edge compressor (history redesign §3.2, M1).

Covers the M1 acceptance items: the legacy-equivalent success-path product
(asserted directly since the legacy ``compress_flash_messages`` reference was
removed in M5), the three race regressions (grace recovery, grace exhaustion
without backfill, failed-summary placeholder), the freeze invariant (bytes
behind the scrub edge never change), and depth-1 XML stripping equivalence.
"""

import json
from unittest.mock import Mock

import pytest
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage

from artemis.agents.flash.context_compressor import ScrubEdgeCompressor
from artemis.agents.flash.summarizer import VisualStepSummarizer
from artemis.context import ArtemisContext


@pytest.fixture
def mock_context():
    ctx = Mock(spec=ArtemisContext)
    ctx.data_engine = None
    return ctx


def _initial_observation():
    return HumanMessage(
        content=[
            {"type": "text", "text": "Task: Search flights"},
            {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_0"}},
            {"type": "text", "text": "--- UI Element List ---\nTree 0"},
        ]
    )


def _step(i: int) -> ToolMessage:
    return ToolMessage(
        tool_call_id=f"tc{i}",
        name="click",
        content=[
            {"type": "text", "text": f"Action {i} completed."},
            {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG_{i}"}},
            {"type": "text", "text": f"--- UI Element List ---\nTree {i}"},
        ],
    )


def _simulate_turns(compressor, messages, new_messages):
    """Mimic FlashRunner's ordering: compress at turn start, then append."""
    for msg in new_messages:
        compressor.compress(messages)
        messages.append(msg)
    compressor.compress(messages)


def _content_fingerprint(msg) -> str:
    return json.dumps(msg.content, sort_keys=True, default=str)


def _has_image(msg) -> bool:
    return any(isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content)


def test_success_path_product_behind_the_scrub_edge(mock_context):
    """With summaries ready in time, every message behind the scrub edge is the
    legacy-shaped product: action text kept, XML stripped, screenshot replaced
    by the ``--- Historical Visual Transition ---`` summary block.

    (Asserted directly; the legacy ``compress_flash_messages`` reference this
    was originally diffed against was removed in M5.) Intended M1 timing
    change: the swap happens at depth K, so the newest K-1 historical steps
    still carry their screenshots.
    """
    summarizer = VisualStepSummarizer(mock_context)
    for i in range(1, 7):
        summarizer._summaries[f"tc{i}"] = f"Visual summary {i}."

    new_messages: list = []
    compressor = ScrubEdgeCompressor(
        summarizer=summarizer,
        prune_history_xml=True,
        image_scrub_depth=3,
        pending_grace_steps=3,
    )
    _simulate_turns(
        compressor, new_messages, [_initial_observation()] + [_step(i) for i in range(1, 7)]
    )

    # Initial observation behind the edge: image silently dropped (no summary
    # job), task text kept, XML stripped.
    assert new_messages[0].content == [{"type": "text", "text": "Task: Search flights"}]

    # Steps 1-4 behind the scrub edge: exact legacy-shaped swap product.
    for i in range(1, 5):
        assert new_messages[i].content == [
            {"type": "text", "text": f"Action {i} completed."},
            {
                "type": "text",
                "text": f"--- Historical Visual Transition ---\nVisual summary {i}.",
            },
        ], f"message {i} diverged from the legacy-shaped product"

    # The live observation is intact.
    assert _has_image(new_messages[6])
    assert "Tree 6" in str(new_messages[6].content)

    # Intended timing change: step 5 (depth 2 < K) keeps its screenshot under
    # the scrub edge; its XML was still stripped at depth 1.
    assert _has_image(new_messages[5])
    assert "Visual summary 5." not in str(new_messages[5].content)
    assert "Tree 5" not in str(new_messages[5].content)


def test_race_pending_at_edge_recovers_within_grace(mock_context):
    """A step whose summary is late keeps its image at the scrub edge and is
    swapped to the summary once it arrives within the grace window."""
    summarizer = VisualStepSummarizer(mock_context)
    summarizer._step_inputs["tc1"] = {"step_number": 1}  # job pending

    messages: list = []
    compressor = ScrubEdgeCompressor(
        summarizer=summarizer,
        image_scrub_depth=3,
        pending_grace_steps=3,
    )
    _simulate_turns(
        compressor, messages, [_initial_observation()] + [_step(i) for i in range(1, 4)]
    )

    # Step 1 sits at the scrub edge with a pending summary: image retained.
    assert _has_image(messages[1])

    # Summary arrives within grace; the next pass applies it.
    summarizer._summaries["tc1"] = "Recovered visual summary 1."
    _simulate_turns(compressor, messages, [_step(4)])

    assert not _has_image(messages[1])
    assert {
        "type": "text",
        "text": "--- Historical Visual Transition ---\nRecovered visual summary 1.",
    } in messages[1].content


def test_race_grace_exhausted_placeholder_never_backfilled(mock_context):
    """After the grace window a pending step becomes a placeholder, and a late
    summary never mutates the frozen message again."""
    summarizer = VisualStepSummarizer(mock_context)
    summarizer._step_inputs["tc1"] = {"step_number": 1}  # pending forever (for now)

    messages: list = []
    compressor = ScrubEdgeCompressor(
        summarizer=summarizer,
        image_scrub_depth=2,
        pending_grace_steps=1,
    )
    _simulate_turns(compressor, messages, [_step(i) for i in range(1, 5)])

    # Depth 4 > K + grace = 3: grace exhausted, placeholder applied.
    step1_text = str(messages[0].content)
    assert not _has_image(messages[0])
    assert "[visual summary pending; evidence at DataEngine step 1]" in step1_text
    frozen_fingerprint = _content_fingerprint(messages[0])

    # The summary arriving later must NOT backfill the frozen message.
    summarizer._summaries["tc1"] = "Too late summary."
    _simulate_turns(compressor, messages, [_step(5)])
    compressor.compress(messages)

    assert _content_fingerprint(messages[0]) == frozen_fingerprint
    assert "Too late summary." not in str(messages[0].content)


def test_race_failed_summary_becomes_unavailable_placeholder(mock_context):
    """A failed summary is replaced by an unavailable placeholder at the scrub
    edge without waiting out the grace window."""
    summarizer = VisualStepSummarizer(mock_context)
    summarizer._step_inputs["tc1"] = {"step_number": 1}
    summarizer._failed.add("tc1")

    messages: list = []
    compressor = ScrubEdgeCompressor(
        summarizer=summarizer,
        image_scrub_depth=2,
        pending_grace_steps=5,
    )
    _simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)])

    assert not _has_image(messages[0])
    assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(messages[0].content)


def test_freeze_invariant_bytes_never_change(mock_context):
    """Once a message crosses the scrub edge its content bytes never change,
    across many turns and late-arriving summaries."""
    summarizer = VisualStepSummarizer(mock_context)
    # Odd steps have summaries ready from the start; even steps stay pending.
    for i in range(1, 11):
        summarizer._step_inputs[f"tc{i}"] = {"step_number": i}
        if i % 2 == 1:
            summarizer._summaries[f"tc{i}"] = f"Ready summary {i}."

    messages: list = []
    compressor = ScrubEdgeCompressor(
        summarizer=summarizer,
        image_scrub_depth=2,
        pending_grace_steps=1,
    )

    frozen_fingerprints: dict[int, str] = {}

    def check_frozen():
        for idx in compressor._frozen:
            fingerprint = _content_fingerprint(messages[idx])
            if idx in frozen_fingerprints:
                assert frozen_fingerprints[idx] == fingerprint, f"frozen message {idx} mutated"
            else:
                frozen_fingerprints[idx] = fingerprint

    for i in range(1, 11):
        compressor.compress(messages)
        check_frozen()
        messages.append(AIMessage(content=f"Thinking about step {i}."))
        messages.append(_step(i))
        if i == 6:
            # Late summaries for already-frozen even steps: must be ignored.
            summarizer._summaries["tc2"] = "Late summary 2."
            summarizer._summaries["tc4"] = "Late summary 4."
    compressor.compress(messages)
    check_frozen()

    assert len(frozen_fingerprints) >= 6
    # A frozen pending step carries the placeholder, not the late summary.
    assert "Late summary 2." not in " ".join(str(m.content) for m in messages)


def test_xml_depth1_strip_matches_legacy_semantics(mock_context):
    """Depth-1 XML stripping keeps combined-block prefixes and the live list."""
    messages: list = []
    compressor = ScrubEdgeCompressor(summarizer=None, prune_history_xml=True)

    combined = ToolMessage(
        tool_call_id="tc1",
        name="click",
        content=[
            {
                "type": "text",
                "text": "Tapped Settings.\n--- UI Element List ---\n[huge tree]",
            },
            {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}},
        ],
    )
    _simulate_turns(compressor, messages, [combined, _step(2)])

    # Historical combined block: prefix retained, tree stripped (the exact
    # legacy ``compress_flash_messages`` product for this input).
    assert messages[0].content[0] == {"type": "text", "text": "Tapped Settings."}
    # Live observation keeps its UI list untouched.
    assert "Tree 2" in str(messages[1].content)


def test_prune_history_xml_disabled_keeps_ui_lists(mock_context):
    """With the switch off, UI lists survive both depth-1 and freeze scrubs."""
    summarizer = VisualStepSummarizer(mock_context)
    summarizer._summaries["tc1"] = "Summary 1."

    messages: list = []
    compressor = ScrubEdgeCompressor(
        summarizer=summarizer,
        prune_history_xml=False,
        image_scrub_depth=2,
        pending_grace_steps=0,
    )
    _simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)])

    # Step 1 froze (summary applied) but its UI list is preserved.
    assert not _has_image(messages[0])
    assert "Tree 1" in str(messages[0].content)
    assert "Summary 1." in str(messages[0].content)


def test_untracked_messages_are_never_touched(mock_context):
    """Injected instructions and AI turns pass through untouched; a frozen
    image-only tool message falls back to non-empty content."""
    messages: list = [
        HumanMessage(content="[REAL-TIME INJECTED INSTRUCTION from user]: stay put"),
    ]
    compressor = ScrubEdgeCompressor(summarizer=None, image_scrub_depth=2)

    image_only = ToolMessage(
        tool_call_id="tc1",
        name="click",
        content=[{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}}],
    )
    _simulate_turns(
        compressor,
        messages,
        [image_only, AIMessage(content="thinking"), _step(2), _step(3)],
    )

    assert messages[0].content == "[REAL-TIME INJECTED INSTRUCTION from user]: stay put"
    assert messages[2].content == "thinking"
    # Image-only tool message froze with the defensive fallback text.
    assert messages[1].content == [{"type": "text", "text": "Action dispatched."}]