feat(memory): add incremental transcript compaction
This commit is contained in:
1 parent
cbe8f4f0c6
commit
41ae386a3a
65 files changed
+9971
-815
No files matched your search
@@ -0,0 +1,325 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for the Flash scrub-edge compressor (history redesign §3.2, M1).
|
||||
|
||||
Covers the M1 acceptance items: the legacy-equivalent success-path product
|
||||
(asserted directly since the legacy ``compress_flash_messages`` reference was
|
||||
removed in M5), the three race regressions (grace recovery, grace exhaustion
|
||||
without backfill, failed-summary placeholder), the freeze invariant (bytes
|
||||
behind the scrub edge never change), and depth-1 XML stripping equivalence.
|
||||
"""
|
||||
|
||||
import json
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
|
||||
from artemis.agents.flash.context_compressor import ScrubEdgeCompressor
|
||||
from artemis.agents.flash.summarizer import VisualStepSummarizer
|
||||
from artemis.context import ArtemisContext
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.data_engine = None
|
||||
return ctx
|
||||
|
||||
|
||||
def _initial_observation():
|
||||
return HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "Task: Search flights"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_0"}},
|
||||
{"type": "text", "text": "--- UI Element List ---\nTree 0"},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _step(i: int) -> ToolMessage:
|
||||
return ToolMessage(
|
||||
tool_call_id=f"tc{i}",
|
||||
name="click",
|
||||
content=[
|
||||
{"type": "text", "text": f"Action {i} completed."},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG_{i}"}},
|
||||
{"type": "text", "text": f"--- UI Element List ---\nTree {i}"},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _simulate_turns(compressor, messages, new_messages):
|
||||
"""Mimic FlashRunner's ordering: compress at turn start, then append."""
|
||||
for msg in new_messages:
|
||||
compressor.compress(messages)
|
||||
messages.append(msg)
|
||||
compressor.compress(messages)
|
||||
|
||||
|
||||
def _content_fingerprint(msg) -> str:
|
||||
return json.dumps(msg.content, sort_keys=True, default=str)
|
||||
|
||||
|
||||
def _has_image(msg) -> bool:
|
||||
return any(
|
||||
isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content
|
||||
)
|
||||
|
||||
|
||||
def test_success_path_product_behind_the_scrub_edge(mock_context):
|
||||
"""With summaries ready in time, every message behind the scrub edge is the
|
||||
legacy-shaped product: action text kept, XML stripped, screenshot replaced
|
||||
by the ``--- Historical Visual Transition ---`` summary block.
|
||||
|
||||
(Asserted directly; the legacy ``compress_flash_messages`` reference this
|
||||
was originally diffed against was removed in M5.) Intended M1 timing
|
||||
change: the swap happens at depth K, so the newest K-1 historical steps
|
||||
still carry their screenshots.
|
||||
"""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
for i in range(1, 7):
|
||||
summarizer._summaries[f"tc{i}"] = f"Visual summary {i}."
|
||||
|
||||
new_messages: list = []
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
prune_history_xml=True,
|
||||
image_scrub_depth=3,
|
||||
pending_grace_steps=3,
|
||||
)
|
||||
_simulate_turns(
|
||||
compressor, new_messages, [_initial_observation()] + [_step(i) for i in range(1, 7)]
|
||||
)
|
||||
|
||||
# Initial observation behind the edge: image silently dropped (no summary
|
||||
# job), task text kept, XML stripped.
|
||||
assert new_messages[0].content == [{"type": "text", "text": "Task: Search flights"}]
|
||||
|
||||
# Steps 1-4 behind the scrub edge: exact legacy-shaped swap product.
|
||||
for i in range(1, 5):
|
||||
assert new_messages[i].content == [
|
||||
{"type": "text", "text": f"Action {i} completed."},
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"--- Historical Visual Transition ---\nVisual summary {i}.",
|
||||
},
|
||||
], f"message {i} diverged from the legacy-shaped product"
|
||||
|
||||
# The live observation is intact.
|
||||
assert _has_image(new_messages[6])
|
||||
assert "Tree 6" in str(new_messages[6].content)
|
||||
|
||||
# Intended timing change: step 5 (depth 2 < K) keeps its screenshot under
|
||||
# the scrub edge; its XML was still stripped at depth 1.
|
||||
assert _has_image(new_messages[5])
|
||||
assert "Visual summary 5." not in str(new_messages[5].content)
|
||||
assert "Tree 5" not in str(new_messages[5].content)
|
||||
|
||||
|
||||
def test_race_pending_at_edge_recovers_within_grace(mock_context):
|
||||
"""A step whose summary is late keeps its image at the scrub edge and is
|
||||
swapped to the summary once it arrives within the grace window."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._step_inputs["tc1"] = {"step_number": 1} # job pending
|
||||
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
image_scrub_depth=3,
|
||||
pending_grace_steps=3,
|
||||
)
|
||||
_simulate_turns(
|
||||
compressor, messages, [_initial_observation()] + [_step(i) for i in range(1, 4)]
|
||||
)
|
||||
|
||||
# Step 1 sits at the scrub edge with a pending summary: image retained.
|
||||
assert _has_image(messages[1])
|
||||
|
||||
# Summary arrives within grace; the next pass applies it.
|
||||
summarizer._summaries["tc1"] = "Recovered visual summary 1."
|
||||
_simulate_turns(compressor, messages, [_step(4)])
|
||||
|
||||
assert not _has_image(messages[1])
|
||||
assert {
|
||||
"type": "text",
|
||||
"text": "--- Historical Visual Transition ---\nRecovered visual summary 1.",
|
||||
} in messages[1].content
|
||||
|
||||
|
||||
def test_race_grace_exhausted_placeholder_never_backfilled(mock_context):
|
||||
"""After the grace window a pending step becomes a placeholder, and a late
|
||||
summary never mutates the frozen message again."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._step_inputs["tc1"] = {"step_number": 1} # pending forever (for now)
|
||||
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
image_scrub_depth=2,
|
||||
pending_grace_steps=1,
|
||||
)
|
||||
_simulate_turns(compressor, messages, [_step(i) for i in range(1, 5)])
|
||||
|
||||
# Depth 4 > K + grace = 3: grace exhausted, placeholder applied.
|
||||
step1_text = str(messages[0].content)
|
||||
assert not _has_image(messages[0])
|
||||
assert "[visual summary pending; evidence at DataEngine step 1]" in step1_text
|
||||
frozen_fingerprint = _content_fingerprint(messages[0])
|
||||
|
||||
# The summary arriving later must NOT backfill the frozen message.
|
||||
summarizer._summaries["tc1"] = "Too late summary."
|
||||
_simulate_turns(compressor, messages, [_step(5)])
|
||||
compressor.compress(messages)
|
||||
|
||||
assert _content_fingerprint(messages[0]) == frozen_fingerprint
|
||||
assert "Too late summary." not in str(messages[0].content)
|
||||
|
||||
|
||||
def test_race_failed_summary_becomes_unavailable_placeholder(mock_context):
|
||||
"""A failed summary is replaced by an unavailable placeholder at the scrub
|
||||
edge without waiting out the grace window."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._step_inputs["tc1"] = {"step_number": 1}
|
||||
summarizer._failed.add("tc1")
|
||||
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
image_scrub_depth=2,
|
||||
pending_grace_steps=5,
|
||||
)
|
||||
_simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)])
|
||||
|
||||
assert not _has_image(messages[0])
|
||||
assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(
|
||||
messages[0].content
|
||||
)
|
||||
|
||||
|
||||
def test_freeze_invariant_bytes_never_change(mock_context):
|
||||
"""Once a message crosses the scrub edge its content bytes never change,
|
||||
across many turns and late-arriving summaries."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
# Odd steps have summaries ready from the start; even steps stay pending.
|
||||
for i in range(1, 11):
|
||||
summarizer._step_inputs[f"tc{i}"] = {"step_number": i}
|
||||
if i % 2 == 1:
|
||||
summarizer._summaries[f"tc{i}"] = f"Ready summary {i}."
|
||||
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
image_scrub_depth=2,
|
||||
pending_grace_steps=1,
|
||||
)
|
||||
|
||||
frozen_fingerprints: dict[int, str] = {}
|
||||
|
||||
def check_frozen():
|
||||
for idx in compressor._frozen:
|
||||
fingerprint = _content_fingerprint(messages[idx])
|
||||
if idx in frozen_fingerprints:
|
||||
assert frozen_fingerprints[idx] == fingerprint, (
|
||||
f"frozen message {idx} mutated"
|
||||
)
|
||||
else:
|
||||
frozen_fingerprints[idx] = fingerprint
|
||||
|
||||
for i in range(1, 11):
|
||||
compressor.compress(messages)
|
||||
check_frozen()
|
||||
messages.append(AIMessage(content=f"Thinking about step {i}."))
|
||||
messages.append(_step(i))
|
||||
if i == 6:
|
||||
# Late summaries for already-frozen even steps: must be ignored.
|
||||
summarizer._summaries["tc2"] = "Late summary 2."
|
||||
summarizer._summaries["tc4"] = "Late summary 4."
|
||||
compressor.compress(messages)
|
||||
check_frozen()
|
||||
|
||||
assert len(frozen_fingerprints) >= 6
|
||||
# A frozen pending step carries the placeholder, not the late summary.
|
||||
assert "Late summary 2." not in " ".join(str(m.content) for m in messages)
|
||||
|
||||
|
||||
def test_xml_depth1_strip_matches_legacy_semantics(mock_context):
|
||||
"""Depth-1 XML stripping keeps combined-block prefixes and the live list."""
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(summarizer=None, prune_history_xml=True)
|
||||
|
||||
combined = ToolMessage(
|
||||
tool_call_id="tc1",
|
||||
name="click",
|
||||
content=[
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Tapped Settings.\n--- UI Element List ---\n[huge tree]",
|
||||
},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}},
|
||||
],
|
||||
)
|
||||
_simulate_turns(compressor, messages, [combined, _step(2)])
|
||||
|
||||
# Historical combined block: prefix retained, tree stripped (the exact
|
||||
# legacy ``compress_flash_messages`` product for this input).
|
||||
assert messages[0].content[0] == {"type": "text", "text": "Tapped Settings."}
|
||||
# Live observation keeps its UI list untouched.
|
||||
assert "Tree 2" in str(messages[1].content)
|
||||
|
||||
|
||||
def test_prune_history_xml_disabled_keeps_ui_lists(mock_context):
|
||||
"""With the switch off, UI lists survive both depth-1 and freeze scrubs."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._summaries["tc1"] = "Summary 1."
|
||||
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
prune_history_xml=False,
|
||||
image_scrub_depth=2,
|
||||
pending_grace_steps=0,
|
||||
)
|
||||
_simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)])
|
||||
|
||||
# Step 1 froze (summary applied) but its UI list is preserved.
|
||||
assert not _has_image(messages[0])
|
||||
assert "Tree 1" in str(messages[0].content)
|
||||
assert "Summary 1." in str(messages[0].content)
|
||||
|
||||
|
||||
def test_untracked_messages_are_never_touched(mock_context):
|
||||
"""Injected instructions and AI turns pass through untouched; a frozen
|
||||
image-only tool message falls back to non-empty content."""
|
||||
messages: list = [
|
||||
HumanMessage(content="[REAL-TIME INJECTED INSTRUCTION from user]: stay put"),
|
||||
]
|
||||
compressor = ScrubEdgeCompressor(summarizer=None, image_scrub_depth=2)
|
||||
|
||||
image_only = ToolMessage(
|
||||
tool_call_id="tc1",
|
||||
name="click",
|
||||
content=[{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}}],
|
||||
)
|
||||
_simulate_turns(
|
||||
compressor,
|
||||
messages,
|
||||
[image_only, AIMessage(content="thinking"), _step(2), _step(3)],
|
||||
)
|
||||
|
||||
assert messages[0].content == "[REAL-TIME INJECTED INSTRUCTION from user]: stay put"
|
||||
assert messages[2].content == "thinking"
|
||||
# Image-only tool message froze with the defensive fallback text.
|
||||
assert messages[1].content == [{"type": "text", "text": "Action completed."}]
|
||||
@@ -14,12 +14,12 @@
|
||||
|
||||
"""Unit tests for VisualStepSummarizer and Context Compressor in Flash profile."""
|
||||
|
||||
from unittest.mock import AsyncMock, Mock, call
|
||||
from unittest.mock import ANY, AsyncMock, Mock, call
|
||||
from uuid import uuid4
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
|
||||
from artemis.agents.flash.context_compressor import compress_flash_messages
|
||||
from artemis.agents.flash.context_compressor import ScrubEdgeCompressor
|
||||
from artemis.agents.flash.summarizer import VisualStepSummarizer
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.sdk.builders import Builders
|
||||
@@ -100,8 +100,94 @@ async def test_summarizer_retries_independently(mock_context):
|
||||
assert attempt_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_bounded_retry_marks_failed(mock_context):
|
||||
"""Persistent failures exit after 1 + retry_limit attempts with an explicit failed state."""
|
||||
mock_context.data_engine = Mock()
|
||||
summarizer = VisualStepSummarizer(mock_context, retry_limit=2)
|
||||
summarizer._retry_delays = (0.0,)
|
||||
attempt_count = 0
|
||||
|
||||
async def mock_ainvoke(messages):
|
||||
nonlocal attempt_count
|
||||
attempt_count += 1
|
||||
raise TimeoutError("Simulated permanent failure")
|
||||
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
|
||||
step_id = uuid4()
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="click",
|
||||
action_args={"target": [100, 100]},
|
||||
pre_img_bytes=b"pre1",
|
||||
post_img_bytes=b"post1",
|
||||
exec_outcome="Outcome 1",
|
||||
action_key="tc-fail",
|
||||
data_engine_step_id=step_id,
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
# Bounded exit: initial attempt + retry_limit retries, then explicit failure.
|
||||
assert attempt_count == 3
|
||||
assert summarizer.has_failed("tc-fail")
|
||||
assert not summarizer.has_summary("tc-fail")
|
||||
# Lossless semantics: a failed step still reads as pending so the
|
||||
# compressor keeps its source image instead of dropping evidence.
|
||||
assert summarizer.is_pending("tc-fail")
|
||||
|
||||
# DataEngine observability: pending on dispatch, failed on exhaustion.
|
||||
mock_context.data_engine.update_step_summary.assert_has_calls(
|
||||
[
|
||||
call(step_id, None, status="pending", source="visual_transition", model=ANY),
|
||||
call(step_id, None, status="failed", source="visual_transition", model=ANY),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_success_within_retry_budget_not_failed(mock_context):
|
||||
"""A summary that succeeds on a retry within the budget never enters failed state."""
|
||||
summarizer = VisualStepSummarizer(mock_context, retry_limit=3)
|
||||
summarizer._retry_delays = (0.0,)
|
||||
attempt_count = 0
|
||||
|
||||
async def mock_ainvoke(messages):
|
||||
nonlocal attempt_count
|
||||
attempt_count += 1
|
||||
if attempt_count < 3:
|
||||
raise TimeoutError("Transient failure")
|
||||
return AIMessage(content="[Step 1 State: Finally worked.]")
|
||||
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="click",
|
||||
action_args={"target": [100, 100]},
|
||||
pre_img_bytes=b"pre1",
|
||||
post_img_bytes=b"post1",
|
||||
exec_outcome="Outcome 1",
|
||||
action_key="tc-ok",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
assert attempt_count == 3
|
||||
assert summarizer.has_summary("tc-ok")
|
||||
assert not summarizer.has_failed("tc-ok")
|
||||
|
||||
|
||||
def test_context_compressor_with_ready_summaries(mock_context):
|
||||
"""Verify that ready summaries replace past image blocks and prune outdated XML."""
|
||||
"""Ready summaries replace past image blocks and outdated XML is pruned.
|
||||
|
||||
Rewritten against ``ScrubEdgeCompressor`` (M5; the legacy
|
||||
``compress_flash_messages`` was removed) with ``image_scrub_depth=1`` so
|
||||
every historical message is scrubbed immediately, mirroring the legacy
|
||||
timing this test was written for. Summaries are keyed by legacy ordinals
|
||||
to keep the older-caller fallback path covered.
|
||||
"""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._summaries[1] = "[Step 1 State: Tapped Search bar. Keyboard appeared.]"
|
||||
summarizer._summaries[2] = "[Step 2 State: Typed 'flights'. Suggestions displayed.]"
|
||||
@@ -134,7 +220,13 @@ def test_context_compressor_with_ready_summaries(mock_context):
|
||||
),
|
||||
]
|
||||
|
||||
compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
prune_history_xml=True,
|
||||
image_scrub_depth=1,
|
||||
pending_grace_steps=0,
|
||||
)
|
||||
compressor.compress(messages)
|
||||
|
||||
# Initial HumanMessage: Historical initial image pruned, task text preserved
|
||||
assert len(messages[0].content) == 1
|
||||
@@ -155,88 +247,15 @@ def test_context_compressor_with_ready_summaries(mock_context):
|
||||
assert "Live Latest Tree 3" in messages[2].content[2]["text"]
|
||||
|
||||
|
||||
def test_context_compressor_graceful_fallback(mock_context):
|
||||
"""Verify that when a summary is not ready yet, it cleanly falls back to standard pruned state."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
# Step 1 has no summary in cache
|
||||
|
||||
messages = [
|
||||
HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "Task: Search"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
|
||||
]
|
||||
),
|
||||
ToolMessage(
|
||||
tool_call_id="tc1",
|
||||
name="click",
|
||||
content=[
|
||||
{"type": "text", "text": "Action 'click' completed."},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
|
||||
|
||||
# Turn 1 should simply have image omitted (standard fallback)
|
||||
assert len(messages[0].content) == 1
|
||||
assert messages[0].content[0]["text"] == "Task: Search"
|
||||
|
||||
# Turn 2 (Latest) preserves image
|
||||
assert len(messages[1].content) == 2
|
||||
assert messages[1].content[1]["type"] == "image_url"
|
||||
|
||||
|
||||
def test_context_compressor_backfills_late_summary_idempotently(mock_context):
|
||||
"""A pending image remains until its late summary replaces it."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._step_inputs["action-1"] = {"step_number": 1}
|
||||
messages = [
|
||||
HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "Task: Search"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
|
||||
]
|
||||
),
|
||||
ToolMessage(
|
||||
tool_call_id="action-1",
|
||||
name="click",
|
||||
content=[
|
||||
{"type": "text", "text": "Tapped Search."},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
|
||||
],
|
||||
),
|
||||
ToolMessage(
|
||||
tool_call_id="action-2",
|
||||
name="input_text",
|
||||
content=[
|
||||
{"type": "text", "text": "Typed query."},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_3"}},
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
compress_flash_messages(messages, summarizer=summarizer)
|
||||
assert any(block["type"] == "image_url" for block in messages[1].content)
|
||||
assert len(messages[1].content) == 2
|
||||
|
||||
summarizer._summaries["action-1"] = "Search field focused; keyboard appeared."
|
||||
compress_flash_messages(messages, summarizer=summarizer)
|
||||
compress_flash_messages(messages, summarizer=summarizer)
|
||||
|
||||
generated_blocks = [
|
||||
block
|
||||
for block in messages[1].content
|
||||
if block.get("text", "").startswith("--- Historical Visual Transition ---")
|
||||
]
|
||||
assert len(generated_blocks) == 1
|
||||
assert generated_blocks[0]["text"].endswith("Search field focused; keyboard appeared.")
|
||||
assert all(block["type"] != "image_url" for block in messages[1].content)
|
||||
|
||||
|
||||
def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context):
|
||||
"""Generic tool messages cannot shift action summaries onto the wrong result."""
|
||||
"""Generic tool messages cannot shift action summaries onto the wrong result.
|
||||
|
||||
(Rewritten against ``ScrubEdgeCompressor`` in M5. The legacy tests for
|
||||
pending-summary fallback and late-summary backfill were removed with
|
||||
``compress_flash_messages``: the scrub edge covers pending handling in
|
||||
``test_flash_scrub_edge.py``, and late backfill is now forbidden by the
|
||||
freeze invariant rather than supported.)
|
||||
"""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._summaries["click-id"] = "Click-specific visual memory."
|
||||
summarizer._summaries["type-id"] = "Typing-specific visual memory."
|
||||
@@ -265,7 +284,12 @@ def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context)
|
||||
),
|
||||
]
|
||||
|
||||
compress_flash_messages(messages, summarizer=summarizer)
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
image_scrub_depth=1,
|
||||
pending_grace_steps=0,
|
||||
)
|
||||
compressor.compress(messages)
|
||||
|
||||
assert "Click-specific visual memory." in str(messages[0].content)
|
||||
assert "Typing-specific visual memory." not in str(messages[0].content)
|
||||
@@ -274,35 +298,6 @@ def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context)
|
||||
assert messages[2].content[-1]["type"] == "image_url"
|
||||
|
||||
|
||||
def test_context_compressor_preserves_text_before_embedded_ui_list(mock_context):
|
||||
"""Pruning a combined XML block must retain its action result prefix."""
|
||||
messages = [
|
||||
ToolMessage(
|
||||
tool_call_id="old",
|
||||
name="click",
|
||||
content=[
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Tapped Settings.\n--- UI Element List ---\n[huge tree]",
|
||||
},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}},
|
||||
],
|
||||
),
|
||||
ToolMessage(
|
||||
tool_call_id="live",
|
||||
name="click",
|
||||
content=[
|
||||
{"type": "text", "text": "Opened page."},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,B"}},
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
compress_flash_messages(messages, prune_history_xml=True)
|
||||
|
||||
assert messages[0].content == [{"type": "text", "text": "Tapped Settings."}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_keys_actions_independently_of_turn_number(mock_context):
|
||||
"""Multiple actions from one LLM turn keep independent inputs and summaries."""
|
||||
@@ -344,17 +339,229 @@ async def test_summarizer_keys_actions_independently_of_turn_number(mock_context
|
||||
|
||||
assert summarizer.get_summary("tc-1") == "First action memory."
|
||||
assert summarizer.get_summary("tc-2") == "Second action memory."
|
||||
assert summarizer._step_inputs["tc-1"]["data_engine_step_id"] == first_step_id
|
||||
assert summarizer._step_inputs["tc-2"]["data_engine_step_id"] == second_step_id
|
||||
# M1 keying: jobs are canonical under the DataEngine step id; the
|
||||
# tool_call_id stays queryable as an alias.
|
||||
assert summarizer.get_job_payload("tc-1")["data_engine_step_id"] == first_step_id
|
||||
assert summarizer.get_job_payload("tc-2")["data_engine_step_id"] == second_step_id
|
||||
assert summarizer.resolve_key("tc-1") == str(first_step_id)
|
||||
mock_context.data_engine.update_step_summary.assert_has_calls(
|
||||
[
|
||||
call(first_step_id, "First action memory."),
|
||||
call(second_step_id, "Second action memory."),
|
||||
call(
|
||||
first_step_id,
|
||||
"First action memory.",
|
||||
status="ready",
|
||||
source="visual_transition",
|
||||
model=ANY,
|
||||
),
|
||||
call(
|
||||
second_step_id,
|
||||
"Second action memory.",
|
||||
status="ready",
|
||||
source="visual_transition",
|
||||
model=ANY,
|
||||
),
|
||||
],
|
||||
any_order=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_frame_dispatch_uses_single_image_prompt_variant(mock_context):
|
||||
"""A missing after-frame (every Pro step, by design) selects the
|
||||
single-image prompt: decision frame + red marker, no AFTER ACTION section."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
captured: list = []
|
||||
|
||||
async def mock_ainvoke(messages):
|
||||
captured.append(messages)
|
||||
return AIMessage(
|
||||
content=(
|
||||
"In Step 1, I tapped the 'Battery' row marked in red in the"
|
||||
" Settings list, which showed 'Network & internet' and 'Display'."
|
||||
)
|
||||
)
|
||||
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="click",
|
||||
action_args={"target": [100, 200]},
|
||||
pre_img_bytes=b"decision_frame",
|
||||
post_img_bytes=None,
|
||||
exec_outcome="executed",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
assert summarizer.has_summary(1)
|
||||
system_text = str(captured[0][0].content)
|
||||
human_text = str(captured[0][1].content)
|
||||
assert "Exactly ONE screenshot" in system_text
|
||||
assert "no independent post-action evidence" in system_text
|
||||
assert "DECISION FRAME" in human_text
|
||||
assert "AFTER ACTION SCREEN" not in human_text
|
||||
assert "--- [2]" not in human_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dual_frame_dispatch_keeps_transition_prompt(mock_context):
|
||||
"""Both frames present (Flash path): the classic BEFORE/AFTER transition
|
||||
prompt and section labels are unchanged."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
captured: list = []
|
||||
|
||||
async def mock_ainvoke(messages):
|
||||
captured.append(messages)
|
||||
return AIMessage(
|
||||
content=(
|
||||
"In Step 1, I tapped the 'Settings' gear icon marked in red;"
|
||||
" the main Settings menu opened showing 'Apps'."
|
||||
)
|
||||
)
|
||||
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="click",
|
||||
action_args={"target": [100, 200]},
|
||||
pre_img_bytes=b"pre",
|
||||
post_img_bytes=b"post",
|
||||
exec_outcome="executed",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
assert summarizer.has_summary(1)
|
||||
system_text = str(captured[0][0].content)
|
||||
human_text = str(captured[0][1].content)
|
||||
assert "Before/After screenshots" in system_text
|
||||
assert "--- [1] BEFORE ACTION SCREEN (Action Marked Visually in Red) ---" in human_text
|
||||
assert "--- [2] AFTER ACTION SCREEN ---" in human_text
|
||||
assert "DECISION FRAME" not in human_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_echo_output_fails_attempt_then_recovers_on_retry(mock_context):
|
||||
"""An input-marker echo is a failed attempt (not a summary): the bounded
|
||||
retry regenerates and the recovered text lands normally."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._retry_delays = (0.0,)
|
||||
attempts = 0
|
||||
|
||||
async def mock_ainvoke(messages):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
if attempts == 1:
|
||||
return AIMessage(content="--- [2] AFTER ACTION SCREEN ---")
|
||||
return AIMessage(
|
||||
content=(
|
||||
"In Step 1, I tapped the search field marked in red; the field"
|
||||
" showed placeholder text 'Search here'."
|
||||
)
|
||||
)
|
||||
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="click",
|
||||
action_args={"target": [1, 2]},
|
||||
pre_img_bytes=b"pre",
|
||||
post_img_bytes=b"post",
|
||||
exec_outcome="executed",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
assert attempts == 2
|
||||
assert summarizer.has_summary(1)
|
||||
assert "Search here" in summarizer.get_summary(1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_degenerate_outputs_exhaust_to_failed(mock_context):
|
||||
"""Echo/too-short/too-long outputs never land as summaries; persistent
|
||||
degeneration exhausts the bounded retries into the explicit failed state."""
|
||||
summarizer = VisualStepSummarizer(mock_context, retry_limit=1)
|
||||
summarizer._retry_delays = (0.0,)
|
||||
|
||||
async def mock_ainvoke(messages):
|
||||
text = str(messages)
|
||||
if "Step 1" in text:
|
||||
return AIMessage(content="--- [1] DECISION FRAME: SCREEN AT ACTION TIME ---")
|
||||
if "Step 2" in text:
|
||||
return AIMessage(content="x" * 2000)
|
||||
return AIMessage(content="ok.")
|
||||
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
|
||||
for step in (1, 2, 3):
|
||||
summarizer.dispatch(
|
||||
step_number=step,
|
||||
action_name="click",
|
||||
action_args={"target": [1, 2]},
|
||||
pre_img_bytes=b"pre",
|
||||
post_img_bytes=None,
|
||||
exec_outcome="executed",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
for step in (1, 2, 3):
|
||||
assert summarizer.has_failed(step), f"step {step} should have failed"
|
||||
assert not summarizer.has_summary(step)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_lens_call_meters_llm_usage_without_touching_context_base(mock_context):
|
||||
"""The raw-model lens bypass records an llm_usage trace, but its tiny
|
||||
prompt never overwrites the session's last_prompt_tokens (the compaction
|
||||
thresholds' live context base)."""
|
||||
from artemis.services.token_meter import get_meter
|
||||
|
||||
engine = Mock()
|
||||
engine.current_session_id = "lens-meter-session"
|
||||
engine.current_step_id = None
|
||||
mock_context.data_engine = engine
|
||||
meter = get_meter("lens-meter-session")
|
||||
meter.last_prompt_tokens = 777
|
||||
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(
|
||||
return_value=AIMessage(
|
||||
content=(
|
||||
"In Step 1, I tapped the 'Battery' row marked in red; the list"
|
||||
" showed 'Display' and 'Battery'."
|
||||
),
|
||||
usage_metadata={"input_tokens": 120, "output_tokens": 30, "total_tokens": 150},
|
||||
)
|
||||
)
|
||||
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="click",
|
||||
action_args={"target": [1, 2]},
|
||||
pre_img_bytes=b"pre",
|
||||
post_img_bytes=b"post",
|
||||
exec_outcome="executed",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
usage_calls = [
|
||||
c
|
||||
for c in engine.record_trace.call_args_list
|
||||
if c.kwargs.get("name") == "llm_usage"
|
||||
]
|
||||
assert usage_calls, "the raw lens bypass must record an llm_usage trace"
|
||||
payload = usage_calls[0].kwargs["payload"]
|
||||
assert payload["source"].startswith("lens:visual_transition:")
|
||||
assert payload["prompt_tokens"] == 120
|
||||
assert meter.last_prompt_tokens == 777
|
||||
|
||||
|
||||
def test_flash_config_and_builder():
|
||||
"""Verify Flash profile configuration model and SDK builder fluent API."""
|
||||
cfg = Builders.AgentConfig.with_flash_config(
|
||||
@@ -395,7 +602,13 @@ async def test_data_engine_integration_with_step_summarizer(mock_context):
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
mock_de.update_step_summary.assert_called_once_with(1, "[Step 1 State: Input text confirmed.]")
|
||||
mock_de.update_step_summary.assert_called_once_with(
|
||||
1,
|
||||
"[Step 1 State: Input text confirmed.]",
|
||||
status="ready",
|
||||
source="visual_transition",
|
||||
model=ANY,
|
||||
)
|
||||
|
||||
|
||||
def test_draw_action_overlay_rendering():
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""HistoricalStateHintPromptComponent (M4): perceptual-hash hint + silence rules.
|
||||
|
||||
Division of labor under test: the hint fires only for matches *older* than the
|
||||
recent window (returned-to-an-early-state regime); any match within the last 3
|
||||
steps silences it entirely — that regime belongs to the pixel-level
|
||||
ScreenshotSimilarityPromptComponent.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import io
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
from artemis.agents.operator.prompts import HistoricalStateHintPromptComponent, PromptBuilder
|
||||
from artemis.utils.image_hash import dhash_hex
|
||||
|
||||
|
||||
def _jpeg(draw_fn) -> bytes:
|
||||
img = Image.new("RGB", (270, 600), "white")
|
||||
draw_fn(ImageDraw.Draw(img))
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=85)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _screen_a(d):
|
||||
d.rectangle([20, 40, 250, 90], fill="#3355ff")
|
||||
d.rectangle([60, 450, 210, 510], fill="#22aa66")
|
||||
|
||||
|
||||
def _screen_b(d):
|
||||
for y in range(60, 560, 80):
|
||||
d.rectangle([10, y, 260, y + 50], fill="#dddddd")
|
||||
|
||||
|
||||
SCREEN_A = _jpeg(_screen_a)
|
||||
SCREEN_B = _jpeg(_screen_b)
|
||||
HASH_A = dhash_hex(SCREEN_A)
|
||||
HASH_B = dhash_hex(SCREEN_B)
|
||||
B64_A = base64.b64encode(SCREEN_A).decode("utf-8")
|
||||
|
||||
|
||||
def _step(number: int, post_hash: str | None) -> dict:
|
||||
meta = {"post_image_dhash": post_hash} if post_hash else {}
|
||||
return {"step_number": number, "extra_metadata": meta}
|
||||
|
||||
|
||||
def _cfg(similarity_hint=True, max_distance=8):
|
||||
return SimpleNamespace(
|
||||
memory=SimpleNamespace(
|
||||
transcript=SimpleNamespace(
|
||||
similarity_hint=similarity_hint,
|
||||
similarity_max_distance=max_distance,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _run(steps, b64=B64_A, cfg=None):
|
||||
builder = PromptBuilder()
|
||||
component = HistoricalStateHintPromptComponent()
|
||||
with patch("artemis.config.load_agent_config", return_value=cfg or _cfg()):
|
||||
await component(
|
||||
builder, MagicMock(), MagicMock(), latest_screenshot_b64=b64, steps=steps
|
||||
)
|
||||
return [p for p in builder.human_parts if isinstance(p, str)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hint_fires_on_old_matching_step():
|
||||
steps = [
|
||||
_step(1, HASH_B),
|
||||
_step(2, HASH_A), # old match
|
||||
_step(3, HASH_B),
|
||||
_step(4, HASH_B),
|
||||
_step(5, HASH_B),
|
||||
_step(6, HASH_B),
|
||||
]
|
||||
parts = await _run(steps)
|
||||
assert len(parts) == 1
|
||||
assert "Historical state hint" in parts[0]
|
||||
assert "Step 2" in parts[0]
|
||||
assert "recall_history" in parts[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_silent_when_recent_step_also_matches():
|
||||
steps = [
|
||||
_step(1, HASH_A), # old match exists...
|
||||
_step(2, HASH_B),
|
||||
_step(3, HASH_B),
|
||||
_step(4, HASH_B),
|
||||
_step(5, HASH_A), # ...but a recent (last-3) match silences the hint
|
||||
_step(6, HASH_B),
|
||||
]
|
||||
assert await _run(steps) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_silent_without_any_match():
|
||||
steps = [_step(n, HASH_B) for n in range(1, 7)]
|
||||
assert await _run(steps) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_silent_when_config_disabled():
|
||||
steps = [
|
||||
_step(1, HASH_A),
|
||||
_step(2, HASH_B),
|
||||
_step(3, HASH_B),
|
||||
_step(4, HASH_B),
|
||||
_step(5, HASH_B),
|
||||
_step(6, HASH_B),
|
||||
]
|
||||
assert await _run(steps, cfg=_cfg(similarity_hint=False)) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_silent_with_too_few_steps():
|
||||
# Everything within the recent window belongs to the pixel-level note.
|
||||
assert await _run([_step(1, HASH_A), _step(2, HASH_A)]) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tie_prefers_most_recent_old_step():
|
||||
steps = [
|
||||
_step(1, HASH_A),
|
||||
_step(2, HASH_A), # same distance, more recent
|
||||
_step(3, HASH_B),
|
||||
_step(4, HASH_B),
|
||||
_step(5, HASH_B),
|
||||
_step(6, HASH_B),
|
||||
]
|
||||
parts = await _run(steps)
|
||||
assert len(parts) == 1 and "Step 2" in parts[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_steps_without_hashes_are_skipped():
|
||||
steps = [
|
||||
_step(1, None),
|
||||
_step(2, None),
|
||||
_step(3, None),
|
||||
_step(4, None),
|
||||
_step(5, None),
|
||||
_step(6, None),
|
||||
]
|
||||
assert await _run(steps) == []
|
||||
|
||||
|
||||
def test_record_step_stamps_perceptual_hashes(tmp_path):
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
setup = MagicMock()
|
||||
setup.traces_path = str(tmp_path)
|
||||
mock_ctx.execution_setup = setup
|
||||
mock_ctx.device = None
|
||||
engine = DataEngine(mock_ctx)
|
||||
engine.start_session("dhash stamping test")
|
||||
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=SCREEN_A,
|
||||
post_screenshot_bytes=SCREEN_B,
|
||||
summary="two distinct screens",
|
||||
)
|
||||
# Same pre/post bytes: post_image_name is nulled but the post-action
|
||||
# screen IS the pre screen, so the post hash mirrors the pre hash.
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=SCREEN_A,
|
||||
post_screenshot_bytes=SCREEN_A,
|
||||
summary="screen unchanged",
|
||||
)
|
||||
for t in list(engine._pending_threads):
|
||||
t.join()
|
||||
|
||||
steps = engine.storage.get_steps(engine.current_session_id)
|
||||
meta1 = steps[0].extra_metadata
|
||||
assert meta1["pre_image_dhash"] == HASH_A
|
||||
assert meta1["post_image_dhash"] == HASH_B
|
||||
meta2 = steps[1].extra_metadata
|
||||
assert meta2["pre_image_dhash"] == HASH_A
|
||||
assert meta2["post_image_dhash"] == HASH_A
|
||||
@@ -344,7 +344,7 @@ async def test_history_analyzer_integration_with_task_tree():
|
||||
mock_path.exists.return_value = True
|
||||
mock_path.read_text.return_value = fake_plan
|
||||
|
||||
# Mock build_plan_and_history
|
||||
# Mock the policy-table compiled-history renderer (M4)
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.history_analyzer.history_analyzer.get_llm",
|
||||
@@ -355,23 +355,23 @@ async def test_history_analyzer_integration_with_task_tree():
|
||||
return_value=mock_path,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.history_analyzer.history_analyzer.build_plan_and_history"
|
||||
"artemis.agents.history_analyzer.history_analyzer.build_history_for"
|
||||
) as mock_build,
|
||||
):
|
||||
mock_build.return_value = "Mocked operation history"
|
||||
analyzer = HistoryAnalyzer(mock_ctx)
|
||||
await analyzer.run("query")
|
||||
|
||||
# Verify build_plan_and_history call args
|
||||
# Verify build_history_for call args
|
||||
mock_build.assert_called_once()
|
||||
args, kwargs = mock_build.call_args
|
||||
assert args[0] == fake_plan
|
||||
assert args[1] == steps
|
||||
assert args[0] == "history_analyzer"
|
||||
assert args[1] == fake_plan
|
||||
assert args[2] == steps
|
||||
|
||||
import hashlib
|
||||
|
||||
expected_hash = hashlib.md5(b"Active Subgoal").hexdigest()
|
||||
assert args[2] == expected_hash
|
||||
assert args[3] == expected_hash
|
||||
|
||||
assert kwargs.get("last_n_detailed") == 1
|
||||
assert kwargs.get("min_summaries") == len(steps)
|
||||
assert kwargs.get("engine") is mock_ctx.data_engine
|
||||
@@ -16,9 +16,15 @@ import json
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.config.agent import MemoryTranscriptConfig
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
# These tests exercise the legacy 2-message prompt path. Since M5 the
|
||||
# transcript path is the default, so flag-off semantics must be requested
|
||||
# explicitly (never via the config default).
|
||||
LEGACY_TRANSCRIPT = MemoryTranscriptConfig(enabled=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_node_fast_path():
|
||||
@@ -58,7 +64,7 @@ async def test_operator_node_fast_path():
|
||||
|
||||
# Patch dependencies
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# Verify state was updated with action
|
||||
@@ -77,7 +83,7 @@ async def test_perform_action_validation():
|
||||
from artemis.graph.state import State
|
||||
|
||||
mock_ctx = MagicMock()
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.indexed_points = [[200, 300], [400, 500]]
|
||||
@@ -114,6 +120,7 @@ async def test_perform_action_validation():
|
||||
"target_bounds": "[100,100][300,500]",
|
||||
"target_resource_id": "btn_1",
|
||||
"target_class": "android.widget.Button",
|
||||
"target_label_source": "index",
|
||||
}
|
||||
]
|
||||
|
||||
@@ -143,6 +150,7 @@ async def test_perform_action_validation():
|
||||
"target_bounds": "[300,400][500,600]",
|
||||
"target_resource_id": "input_1",
|
||||
"target_class": "android.widget.EditText",
|
||||
"target_label_source": "index",
|
||||
}
|
||||
|
||||
# Test swipe direction
|
||||
@@ -170,6 +178,7 @@ async def test_perform_action_validation():
|
||||
"target_bounds": "[300,400][500,600]",
|
||||
"target_resource_id": "input_1",
|
||||
"target_class": "android.widget.EditText",
|
||||
"target_label_source": "index",
|
||||
}
|
||||
]
|
||||
|
||||
@@ -222,6 +231,59 @@ async def test_perform_action_validation():
|
||||
assert "Unsupported app management action" in err
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bare_coordinate_click_enriched_by_hit_test():
|
||||
"""A coordinate click hit tests the pre-action frame for element semantics."""
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from unittest.mock import MagicMock
|
||||
from artemis.graph.state import State
|
||||
|
||||
mock_ctx = MagicMock()
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.indexed_elements = [
|
||||
{
|
||||
"center": [550, 1440],
|
||||
"text": "Confirm",
|
||||
"bounds": [500, 1400, 600, 1480],
|
||||
"class": "android.widget.Button",
|
||||
"resource_id": "btn_confirm",
|
||||
"is_ocr": False,
|
||||
}
|
||||
]
|
||||
|
||||
# Normalized (500, 600) on the default 1080x2400 device -> pixel (540, 1440),
|
||||
# which falls inside the "Confirm" button bounds.
|
||||
actions, err = node._translate_and_validate_tool(
|
||||
{"name": "click", "args": {"target": [500, 600]}}, mock_state
|
||||
)
|
||||
assert err is None
|
||||
assert actions == [
|
||||
{
|
||||
"action": "tap",
|
||||
"coordinates": [540, 1440],
|
||||
"normalized_coordinates": [500, 600],
|
||||
"times": 1,
|
||||
"delay_ms": 100,
|
||||
"target_text": "Confirm",
|
||||
"target_bounds": [500, 1400, 600, 1480],
|
||||
"target_resource_id": "btn_confirm",
|
||||
"target_class": "android.widget.Button",
|
||||
"target_label_source": "hit_test",
|
||||
}
|
||||
]
|
||||
|
||||
# Graceful degradation: no perception data at all -> bare coordinates, "none".
|
||||
mock_state.indexed_elements = []
|
||||
actions, err = node._translate_and_validate_tool(
|
||||
{"name": "click", "args": {"target": [500, 600]}}, mock_state
|
||||
)
|
||||
assert err is None
|
||||
assert actions[0]["target_text"] is None
|
||||
assert actions[0]["target_label_source"] == "none"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_node_multiple_actions():
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
@@ -258,7 +320,7 @@ async def test_operator_node_multiple_actions():
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
update_dict = node_update
|
||||
@@ -302,7 +364,7 @@ async def test_operator_node_no_record_step():
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
mock_ctx.data_engine.record_step.assert_not_called()
|
||||
@@ -352,7 +414,7 @@ async def test_operator_dynamic_prompt():
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx, prompt_components=[MockComponent()])
|
||||
node = OperatorNode(mock_ctx, prompt_components=[MockComponent()], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
|
||||
@@ -488,7 +550,7 @@ async def test_operator_history_compression():
|
||||
patch("pathlib.Path.exists", return_value=True),
|
||||
patch("pathlib.Path.read_text", mock_read_text),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, last_n_detailed=5)
|
||||
node = OperatorNode(mock_ctx, last_n_detailed=5, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
|
||||
@@ -506,7 +568,7 @@ async def test_operator_loop_detection():
|
||||
mock_state.subagent_calls = ["ask_explorer", "ask_explorer", "ask_explorer"]
|
||||
mock_state.initial_goal = "Test goal"
|
||||
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
node_update = await node(mock_state)
|
||||
@@ -568,7 +630,7 @@ async def test_operator_tracks_subagent_calls():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool])
|
||||
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
update_dict = node_update
|
||||
@@ -660,7 +722,7 @@ async def test_operator_defer_action_for_gathering():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool])
|
||||
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# Subagent tool should have been executed
|
||||
@@ -683,7 +745,7 @@ async def test_wait_actions_translation():
|
||||
from artemis.graph.state import State
|
||||
|
||||
mock_ctx = MagicMock()
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
mock_state = MagicMock(spec=State)
|
||||
|
||||
# 1. Test wait_for_delay translation
|
||||
@@ -708,7 +770,7 @@ async def test_long_press_action_translation():
|
||||
from artemis.graph.state import State
|
||||
|
||||
mock_ctx = MagicMock()
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.indexed_points = [[200, 300], [400, 500]]
|
||||
mock_state.indexed_elements = [
|
||||
@@ -744,6 +806,7 @@ async def test_long_press_action_translation():
|
||||
"target_bounds": "[100,100][300,500]",
|
||||
"target_resource_id": "btn_1",
|
||||
"target_class": "android.widget.Button",
|
||||
"target_label_source": "index",
|
||||
}
|
||||
]
|
||||
|
||||
@@ -765,6 +828,7 @@ async def test_long_press_action_translation():
|
||||
"target_bounds": None,
|
||||
"target_resource_id": None,
|
||||
"target_class": None,
|
||||
"target_label_source": "none",
|
||||
}
|
||||
]
|
||||
|
||||
@@ -844,7 +908,7 @@ async def test_operator_no_defer_for_note_updating():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_update_note_tool])
|
||||
node = OperatorNode(mock_ctx, tools=[mock_update_note_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# The update_note tool should have been executed immediately
|
||||
@@ -934,7 +998,7 @@ async def test_operator_defer_for_note_reading():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_read_note_tool])
|
||||
node = OperatorNode(mock_ctx, tools=[mock_read_note_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# The read_note tool should have been executed
|
||||
@@ -1022,7 +1086,7 @@ async def test_operator_no_defer_for_note_appending():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_append_note_tool])
|
||||
node = OperatorNode(mock_ctx, tools=[mock_append_note_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# The append_note tool should have been executed immediately
|
||||
@@ -1111,7 +1175,7 @@ async def test_operator_background_tasks_prompt_injection():
|
||||
side_effect=mock_invoke_llm_loop,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# Assert that the captured messages (prompts) contain the background tasks section
|
||||
@@ -1202,7 +1266,7 @@ async def test_operator_task_plan_warning_prompt_injection():
|
||||
side_effect=mock_invoke_llm_loop_1,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
full_content_1 = ""
|
||||
@@ -1255,7 +1319,7 @@ async def test_operator_task_plan_warning_prompt_injection():
|
||||
side_effect=mock_invoke_llm_loop_2,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
full_content_2 = ""
|
||||
@@ -1322,7 +1386,7 @@ async def test_operator_tool_limit_exceeded_warning():
|
||||
side_effect=mock_invoke_llm_loop,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
full_content = ""
|
||||
@@ -1380,7 +1444,7 @@ async def test_operator_fallback_function_call_parsing():
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# Verify state was updated with parsed click action
|
||||
@@ -1402,7 +1466,7 @@ async def test_operator_swipe_translation():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_state = MagicMock()
|
||||
mock_state.operator_raw_data = {"width": 1080, "height": 2400}
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
|
||||
# 1. Directional swipe
|
||||
actions, err = node._translate_and_validate_tool(
|
||||
|
||||
@@ -24,7 +24,7 @@ def test_operator_prompt_contract_matches_note_runtime_semantics():
|
||||
|
||||
assert "write-through tools `update_note` and `append_note`" in prompt
|
||||
assert "may accompany at most one Turn-Ending Action" in prompt
|
||||
assert "`read_note`, `list_notes`, and `save_note`" in prompt
|
||||
assert "`read_note`, `list_notes`, `save_note`, and `recall_history`" in prompt
|
||||
assert "Do NOT submit a Turn-Ending Action at the same time" not in prompt
|
||||
assert "memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`" not in prompt
|
||||
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Operator dual prompt path (M2): legacy golden parity and transcript mode.
|
||||
|
||||
Flag off (``agent.memory.transcript.enabled=false``): the legacy 2-message
|
||||
build must stay byte-for-byte identical — pinned by SHA-256 goldens recorded
|
||||
against the pre-split implementation.
|
||||
|
||||
Flag on: ``_build_prompt`` renders ``S + F + A + tail`` from the session
|
||||
transcript ledger; the turn's messages are staged after the tool loop and
|
||||
committed (with step key and validator result) at the next build.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import re
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.agents.operator.prompt_builder import load_operator_prompts
|
||||
from artemis.agents.operator.prompts import (
|
||||
PLAN_HISTORY_STATIC_POINTER,
|
||||
PLAN_HISTORY_TEMPLATE_SECTION,
|
||||
PromptBuilder,
|
||||
TemplatePromptComponent,
|
||||
render_transcript_static_system,
|
||||
)
|
||||
from artemis.config.agent import MemoryTranscriptConfig
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.memory.step_memory import StepMemoryService
|
||||
from artemis.memory.transcript import (
|
||||
EXECUTION_RESULT_MARKER,
|
||||
PLAN_RECITATION_MARKER,
|
||||
RESTORED_HISTORY_HEADER,
|
||||
TranscriptLedger,
|
||||
)
|
||||
|
||||
# SHA-256 of the legacy system message rendered by TemplatePromptComponent
|
||||
# with the fixed inputs below. Originally recorded BEFORE the M2 template
|
||||
# split; re-recorded once in M5 (2026-09-01) after the only intended template
|
||||
# change since — removing the Short-Term Memory instruction block from
|
||||
# operator.json (the <short_term_memory> relay was removed end to end per
|
||||
# pro-context-memory-redesign §3.3). Any other drift here means the flag-off
|
||||
# path is no longer byte-identical.
|
||||
GOLDEN_EMPTY_PLAN = "c45aec7d67069dad3a195534dd3c97dadb48454ca186a40eb471d5f9136589fa"
|
||||
GOLDEN_SENTINEL_PLAN = "670580efd48f237c3257d996a814145fbef4ec0bb467aa4dfa6eb2db4a613826"
|
||||
|
||||
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
|
||||
|
||||
|
||||
async def _render_legacy_system(plan_and_history: str) -> str:
|
||||
builder = PromptBuilder()
|
||||
component = TemplatePromptComponent()
|
||||
ctx = SimpleNamespace(execution_setup=None, actuator=None, data_engine=None)
|
||||
state = SimpleNamespace(initial_goal="GOLDEN_FIXED_GOAL")
|
||||
await component(
|
||||
builder,
|
||||
state,
|
||||
ctx,
|
||||
prompts=load_operator_prompts(),
|
||||
template_name="main_template",
|
||||
plan_and_history=plan_and_history,
|
||||
)
|
||||
return "".join(builder.system_parts)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_legacy_prompt_bytes_match_pre_split_golden():
|
||||
empty = await _render_legacy_system("No plan or history yet.")
|
||||
sentinel = await _render_legacy_system("PLAN_AND_HISTORY_SENTINEL\nline2")
|
||||
assert hashlib.sha256(empty.encode("utf-8")).hexdigest() == GOLDEN_EMPTY_PLAN
|
||||
assert hashlib.sha256(sentinel.encode("utf-8")).hexdigest() == GOLDEN_SENTINEL_PLAN
|
||||
|
||||
|
||||
def test_main_template_contains_exactly_one_plan_history_section():
|
||||
template = load_operator_prompts()["main_template"]
|
||||
assert template.count(PLAN_HISTORY_TEMPLATE_SECTION) == 1
|
||||
|
||||
|
||||
def test_transcript_static_system_is_stable_and_carries_no_history():
|
||||
ctx = SimpleNamespace(execution_setup=None, actuator=None, data_engine=None)
|
||||
state = SimpleNamespace(initial_goal="STATIC_GOAL")
|
||||
prompts = load_operator_prompts()
|
||||
|
||||
first = render_transcript_static_system(prompts, ctx, state)
|
||||
second = render_transcript_static_system(prompts, ctx, state)
|
||||
assert first == second # byte-stable S region
|
||||
|
||||
assert "STATIC_GOAL" in first
|
||||
# The volatile section is replaced by the static pointer...
|
||||
assert PLAN_HISTORY_STATIC_POINTER.splitlines()[1] in first
|
||||
# ...and no unrendered slot or trailing observation header survives.
|
||||
assert "{{" not in first
|
||||
assert not first.endswith("# CURRENT OBSERVATION\n")
|
||||
|
||||
|
||||
def _transcript_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.execution_setup = None
|
||||
ctx.actuator = None
|
||||
ctx.data_engine = None
|
||||
ctx.step_memory = StepMemoryService(ctx=None)
|
||||
ctx.transcript_ledger = None
|
||||
return ctx
|
||||
|
||||
|
||||
def _transcript_state(**overrides):
|
||||
state = MagicMock()
|
||||
state.subagent_calls = []
|
||||
state.initial_goal = "Transcript goal"
|
||||
state.operator_feedback = None
|
||||
state.injected_instruction = None
|
||||
state.operator_tool_limit_exceeded = False
|
||||
state.structured_decisions = None
|
||||
state.last_execution_result = None
|
||||
state.current_step_id = None
|
||||
state.latest_ui_hierarchy = []
|
||||
state.operator_raw_data = {
|
||||
"screenshot_b64": SCREENSHOT_B64,
|
||||
"xml_hierarchy": [],
|
||||
"ocr_results": None,
|
||||
"width": 1080,
|
||||
"height": 2400,
|
||||
}
|
||||
for key, value in overrides.items():
|
||||
setattr(state, key, value)
|
||||
return state
|
||||
|
||||
|
||||
def _no_action_llm(captured: list):
|
||||
mock_llm = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.tool_calls = []
|
||||
mock_response.content = "no action this turn"
|
||||
|
||||
async def mock_ainvoke(*args, **kwargs):
|
||||
captured.append(list(args[0]))
|
||||
return mock_response
|
||||
|
||||
mock_llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
return mock_llm
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_mode_two_turns_build_four_regions(tmp_path):
|
||||
ctx = _transcript_ctx()
|
||||
# Regression data shape (M2 cold-start bug, on-device A/B 2026-09-01):
|
||||
# turn 1 runs before any step record exists; by turn 2 the first step HAS
|
||||
# been recorded, so `steps` is non-empty while the previous turn is still
|
||||
# staged (turn_count == 0). The cold-start check must not read that
|
||||
# live-session state as a process restart — seeding would be rejected by
|
||||
# the empty-ledger invariant and silently demote every turn to the legacy
|
||||
# 2-message path.
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
ctx.data_engine.global_base_dir = str(tmp_path)
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
captured: list = []
|
||||
mock_llm = _no_action_llm(captured)
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
|
||||
|
||||
# Turn 1
|
||||
await node(_transcript_state())
|
||||
ledger = ctx.transcript_ledger
|
||||
assert isinstance(ledger, TranscriptLedger)
|
||||
assert ledger.has_static_prefix
|
||||
assert ledger.has_staged_turn
|
||||
assert ledger.turn_count == 0
|
||||
|
||||
turn1_messages = captured[0]
|
||||
assert isinstance(turn1_messages[0], SystemMessage)
|
||||
assert isinstance(turn1_messages[-1], HumanMessage)
|
||||
tail1_text = str(turn1_messages[-1].content)
|
||||
assert "# CURRENT OBSERVATION [T+" in tail1_text
|
||||
assert PLAN_RECITATION_MARKER in tail1_text
|
||||
|
||||
# Turn 2: step 1 is now recorded, and the previous turn is committed
|
||||
# with its step id and result.
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = [
|
||||
{
|
||||
"step_id": "step-1",
|
||||
"step_number": 1,
|
||||
"summary": "Step 1 summary",
|
||||
"relative_time": "1.0s",
|
||||
"extra_metadata": {"subgoal_hash": "default"},
|
||||
}
|
||||
]
|
||||
await node(
|
||||
_transcript_state(
|
||||
current_step_id="step-1",
|
||||
structured_decisions='[{"action": "tap"}]',
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
)
|
||||
|
||||
# The staged-but-uncommitted turn 2 state never triggers cold-start
|
||||
# seeding (the transcript path stayed live instead of falling back).
|
||||
assert not ledger.has_restored_history
|
||||
assert ledger.turn_count == 1
|
||||
turn2_messages = captured[1]
|
||||
# S region is the same object with the same bytes.
|
||||
assert turn2_messages[0] is turn1_messages[0]
|
||||
|
||||
active = list(ledger.active_messages)
|
||||
# Committed turn-1 observation followed by its validator result.
|
||||
assert any(EXECUTION_RESULT_MARKER in str(m.content) for m in active)
|
||||
committed_obs = active[0]
|
||||
# Depth-1 scrub removed the old plan recitation copy.
|
||||
assert PLAN_RECITATION_MARKER not in str(committed_obs.content)
|
||||
# The live tail still recites the plan.
|
||||
assert PLAN_RECITATION_MARKER in str(turn2_messages[-1].content)
|
||||
# Message layout is S + A + tail.
|
||||
assert turn2_messages[-1] is not turn1_messages[-1]
|
||||
assert re.search(r"T\+\d{2,}:\d{2}", str(turn2_messages[-1].content))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_actionless_turn_commits_without_validator_message():
|
||||
ctx = _transcript_ctx()
|
||||
captured: list = []
|
||||
mock_llm = _no_action_llm(captured)
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
|
||||
await node(_transcript_state())
|
||||
# Planner-rejected turns clear structured_decisions: no validator ran.
|
||||
await node(
|
||||
_transcript_state(
|
||||
current_step_id="step-1",
|
||||
structured_decisions="",
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
)
|
||||
|
||||
ledger = ctx.transcript_ledger
|
||||
assert ledger.turn_count == 1
|
||||
assert not any(
|
||||
EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_cold_start_builds_restored_history_block(tmp_path):
|
||||
ctx = _transcript_ctx()
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
ctx.data_engine.global_base_dir = str(tmp_path)
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = [
|
||||
{
|
||||
"step_id": f"step_{i}",
|
||||
"step_number": i,
|
||||
"summary": f"Step {i} summary",
|
||||
"relative_time": f"{i}.0s",
|
||||
"extra_metadata": {"subgoal_hash": "default"},
|
||||
}
|
||||
for i in range(1, 4)
|
||||
]
|
||||
|
||||
captured: list = []
|
||||
mock_llm = _no_action_llm(captured)
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
|
||||
await node(_transcript_state())
|
||||
|
||||
ledger = ctx.transcript_ledger
|
||||
assert ledger.has_restored_history
|
||||
|
||||
messages = captured[0]
|
||||
restored_text = str(messages[1].content)
|
||||
assert RESTORED_HISTORY_HEADER in restored_text
|
||||
assert "--- Execution History ---" in restored_text
|
||||
assert "Step 1 summary" in restored_text
|
||||
# The live tail still follows normally and A stays appendable.
|
||||
assert "# CURRENT OBSERVATION [T+" in str(messages[-1].content)
|
||||
assert ledger.has_staged_turn
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flag_off_keeps_legacy_two_message_build_and_no_ledger():
|
||||
ctx = _transcript_ctx()
|
||||
captured: list = []
|
||||
mock_llm = _no_action_llm(captured)
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=False))
|
||||
await node(_transcript_state())
|
||||
|
||||
assert len(captured[0]) == 2
|
||||
assert isinstance(captured[0][0], SystemMessage)
|
||||
assert isinstance(captured[0][1], HumanMessage)
|
||||
assert ctx.transcript_ledger is None
|
||||
@@ -286,6 +286,9 @@ async def test_outputter_builds_concise_history(mock_get_llm):
|
||||
}
|
||||
]
|
||||
mock_data_engine.get_agent_friendly_steps.return_value = steps
|
||||
# M5: the transcript flag defaults on, so the outputter's compiled view
|
||||
# consults the chunk store; no persisted chunks keeps the classic path.
|
||||
mock_data_engine.get_history_chunks.return_value = []
|
||||
mock_data_engine.base_dir = "/tmp/mock_session"
|
||||
ctx.data_engine = mock_data_engine
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ def test_full_toolset_is_the_default():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer",))
|
||||
@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer", "recall_history"))
|
||||
def test_removed_tool_leaves_no_reference(tool):
|
||||
"""The executable definition of 'an absent tool costs the model nothing'."""
|
||||
for name, template in load_operator_prompts().items():
|
||||
@@ -154,8 +154,8 @@ def test_full_set_enum_slots_render_verbatim():
|
||||
)
|
||||
assert (
|
||||
"(`ask_explorer`, `ask_diagnoser`, `video_analyzer`, `run_adb_command`,"
|
||||
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`, and"
|
||||
" `save_note`)" in out
|
||||
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`,"
|
||||
" `save_note`, and `recall_history`)" in out
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -12,11 +12,21 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
"""SummarizerNode tests (M2: visual-transition lens dispatch).
|
||||
|
||||
Per the history redesign §5 the Pro SummarizerNode no longer runs its own LLM
|
||||
call to produce a text capsule: it dispatches the step to the shared
|
||||
step-memory service (``ctx.step_memory``) with the pre/post screenshot bytes
|
||||
pulled from the DataEngine, and degrades gracefully when images are missing.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.agents.summarizer.summarizer import SummarizerNode
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
|
||||
class DummyState:
|
||||
@@ -36,181 +46,107 @@ class DummyState:
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
def mock_context(tmp_path):
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.device = None
|
||||
ctx.data_engine = Mock()
|
||||
ctx.background_tasks = []
|
||||
ctx.step_memory = MagicMock()
|
||||
|
||||
pre_path = tmp_path / "pre.jpg"
|
||||
post_path = tmp_path / "post.jpg"
|
||||
pre_path.write_bytes(b"PRE_IMAGE")
|
||||
post_path.write_bytes(b"POST_IMAGE")
|
||||
|
||||
engine = Mock()
|
||||
engine.get_step_number.return_value = 7
|
||||
engine.get_step_record.return_value = SimpleNamespace(
|
||||
action_taken=[{"action": "tap", "coordinates": [500, 600], "target_text": "Search"}]
|
||||
)
|
||||
engine.get_step_image_path.side_effect = lambda step_number, which="pre": (
|
||||
pre_path if which == "pre" else post_path
|
||||
)
|
||||
ctx.data_engine = engine
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_success(mock_context):
|
||||
"""Test that SummarizerNode successfully generates summary from raw thinking and execution result."""
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}])
|
||||
thinking = "The operator clicked the search button."
|
||||
result = {
|
||||
"status": "success",
|
||||
"executed_actions": [{"action": "tap", "coordinates": [500, 600]}],
|
||||
}
|
||||
|
||||
async def test_summarizer_dispatches_visual_lens(mock_context):
|
||||
"""The node dispatches to the shared service instead of calling an LLM."""
|
||||
state = DummyState(
|
||||
structured_decisions=decisions,
|
||||
operator_raw_thinking=thinking,
|
||||
last_execution_result=result,
|
||||
last_execution_result={"status": "success"},
|
||||
current_step_id="12345678-1234-5678-1234-567812345678",
|
||||
)
|
||||
|
||||
# Mock LLM response
|
||||
mock_response = Mock()
|
||||
mock_response.content = "Step completed successfully."
|
||||
mock_llm = MagicMock()
|
||||
node = SummarizerNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm
|
||||
) as mock_get_llm,
|
||||
patch("asyncio.create_task") as mock_create_task,
|
||||
):
|
||||
# Run the node call
|
||||
node = SummarizerNode(mock_context)
|
||||
await node(state)
|
||||
|
||||
# Verify task was registered in context
|
||||
assert len(mock_context.background_tasks) == 1
|
||||
assert mock_context.background_tasks[0] == mock_create_task.return_value
|
||||
|
||||
# Verify background task was spawned
|
||||
assert mock_create_task.called
|
||||
background_coroutine = mock_create_task.call_args[0][0]
|
||||
|
||||
# Await the background task synchronously for test verification
|
||||
await background_coroutine
|
||||
|
||||
# Verify LLM was invoked with proper structured messages
|
||||
assert mock_llm.astream.called
|
||||
invoked_messages = mock_llm.astream.call_args[0][0]
|
||||
|
||||
system_msg = invoked_messages[0].content
|
||||
human_msg = invoked_messages[1].content
|
||||
|
||||
assert "Summarizer Agent" in system_msg
|
||||
assert "Decisions made" in human_msg
|
||||
assert "The operator clicked the search button." in human_msg
|
||||
|
||||
# Verify Data Engine updated the step summary
|
||||
from uuid import UUID
|
||||
|
||||
mock_context.data_engine.update_step_summary.assert_called_once_with(
|
||||
UUID("12345678-1234-5678-1234-567812345678"),
|
||||
"Step completed successfully.",
|
||||
assert result == {}
|
||||
mock_context.step_memory.dispatch.assert_called_once_with(
|
||||
step_number=7,
|
||||
action_name="tap",
|
||||
action_args={"coordinates": [500, 600], "target_text": "Search"},
|
||||
pre_img_bytes=b"PRE_IMAGE",
|
||||
post_img_bytes=b"POST_IMAGE",
|
||||
exec_outcome="success",
|
||||
data_engine_step_id="12345678-1234-5678-1234-567812345678",
|
||||
)
|
||||
# No direct capsule write from the node: the lens owns the versioned
|
||||
# summary write when its background job completes.
|
||||
mock_context.data_engine.update_step_summary.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_with_native_thinking(mock_context):
|
||||
"""Test that SummarizerNode successfully generates summary from both explicit and native thoughts."""
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}])
|
||||
raw_thinking = "Explicit monologue."
|
||||
native_thinking = "Silent core reasoning."
|
||||
result = {"status": "success"}
|
||||
async def test_summarizer_degrades_gracefully_without_images(mock_context):
|
||||
"""Missing DataEngine screenshots dispatch with None bytes, no exception."""
|
||||
mock_context.data_engine.get_step_image_path.side_effect = None
|
||||
mock_context.data_engine.get_step_image_path.return_value = None
|
||||
|
||||
state = DummyState(
|
||||
structured_decisions=decisions,
|
||||
operator_raw_thinking=raw_thinking,
|
||||
operator_native_thinking=native_thinking,
|
||||
last_execution_result=result,
|
||||
last_execution_result={"status": "failed", "error": "tap missed"},
|
||||
current_step_id="12345678-1234-5678-1234-567812345678",
|
||||
)
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.content = "Summary result."
|
||||
mock_llm = MagicMock()
|
||||
node = SummarizerNode(mock_context)
|
||||
await node(state)
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm
|
||||
) as mock_get_llm,
|
||||
patch("asyncio.create_task") as mock_create_task,
|
||||
):
|
||||
node = SummarizerNode(mock_context)
|
||||
await node(state)
|
||||
|
||||
# Verify task was registered in context
|
||||
assert len(mock_context.background_tasks) == 1
|
||||
assert mock_context.background_tasks[0] == mock_create_task.return_value
|
||||
|
||||
background_coroutine = mock_create_task.call_args[0][0]
|
||||
await background_coroutine
|
||||
|
||||
assert mock_llm.astream.called
|
||||
invoked_messages = mock_llm.astream.call_args[0][0]
|
||||
human_msg = invoked_messages[1].content
|
||||
|
||||
assert "Operator explicit thoughts:" in human_msg
|
||||
assert "Explicit monologue." in human_msg
|
||||
assert "Operator native thoughts:" in human_msg
|
||||
assert "Silent core reasoning." in human_msg
|
||||
kwargs = mock_context.step_memory.dispatch.call_args.kwargs
|
||||
assert kwargs["pre_img_bytes"] is None
|
||||
assert kwargs["post_img_bytes"] is None
|
||||
assert kwargs["exec_outcome"] == "Error: tap missed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_with_chronological_trace(mock_context):
|
||||
"""Test that SummarizerNode generates a prompt containing the chronological step trace when available."""
|
||||
step_id = "12345678-1234-5678-1234-567812345678"
|
||||
async def test_summarizer_falls_back_to_structured_decisions(mock_context):
|
||||
"""Without a step record the action comes from structured_decisions, and
|
||||
extra actions ride along as additional_actions."""
|
||||
mock_context.data_engine.get_step_record.return_value = None
|
||||
|
||||
state = DummyState(
|
||||
current_step_id=step_id,
|
||||
structured_decisions=(
|
||||
'[{"action": "swipe", "coordinates": [1, 2, 3, 4]},'
|
||||
' {"action": "tap", "coordinates": [9, 9]}]'
|
||||
),
|
||||
last_execution_result={"status": "success"},
|
||||
current_step_id="12345678-1234-5678-1234-567812345678",
|
||||
)
|
||||
|
||||
mock_step_data = {
|
||||
"step_id": step_id,
|
||||
"step_number": 1,
|
||||
"relative_time": "5.0s",
|
||||
"action_taken": [{"action": "click", "target_text": "Search"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": "Let's click search."},
|
||||
],
|
||||
}
|
||||
node = SummarizerNode(mock_context)
|
||||
await node(state)
|
||||
|
||||
mock_context.data_engine.get_agent_friendly_steps.return_value = [mock_step_data]
|
||||
kwargs = mock_context.step_memory.dispatch.call_args.kwargs
|
||||
assert kwargs["action_name"] == "swipe"
|
||||
assert kwargs["action_args"]["coordinates"] == [1, 2, 3, 4]
|
||||
assert kwargs["action_args"]["additional_actions"] == [
|
||||
{"action": "tap", "coordinates": [9, 9]}
|
||||
]
|
||||
|
||||
# Mock LLM response
|
||||
mock_response = Mock()
|
||||
mock_response.content = "Summary using chronological trace."
|
||||
mock_llm = MagicMock()
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_noop_without_step_id(mock_context):
|
||||
state = DummyState(current_step_id=None)
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
node = SummarizerNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm),
|
||||
patch("asyncio.create_task") as mock_create_task,
|
||||
):
|
||||
node = SummarizerNode(mock_context)
|
||||
await node(state)
|
||||
|
||||
# Verify task was registered in context
|
||||
assert len(mock_context.background_tasks) == 1
|
||||
assert mock_context.background_tasks[0] == mock_create_task.return_value
|
||||
|
||||
background_coroutine = mock_create_task.call_args[0][0]
|
||||
await background_coroutine
|
||||
|
||||
assert mock_llm.astream.called
|
||||
invoked_messages = mock_llm.astream.call_args[0][0]
|
||||
human_msg = invoked_messages[1].content
|
||||
|
||||
assert "--- Execution History ---" in human_msg
|
||||
assert "Let's click search." in human_msg
|
||||
assert result == {}
|
||||
mock_context.step_memory.dispatch.assert_not_called()
|
||||
Reference in new issue
Block a user