merge(memory): integrate incremental transcript compaction

This commit is contained in:
somew1nd committed 2026-09-01 17:47:16 -07:00
commit 0bcddbf1d7
65 files changed
+9971 -815

No files matched your search

+325
View File
@@ -0,0 +1,325 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for the Flash scrub-edge compressor (history redesign §3.2, M1).
Covers the M1 acceptance items: the legacy-equivalent success-path product
(asserted directly since the legacy ``compress_flash_messages`` reference was
removed in M5), the three race regressions (grace recovery, grace exhaustion
without backfill, failed-summary placeholder), the freeze invariant (bytes
behind the scrub edge never change), and depth-1 XML stripping equivalence.
"""
import json
from unittest.mock import Mock
import pytest
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from artemis.agents.flash.context_compressor import ScrubEdgeCompressor
from artemis.agents.flash.summarizer import VisualStepSummarizer
from artemis.context import ArtemisContext
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.data_engine = None
return ctx
def _initial_observation():
return HumanMessage(
content=[
{"type": "text", "text": "Task: Search flights"},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_0"}},
{"type": "text", "text": "--- UI Element List ---\nTree 0"},
]
)
def _step(i: int) -> ToolMessage:
return ToolMessage(
tool_call_id=f"tc{i}",
name="click",
content=[
{"type": "text", "text": f"Action {i} completed."},
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG_{i}"}},
{"type": "text", "text": f"--- UI Element List ---\nTree {i}"},
],
)
def _simulate_turns(compressor, messages, new_messages):
"""Mimic FlashRunner's ordering: compress at turn start, then append."""
for msg in new_messages:
compressor.compress(messages)
messages.append(msg)
compressor.compress(messages)
def _content_fingerprint(msg) -> str:
return json.dumps(msg.content, sort_keys=True, default=str)
def _has_image(msg) -> bool:
return any(
isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content
)
def test_success_path_product_behind_the_scrub_edge(mock_context):
"""With summaries ready in time, every message behind the scrub edge is the
legacy-shaped product: action text kept, XML stripped, screenshot replaced
by the ``--- Historical Visual Transition ---`` summary block.
(Asserted directly; the legacy ``compress_flash_messages`` reference this
was originally diffed against was removed in M5.) Intended M1 timing
change: the swap happens at depth K, so the newest K-1 historical steps
still carry their screenshots.
"""
summarizer = VisualStepSummarizer(mock_context)
for i in range(1, 7):
summarizer._summaries[f"tc{i}"] = f"Visual summary {i}."
new_messages: list = []
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
prune_history_xml=True,
image_scrub_depth=3,
pending_grace_steps=3,
)
_simulate_turns(
compressor, new_messages, [_initial_observation()] + [_step(i) for i in range(1, 7)]
)
# Initial observation behind the edge: image silently dropped (no summary
# job), task text kept, XML stripped.
assert new_messages[0].content == [{"type": "text", "text": "Task: Search flights"}]
# Steps 1-4 behind the scrub edge: exact legacy-shaped swap product.
for i in range(1, 5):
assert new_messages[i].content == [
{"type": "text", "text": f"Action {i} completed."},
{
"type": "text",
"text": f"--- Historical Visual Transition ---\nVisual summary {i}.",
},
], f"message {i} diverged from the legacy-shaped product"
# The live observation is intact.
assert _has_image(new_messages[6])
assert "Tree 6" in str(new_messages[6].content)
# Intended timing change: step 5 (depth 2 < K) keeps its screenshot under
# the scrub edge; its XML was still stripped at depth 1.
assert _has_image(new_messages[5])
assert "Visual summary 5." not in str(new_messages[5].content)
assert "Tree 5" not in str(new_messages[5].content)
def test_race_pending_at_edge_recovers_within_grace(mock_context):
"""A step whose summary is late keeps its image at the scrub edge and is
swapped to the summary once it arrives within the grace window."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._step_inputs["tc1"] = {"step_number": 1} # job pending
messages: list = []
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
image_scrub_depth=3,
pending_grace_steps=3,
)
_simulate_turns(
compressor, messages, [_initial_observation()] + [_step(i) for i in range(1, 4)]
)
# Step 1 sits at the scrub edge with a pending summary: image retained.
assert _has_image(messages[1])
# Summary arrives within grace; the next pass applies it.
summarizer._summaries["tc1"] = "Recovered visual summary 1."
_simulate_turns(compressor, messages, [_step(4)])
assert not _has_image(messages[1])
assert {
"type": "text",
"text": "--- Historical Visual Transition ---\nRecovered visual summary 1.",
} in messages[1].content
def test_race_grace_exhausted_placeholder_never_backfilled(mock_context):
"""After the grace window a pending step becomes a placeholder, and a late
summary never mutates the frozen message again."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._step_inputs["tc1"] = {"step_number": 1} # pending forever (for now)
messages: list = []
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
image_scrub_depth=2,
pending_grace_steps=1,
)
_simulate_turns(compressor, messages, [_step(i) for i in range(1, 5)])
# Depth 4 > K + grace = 3: grace exhausted, placeholder applied.
step1_text = str(messages[0].content)
assert not _has_image(messages[0])
assert "[visual summary pending; evidence at DataEngine step 1]" in step1_text
frozen_fingerprint = _content_fingerprint(messages[0])
# The summary arriving later must NOT backfill the frozen message.
summarizer._summaries["tc1"] = "Too late summary."
_simulate_turns(compressor, messages, [_step(5)])
compressor.compress(messages)
assert _content_fingerprint(messages[0]) == frozen_fingerprint
assert "Too late summary." not in str(messages[0].content)
def test_race_failed_summary_becomes_unavailable_placeholder(mock_context):
"""A failed summary is replaced by an unavailable placeholder at the scrub
edge without waiting out the grace window."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._step_inputs["tc1"] = {"step_number": 1}
summarizer._failed.add("tc1")
messages: list = []
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
image_scrub_depth=2,
pending_grace_steps=5,
)
_simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)])
assert not _has_image(messages[0])
assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(
messages[0].content
)
def test_freeze_invariant_bytes_never_change(mock_context):
"""Once a message crosses the scrub edge its content bytes never change,
across many turns and late-arriving summaries."""
summarizer = VisualStepSummarizer(mock_context)
# Odd steps have summaries ready from the start; even steps stay pending.
for i in range(1, 11):
summarizer._step_inputs[f"tc{i}"] = {"step_number": i}
if i % 2 == 1:
summarizer._summaries[f"tc{i}"] = f"Ready summary {i}."
messages: list = []
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
image_scrub_depth=2,
pending_grace_steps=1,
)
frozen_fingerprints: dict[int, str] = {}
def check_frozen():
for idx in compressor._frozen:
fingerprint = _content_fingerprint(messages[idx])
if idx in frozen_fingerprints:
assert frozen_fingerprints[idx] == fingerprint, (
f"frozen message {idx} mutated"
)
else:
frozen_fingerprints[idx] = fingerprint
for i in range(1, 11):
compressor.compress(messages)
check_frozen()
messages.append(AIMessage(content=f"Thinking about step {i}."))
messages.append(_step(i))
if i == 6:
# Late summaries for already-frozen even steps: must be ignored.
summarizer._summaries["tc2"] = "Late summary 2."
summarizer._summaries["tc4"] = "Late summary 4."
compressor.compress(messages)
check_frozen()
assert len(frozen_fingerprints) >= 6
# A frozen pending step carries the placeholder, not the late summary.
assert "Late summary 2." not in " ".join(str(m.content) for m in messages)
def test_xml_depth1_strip_matches_legacy_semantics(mock_context):
"""Depth-1 XML stripping keeps combined-block prefixes and the live list."""
messages: list = []
compressor = ScrubEdgeCompressor(summarizer=None, prune_history_xml=True)
combined = ToolMessage(
tool_call_id="tc1",
name="click",
content=[
{
"type": "text",
"text": "Tapped Settings.\n--- UI Element List ---\n[huge tree]",
},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}},
],
)
_simulate_turns(compressor, messages, [combined, _step(2)])
# Historical combined block: prefix retained, tree stripped (the exact
# legacy ``compress_flash_messages`` product for this input).
assert messages[0].content[0] == {"type": "text", "text": "Tapped Settings."}
# Live observation keeps its UI list untouched.
assert "Tree 2" in str(messages[1].content)
def test_prune_history_xml_disabled_keeps_ui_lists(mock_context):
"""With the switch off, UI lists survive both depth-1 and freeze scrubs."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._summaries["tc1"] = "Summary 1."
messages: list = []
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
prune_history_xml=False,
image_scrub_depth=2,
pending_grace_steps=0,
)
_simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)])
# Step 1 froze (summary applied) but its UI list is preserved.
assert not _has_image(messages[0])
assert "Tree 1" in str(messages[0].content)
assert "Summary 1." in str(messages[0].content)
def test_untracked_messages_are_never_touched(mock_context):
"""Injected instructions and AI turns pass through untouched; a frozen
image-only tool message falls back to non-empty content."""
messages: list = [
HumanMessage(content="[REAL-TIME INJECTED INSTRUCTION from user]: stay put"),
]
compressor = ScrubEdgeCompressor(summarizer=None, image_scrub_depth=2)
image_only = ToolMessage(
tool_call_id="tc1",
name="click",
content=[{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}}],
)
_simulate_turns(
compressor,
messages,
[image_only, AIMessage(content="thinking"), _step(2), _step(3)],
)
assert messages[0].content == "[REAL-TIME INJECTED INSTRUCTION from user]: stay put"
assert messages[2].content == "thinking"
# Image-only tool message froze with the defensive fallback text.
assert messages[1].content == [{"type": "text", "text": "Action completed."}]
+333 -120
View File
@@ -14,12 +14,12 @@
"""Unit tests for VisualStepSummarizer and Context Compressor in Flash profile."""
from unittest.mock import AsyncMock, Mock, call
from unittest.mock import ANY, AsyncMock, Mock, call
from uuid import uuid4
import pytest
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from artemis.agents.flash.context_compressor import compress_flash_messages
from artemis.agents.flash.context_compressor import ScrubEdgeCompressor
from artemis.agents.flash.summarizer import VisualStepSummarizer
from artemis.context import ArtemisContext
from artemis.sdk.builders import Builders
@@ -100,8 +100,94 @@ async def test_summarizer_retries_independently(mock_context):
assert attempt_count == 2
@pytest.mark.asyncio
async def test_summarizer_bounded_retry_marks_failed(mock_context):
"""Persistent failures exit after 1 + retry_limit attempts with an explicit failed state."""
mock_context.data_engine = Mock()
summarizer = VisualStepSummarizer(mock_context, retry_limit=2)
summarizer._retry_delays = (0.0,)
attempt_count = 0
async def mock_ainvoke(messages):
nonlocal attempt_count
attempt_count += 1
raise TimeoutError("Simulated permanent failure")
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
step_id = uuid4()
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [100, 100]},
pre_img_bytes=b"pre1",
post_img_bytes=b"post1",
exec_outcome="Outcome 1",
action_key="tc-fail",
data_engine_step_id=step_id,
)
await summarizer.flush()
# Bounded exit: initial attempt + retry_limit retries, then explicit failure.
assert attempt_count == 3
assert summarizer.has_failed("tc-fail")
assert not summarizer.has_summary("tc-fail")
# Lossless semantics: a failed step still reads as pending so the
# compressor keeps its source image instead of dropping evidence.
assert summarizer.is_pending("tc-fail")
# DataEngine observability: pending on dispatch, failed on exhaustion.
mock_context.data_engine.update_step_summary.assert_has_calls(
[
call(step_id, None, status="pending", source="visual_transition", model=ANY),
call(step_id, None, status="failed", source="visual_transition", model=ANY),
]
)
@pytest.mark.asyncio
async def test_summarizer_success_within_retry_budget_not_failed(mock_context):
"""A summary that succeeds on a retry within the budget never enters failed state."""
summarizer = VisualStepSummarizer(mock_context, retry_limit=3)
summarizer._retry_delays = (0.0,)
attempt_count = 0
async def mock_ainvoke(messages):
nonlocal attempt_count
attempt_count += 1
if attempt_count < 3:
raise TimeoutError("Transient failure")
return AIMessage(content="[Step 1 State: Finally worked.]")
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [100, 100]},
pre_img_bytes=b"pre1",
post_img_bytes=b"post1",
exec_outcome="Outcome 1",
action_key="tc-ok",
)
await summarizer.flush()
assert attempt_count == 3
assert summarizer.has_summary("tc-ok")
assert not summarizer.has_failed("tc-ok")
def test_context_compressor_with_ready_summaries(mock_context):
"""Verify that ready summaries replace past image blocks and prune outdated XML."""
"""Ready summaries replace past image blocks and outdated XML is pruned.
Rewritten against ``ScrubEdgeCompressor`` (M5; the legacy
``compress_flash_messages`` was removed) with ``image_scrub_depth=1`` so
every historical message is scrubbed immediately, mirroring the legacy
timing this test was written for. Summaries are keyed by legacy ordinals
to keep the older-caller fallback path covered.
"""
summarizer = VisualStepSummarizer(mock_context)
summarizer._summaries[1] = "[Step 1 State: Tapped Search bar. Keyboard appeared.]"
summarizer._summaries[2] = "[Step 2 State: Typed 'flights'. Suggestions displayed.]"
@@ -134,7 +220,13 @@ def test_context_compressor_with_ready_summaries(mock_context):
),
]
compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
prune_history_xml=True,
image_scrub_depth=1,
pending_grace_steps=0,
)
compressor.compress(messages)
# Initial HumanMessage: Historical initial image pruned, task text preserved
assert len(messages[0].content) == 1
@@ -155,88 +247,15 @@ def test_context_compressor_with_ready_summaries(mock_context):
assert "Live Latest Tree 3" in messages[2].content[2]["text"]
def test_context_compressor_graceful_fallback(mock_context):
"""Verify that when a summary is not ready yet, it cleanly falls back to standard pruned state."""
summarizer = VisualStepSummarizer(mock_context)
# Step 1 has no summary in cache
messages = [
HumanMessage(
content=[
{"type": "text", "text": "Task: Search"},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
]
),
ToolMessage(
tool_call_id="tc1",
name="click",
content=[
{"type": "text", "text": "Action 'click' completed."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
],
),
]
compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
# Turn 1 should simply have image omitted (standard fallback)
assert len(messages[0].content) == 1
assert messages[0].content[0]["text"] == "Task: Search"
# Turn 2 (Latest) preserves image
assert len(messages[1].content) == 2
assert messages[1].content[1]["type"] == "image_url"
def test_context_compressor_backfills_late_summary_idempotently(mock_context):
"""A pending image remains until its late summary replaces it."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._step_inputs["action-1"] = {"step_number": 1}
messages = [
HumanMessage(
content=[
{"type": "text", "text": "Task: Search"},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
]
),
ToolMessage(
tool_call_id="action-1",
name="click",
content=[
{"type": "text", "text": "Tapped Search."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
],
),
ToolMessage(
tool_call_id="action-2",
name="input_text",
content=[
{"type": "text", "text": "Typed query."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_3"}},
],
),
]
compress_flash_messages(messages, summarizer=summarizer)
assert any(block["type"] == "image_url" for block in messages[1].content)
assert len(messages[1].content) == 2
summarizer._summaries["action-1"] = "Search field focused; keyboard appeared."
compress_flash_messages(messages, summarizer=summarizer)
compress_flash_messages(messages, summarizer=summarizer)
generated_blocks = [
block
for block in messages[1].content
if block.get("text", "").startswith("--- Historical Visual Transition ---")
]
assert len(generated_blocks) == 1
assert generated_blocks[0]["text"].endswith("Search field focused; keyboard appeared.")
assert all(block["type"] != "image_url" for block in messages[1].content)
def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context):
"""Generic tool messages cannot shift action summaries onto the wrong result."""
"""Generic tool messages cannot shift action summaries onto the wrong result.
(Rewritten against ``ScrubEdgeCompressor`` in M5. The legacy tests for
pending-summary fallback and late-summary backfill were removed with
``compress_flash_messages``: the scrub edge covers pending handling in
``test_flash_scrub_edge.py``, and late backfill is now forbidden by the
freeze invariant rather than supported.)
"""
summarizer = VisualStepSummarizer(mock_context)
summarizer._summaries["click-id"] = "Click-specific visual memory."
summarizer._summaries["type-id"] = "Typing-specific visual memory."
@@ -265,7 +284,12 @@ def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context)
),
]
compress_flash_messages(messages, summarizer=summarizer)
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
image_scrub_depth=1,
pending_grace_steps=0,
)
compressor.compress(messages)
assert "Click-specific visual memory." in str(messages[0].content)
assert "Typing-specific visual memory." not in str(messages[0].content)
@@ -274,35 +298,6 @@ def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context)
assert messages[2].content[-1]["type"] == "image_url"
def test_context_compressor_preserves_text_before_embedded_ui_list(mock_context):
"""Pruning a combined XML block must retain its action result prefix."""
messages = [
ToolMessage(
tool_call_id="old",
name="click",
content=[
{
"type": "text",
"text": "Tapped Settings.\n--- UI Element List ---\n[huge tree]",
},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}},
],
),
ToolMessage(
tool_call_id="live",
name="click",
content=[
{"type": "text", "text": "Opened page."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,B"}},
],
),
]
compress_flash_messages(messages, prune_history_xml=True)
assert messages[0].content == [{"type": "text", "text": "Tapped Settings."}]
@pytest.mark.asyncio
async def test_summarizer_keys_actions_independently_of_turn_number(mock_context):
"""Multiple actions from one LLM turn keep independent inputs and summaries."""
@@ -344,17 +339,229 @@ async def test_summarizer_keys_actions_independently_of_turn_number(mock_context
assert summarizer.get_summary("tc-1") == "First action memory."
assert summarizer.get_summary("tc-2") == "Second action memory."
assert summarizer._step_inputs["tc-1"]["data_engine_step_id"] == first_step_id
assert summarizer._step_inputs["tc-2"]["data_engine_step_id"] == second_step_id
# M1 keying: jobs are canonical under the DataEngine step id; the
# tool_call_id stays queryable as an alias.
assert summarizer.get_job_payload("tc-1")["data_engine_step_id"] == first_step_id
assert summarizer.get_job_payload("tc-2")["data_engine_step_id"] == second_step_id
assert summarizer.resolve_key("tc-1") == str(first_step_id)
mock_context.data_engine.update_step_summary.assert_has_calls(
[
call(first_step_id, "First action memory."),
call(second_step_id, "Second action memory."),
call(
first_step_id,
"First action memory.",
status="ready",
source="visual_transition",
model=ANY,
),
call(
second_step_id,
"Second action memory.",
status="ready",
source="visual_transition",
model=ANY,
),
],
any_order=True,
)
@pytest.mark.asyncio
async def test_single_frame_dispatch_uses_single_image_prompt_variant(mock_context):
"""A missing after-frame (every Pro step, by design) selects the
single-image prompt: decision frame + red marker, no AFTER ACTION section."""
summarizer = VisualStepSummarizer(mock_context)
captured: list = []
async def mock_ainvoke(messages):
captured.append(messages)
return AIMessage(
content=(
"In Step 1, I tapped the 'Battery' row marked in red in the"
" Settings list, which showed 'Network & internet' and 'Display'."
)
)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [100, 200]},
pre_img_bytes=b"decision_frame",
post_img_bytes=None,
exec_outcome="executed",
)
await summarizer.flush()
assert summarizer.has_summary(1)
system_text = str(captured[0][0].content)
human_text = str(captured[0][1].content)
assert "Exactly ONE screenshot" in system_text
assert "no independent post-action evidence" in system_text
assert "DECISION FRAME" in human_text
assert "AFTER ACTION SCREEN" not in human_text
assert "--- [2]" not in human_text
@pytest.mark.asyncio
async def test_dual_frame_dispatch_keeps_transition_prompt(mock_context):
"""Both frames present (Flash path): the classic BEFORE/AFTER transition
prompt and section labels are unchanged."""
summarizer = VisualStepSummarizer(mock_context)
captured: list = []
async def mock_ainvoke(messages):
captured.append(messages)
return AIMessage(
content=(
"In Step 1, I tapped the 'Settings' gear icon marked in red;"
" the main Settings menu opened showing 'Apps'."
)
)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [100, 200]},
pre_img_bytes=b"pre",
post_img_bytes=b"post",
exec_outcome="executed",
)
await summarizer.flush()
assert summarizer.has_summary(1)
system_text = str(captured[0][0].content)
human_text = str(captured[0][1].content)
assert "Before/After screenshots" in system_text
assert "--- [1] BEFORE ACTION SCREEN (Action Marked Visually in Red) ---" in human_text
assert "--- [2] AFTER ACTION SCREEN ---" in human_text
assert "DECISION FRAME" not in human_text
@pytest.mark.asyncio
async def test_echo_output_fails_attempt_then_recovers_on_retry(mock_context):
"""An input-marker echo is a failed attempt (not a summary): the bounded
retry regenerates and the recovered text lands normally."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._retry_delays = (0.0,)
attempts = 0
async def mock_ainvoke(messages):
nonlocal attempts
attempts += 1
if attempts == 1:
return AIMessage(content="--- [2] AFTER ACTION SCREEN ---")
return AIMessage(
content=(
"In Step 1, I tapped the search field marked in red; the field"
" showed placeholder text 'Search here'."
)
)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [1, 2]},
pre_img_bytes=b"pre",
post_img_bytes=b"post",
exec_outcome="executed",
)
await summarizer.flush()
assert attempts == 2
assert summarizer.has_summary(1)
assert "Search here" in summarizer.get_summary(1)
@pytest.mark.asyncio
async def test_degenerate_outputs_exhaust_to_failed(mock_context):
"""Echo/too-short/too-long outputs never land as summaries; persistent
degeneration exhausts the bounded retries into the explicit failed state."""
summarizer = VisualStepSummarizer(mock_context, retry_limit=1)
summarizer._retry_delays = (0.0,)
async def mock_ainvoke(messages):
text = str(messages)
if "Step 1" in text:
return AIMessage(content="--- [1] DECISION FRAME: SCREEN AT ACTION TIME ---")
if "Step 2" in text:
return AIMessage(content="x" * 2000)
return AIMessage(content="ok.")
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
for step in (1, 2, 3):
summarizer.dispatch(
step_number=step,
action_name="click",
action_args={"target": [1, 2]},
pre_img_bytes=b"pre",
post_img_bytes=None,
exec_outcome="executed",
)
await summarizer.flush()
for step in (1, 2, 3):
assert summarizer.has_failed(step), f"step {step} should have failed"
assert not summarizer.has_summary(step)
@pytest.mark.asyncio
async def test_raw_lens_call_meters_llm_usage_without_touching_context_base(mock_context):
"""The raw-model lens bypass records an llm_usage trace, but its tiny
prompt never overwrites the session's last_prompt_tokens (the compaction
thresholds' live context base)."""
from artemis.services.token_meter import get_meter
engine = Mock()
engine.current_session_id = "lens-meter-session"
engine.current_step_id = None
mock_context.data_engine = engine
meter = get_meter("lens-meter-session")
meter.last_prompt_tokens = 777
summarizer = VisualStepSummarizer(mock_context)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(
return_value=AIMessage(
content=(
"In Step 1, I tapped the 'Battery' row marked in red; the list"
" showed 'Display' and 'Battery'."
),
usage_metadata={"input_tokens": 120, "output_tokens": 30, "total_tokens": 150},
)
)
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [1, 2]},
pre_img_bytes=b"pre",
post_img_bytes=b"post",
exec_outcome="executed",
)
await summarizer.flush()
usage_calls = [
c
for c in engine.record_trace.call_args_list
if c.kwargs.get("name") == "llm_usage"
]
assert usage_calls, "the raw lens bypass must record an llm_usage trace"
payload = usage_calls[0].kwargs["payload"]
assert payload["source"].startswith("lens:visual_transition:")
assert payload["prompt_tokens"] == 120
assert meter.last_prompt_tokens == 777
def test_flash_config_and_builder():
"""Verify Flash profile configuration model and SDK builder fluent API."""
cfg = Builders.AgentConfig.with_flash_config(
@@ -395,7 +602,13 @@ async def test_data_engine_integration_with_step_summarizer(mock_context):
)
await summarizer.flush()
mock_de.update_step_summary.assert_called_once_with(1, "[Step 1 State: Input text confirmed.]")
mock_de.update_step_summary.assert_called_once_with(
1,
"[Step 1 State: Input text confirmed.]",
status="ready",
source="visual_transition",
model=ANY,
)
def test_draw_action_overlay_rendering():
@@ -0,0 +1,201 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""HistoricalStateHintPromptComponent (M4): perceptual-hash hint + silence rules.
Division of labor under test: the hint fires only for matches *older* than the
recent window (returned-to-an-early-state regime); any match within the last 3
steps silences it entirely — that regime belongs to the pixel-level
ScreenshotSimilarityPromptComponent.
"""
import base64
import io
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from PIL import Image, ImageDraw
from artemis.agents.operator.prompts import HistoricalStateHintPromptComponent, PromptBuilder
from artemis.utils.image_hash import dhash_hex
def _jpeg(draw_fn) -> bytes:
img = Image.new("RGB", (270, 600), "white")
draw_fn(ImageDraw.Draw(img))
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=85)
return buf.getvalue()
def _screen_a(d):
d.rectangle([20, 40, 250, 90], fill="#3355ff")
d.rectangle([60, 450, 210, 510], fill="#22aa66")
def _screen_b(d):
for y in range(60, 560, 80):
d.rectangle([10, y, 260, y + 50], fill="#dddddd")
SCREEN_A = _jpeg(_screen_a)
SCREEN_B = _jpeg(_screen_b)
HASH_A = dhash_hex(SCREEN_A)
HASH_B = dhash_hex(SCREEN_B)
B64_A = base64.b64encode(SCREEN_A).decode("utf-8")
def _step(number: int, post_hash: str | None) -> dict:
meta = {"post_image_dhash": post_hash} if post_hash else {}
return {"step_number": number, "extra_metadata": meta}
def _cfg(similarity_hint=True, max_distance=8):
return SimpleNamespace(
memory=SimpleNamespace(
transcript=SimpleNamespace(
similarity_hint=similarity_hint,
similarity_max_distance=max_distance,
)
)
)
async def _run(steps, b64=B64_A, cfg=None):
builder = PromptBuilder()
component = HistoricalStateHintPromptComponent()
with patch("artemis.config.load_agent_config", return_value=cfg or _cfg()):
await component(
builder, MagicMock(), MagicMock(), latest_screenshot_b64=b64, steps=steps
)
return [p for p in builder.human_parts if isinstance(p, str)]
@pytest.mark.asyncio
async def test_hint_fires_on_old_matching_step():
steps = [
_step(1, HASH_B),
_step(2, HASH_A), # old match
_step(3, HASH_B),
_step(4, HASH_B),
_step(5, HASH_B),
_step(6, HASH_B),
]
parts = await _run(steps)
assert len(parts) == 1
assert "Historical state hint" in parts[0]
assert "Step 2" in parts[0]
assert "recall_history" in parts[0]
@pytest.mark.asyncio
async def test_silent_when_recent_step_also_matches():
steps = [
_step(1, HASH_A), # old match exists...
_step(2, HASH_B),
_step(3, HASH_B),
_step(4, HASH_B),
_step(5, HASH_A), # ...but a recent (last-3) match silences the hint
_step(6, HASH_B),
]
assert await _run(steps) == []
@pytest.mark.asyncio
async def test_silent_without_any_match():
steps = [_step(n, HASH_B) for n in range(1, 7)]
assert await _run(steps) == []
@pytest.mark.asyncio
async def test_silent_when_config_disabled():
steps = [
_step(1, HASH_A),
_step(2, HASH_B),
_step(3, HASH_B),
_step(4, HASH_B),
_step(5, HASH_B),
_step(6, HASH_B),
]
assert await _run(steps, cfg=_cfg(similarity_hint=False)) == []
@pytest.mark.asyncio
async def test_silent_with_too_few_steps():
# Everything within the recent window belongs to the pixel-level note.
assert await _run([_step(1, HASH_A), _step(2, HASH_A)]) == []
@pytest.mark.asyncio
async def test_tie_prefers_most_recent_old_step():
steps = [
_step(1, HASH_A),
_step(2, HASH_A), # same distance, more recent
_step(3, HASH_B),
_step(4, HASH_B),
_step(5, HASH_B),
_step(6, HASH_B),
]
parts = await _run(steps)
assert len(parts) == 1 and "Step 2" in parts[0]
@pytest.mark.asyncio
async def test_steps_without_hashes_are_skipped():
steps = [
_step(1, None),
_step(2, None),
_step(3, None),
_step(4, None),
_step(5, None),
_step(6, None),
]
assert await _run(steps) == []
def test_record_step_stamps_perceptual_hashes(tmp_path):
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
mock_ctx = MagicMock(spec=ArtemisContext)
setup = MagicMock()
setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = setup
mock_ctx.device = None
engine = DataEngine(mock_ctx)
engine.start_session("dhash stamping test")
engine.record_step(
pre_screenshot_bytes=SCREEN_A,
post_screenshot_bytes=SCREEN_B,
summary="two distinct screens",
)
# Same pre/post bytes: post_image_name is nulled but the post-action
# screen IS the pre screen, so the post hash mirrors the pre hash.
engine.record_step(
pre_screenshot_bytes=SCREEN_A,
post_screenshot_bytes=SCREEN_A,
summary="screen unchanged",
)
for t in list(engine._pending_threads):
t.join()
steps = engine.storage.get_steps(engine.current_session_id)
meta1 = steps[0].extra_metadata
assert meta1["pre_image_dhash"] == HASH_A
assert meta1["post_image_dhash"] == HASH_B
meta2 = steps[1].extra_metadata
assert meta2["pre_image_dhash"] == HASH_A
assert meta2["post_image_dhash"] == HASH_A
+8 -8
View File
@@ -344,7 +344,7 @@ async def test_history_analyzer_integration_with_task_tree():
mock_path.exists.return_value = True
mock_path.read_text.return_value = fake_plan
# Mock build_plan_and_history
# Mock the policy-table compiled-history renderer (M4)
with (
patch(
"artemis.agents.history_analyzer.history_analyzer.get_llm",
@@ -355,23 +355,23 @@ async def test_history_analyzer_integration_with_task_tree():
return_value=mock_path,
),
patch(
"artemis.agents.history_analyzer.history_analyzer.build_plan_and_history"
"artemis.agents.history_analyzer.history_analyzer.build_history_for"
) as mock_build,
):
mock_build.return_value = "Mocked operation history"
analyzer = HistoryAnalyzer(mock_ctx)
await analyzer.run("query")
# Verify build_plan_and_history call args
# Verify build_history_for call args
mock_build.assert_called_once()
args, kwargs = mock_build.call_args
assert args[0] == fake_plan
assert args[1] == steps
assert args[0] == "history_analyzer"
assert args[1] == fake_plan
assert args[2] == steps
import hashlib
expected_hash = hashlib.md5(b"Active Subgoal").hexdigest()
assert args[2] == expected_hash
assert args[3] == expected_hash
assert kwargs.get("last_n_detailed") == 1
assert kwargs.get("min_summaries") == len(steps)
assert kwargs.get("engine") is mock_ctx.data_engine
+84 -20
View File
@@ -16,9 +16,15 @@ import json
from unittest.mock import AsyncMock, MagicMock, Mock, patch
from artemis.agents.operator.operator import OperatorNode
from artemis.config.agent import MemoryTranscriptConfig
from artemis.context import ArtemisContext
import pytest
# These tests exercise the legacy 2-message prompt path. Since M5 the
# transcript path is the default, so flag-off semantics must be requested
# explicitly (never via the config default).
LEGACY_TRANSCRIPT = MemoryTranscriptConfig(enabled=False)
@pytest.mark.asyncio
async def test_operator_node_fast_path():
@@ -58,7 +64,7 @@ async def test_operator_node_fast_path():
# Patch dependencies
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
# Verify state was updated with action
@@ -77,7 +83,7 @@ async def test_perform_action_validation():
from artemis.graph.state import State
mock_ctx = MagicMock()
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
mock_state = MagicMock(spec=State)
mock_state.indexed_points = [[200, 300], [400, 500]]
@@ -114,6 +120,7 @@ async def test_perform_action_validation():
"target_bounds": "[100,100][300,500]",
"target_resource_id": "btn_1",
"target_class": "android.widget.Button",
"target_label_source": "index",
}
]
@@ -143,6 +150,7 @@ async def test_perform_action_validation():
"target_bounds": "[300,400][500,600]",
"target_resource_id": "input_1",
"target_class": "android.widget.EditText",
"target_label_source": "index",
}
# Test swipe direction
@@ -170,6 +178,7 @@ async def test_perform_action_validation():
"target_bounds": "[300,400][500,600]",
"target_resource_id": "input_1",
"target_class": "android.widget.EditText",
"target_label_source": "index",
}
]
@@ -222,6 +231,59 @@ async def test_perform_action_validation():
assert "Unsupported app management action" in err
@pytest.mark.asyncio
async def test_bare_coordinate_click_enriched_by_hit_test():
"""A coordinate click hit tests the pre-action frame for element semantics."""
from artemis.agents.operator.operator import OperatorNode
from unittest.mock import MagicMock
from artemis.graph.state import State
mock_ctx = MagicMock()
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
mock_state = MagicMock(spec=State)
mock_state.indexed_elements = [
{
"center": [550, 1440],
"text": "Confirm",
"bounds": [500, 1400, 600, 1480],
"class": "android.widget.Button",
"resource_id": "btn_confirm",
"is_ocr": False,
}
]
# Normalized (500, 600) on the default 1080x2400 device -> pixel (540, 1440),
# which falls inside the "Confirm" button bounds.
actions, err = node._translate_and_validate_tool(
{"name": "click", "args": {"target": [500, 600]}}, mock_state
)
assert err is None
assert actions == [
{
"action": "tap",
"coordinates": [540, 1440],
"normalized_coordinates": [500, 600],
"times": 1,
"delay_ms": 100,
"target_text": "Confirm",
"target_bounds": [500, 1400, 600, 1480],
"target_resource_id": "btn_confirm",
"target_class": "android.widget.Button",
"target_label_source": "hit_test",
}
]
# Graceful degradation: no perception data at all -> bare coordinates, "none".
mock_state.indexed_elements = []
actions, err = node._translate_and_validate_tool(
{"name": "click", "args": {"target": [500, 600]}}, mock_state
)
assert err is None
assert actions[0]["target_text"] is None
assert actions[0]["target_label_source"] == "none"
@pytest.mark.asyncio
async def test_operator_node_multiple_actions():
from artemis.agents.operator.operator import OperatorNode
@@ -258,7 +320,7 @@ async def test_operator_node_multiple_actions():
mock_llm.bind_tools.return_value = mock_llm
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
update_dict = node_update
@@ -302,7 +364,7 @@ async def test_operator_node_no_record_step():
mock_llm.bind_tools.return_value = mock_llm
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
mock_ctx.data_engine.record_step.assert_not_called()
@@ -352,7 +414,7 @@ async def test_operator_dynamic_prompt():
mock_llm.bind_tools.return_value = mock_llm
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx, prompt_components=[MockComponent()])
node = OperatorNode(mock_ctx, prompt_components=[MockComponent()], transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
@@ -488,7 +550,7 @@ async def test_operator_history_compression():
patch("pathlib.Path.exists", return_value=True),
patch("pathlib.Path.read_text", mock_read_text),
):
node = OperatorNode(mock_ctx, last_n_detailed=5)
node = OperatorNode(mock_ctx, last_n_detailed=5, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
@@ -506,7 +568,7 @@ async def test_operator_loop_detection():
mock_state.subagent_calls = ["ask_explorer", "ask_explorer", "ask_explorer"]
mock_state.initial_goal = "Test goal"
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
with pytest.raises(RuntimeError) as exc_info:
node_update = await node(mock_state)
@@ -568,7 +630,7 @@ async def test_operator_tracks_subagent_calls():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool])
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
update_dict = node_update
@@ -660,7 +722,7 @@ async def test_operator_defer_action_for_gathering():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool])
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
# Subagent tool should have been executed
@@ -683,7 +745,7 @@ async def test_wait_actions_translation():
from artemis.graph.state import State
mock_ctx = MagicMock()
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
mock_state = MagicMock(spec=State)
# 1. Test wait_for_delay translation
@@ -708,7 +770,7 @@ async def test_long_press_action_translation():
from artemis.graph.state import State
mock_ctx = MagicMock()
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
mock_state = MagicMock(spec=State)
mock_state.indexed_points = [[200, 300], [400, 500]]
mock_state.indexed_elements = [
@@ -744,6 +806,7 @@ async def test_long_press_action_translation():
"target_bounds": "[100,100][300,500]",
"target_resource_id": "btn_1",
"target_class": "android.widget.Button",
"target_label_source": "index",
}
]
@@ -765,6 +828,7 @@ async def test_long_press_action_translation():
"target_bounds": None,
"target_resource_id": None,
"target_class": None,
"target_label_source": "none",
}
]
@@ -844,7 +908,7 @@ async def test_operator_no_defer_for_note_updating():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_update_note_tool])
node = OperatorNode(mock_ctx, tools=[mock_update_note_tool], transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
# The update_note tool should have been executed immediately
@@ -934,7 +998,7 @@ async def test_operator_defer_for_note_reading():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_read_note_tool])
node = OperatorNode(mock_ctx, tools=[mock_read_note_tool], transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
# The read_note tool should have been executed
@@ -1022,7 +1086,7 @@ async def test_operator_no_defer_for_note_appending():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_append_note_tool])
node = OperatorNode(mock_ctx, tools=[mock_append_note_tool], transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
# The append_note tool should have been executed immediately
@@ -1111,7 +1175,7 @@ async def test_operator_background_tasks_prompt_injection():
side_effect=mock_invoke_llm_loop,
),
):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
# Assert that the captured messages (prompts) contain the background tasks section
@@ -1202,7 +1266,7 @@ async def test_operator_task_plan_warning_prompt_injection():
side_effect=mock_invoke_llm_loop_1,
),
):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
full_content_1 = ""
@@ -1255,7 +1319,7 @@ async def test_operator_task_plan_warning_prompt_injection():
side_effect=mock_invoke_llm_loop_2,
),
):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
full_content_2 = ""
@@ -1322,7 +1386,7 @@ async def test_operator_tool_limit_exceeded_warning():
side_effect=mock_invoke_llm_loop,
),
):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
full_content = ""
@@ -1380,7 +1444,7 @@ async def test_operator_fallback_function_call_parsing():
mock_llm.bind_tools.return_value = mock_llm
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
# Verify state was updated with parsed click action
@@ -1402,7 +1466,7 @@ async def test_operator_swipe_translation():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_state = MagicMock()
mock_state.operator_raw_data = {"width": 1080, "height": 2400}
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
# 1. Directional swipe
actions, err = node._translate_and_validate_tool(
+1 -1
View File
@@ -24,7 +24,7 @@ def test_operator_prompt_contract_matches_note_runtime_semantics():
assert "write-through tools `update_note` and `append_note`" in prompt
assert "may accompany at most one Turn-Ending Action" in prompt
assert "`read_note`, `list_notes`, and `save_note`" in prompt
assert "`read_note`, `list_notes`, `save_note`, and `recall_history`" in prompt
assert "Do NOT submit a Turn-Ending Action at the same time" not in prompt
assert "memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`" not in prompt
@@ -0,0 +1,311 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Operator dual prompt path (M2): legacy golden parity and transcript mode.
Flag off (``agent.memory.transcript.enabled=false``): the legacy 2-message
build must stay byte-for-byte identical — pinned by SHA-256 goldens recorded
against the pre-split implementation.
Flag on: ``_build_prompt`` renders ``S + F + A + tail`` from the session
transcript ledger; the turn's messages are staged after the tool loop and
committed (with step key and validator result) at the next build.
"""
import base64
import hashlib
import re
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from langchain_core.messages import HumanMessage, SystemMessage
from artemis.agents.operator.operator import OperatorNode
from artemis.agents.operator.prompt_builder import load_operator_prompts
from artemis.agents.operator.prompts import (
PLAN_HISTORY_STATIC_POINTER,
PLAN_HISTORY_TEMPLATE_SECTION,
PromptBuilder,
TemplatePromptComponent,
render_transcript_static_system,
)
from artemis.config.agent import MemoryTranscriptConfig
from artemis.context import ArtemisContext
from artemis.memory.step_memory import StepMemoryService
from artemis.memory.transcript import (
EXECUTION_RESULT_MARKER,
PLAN_RECITATION_MARKER,
RESTORED_HISTORY_HEADER,
TranscriptLedger,
)
# SHA-256 of the legacy system message rendered by TemplatePromptComponent
# with the fixed inputs below. Originally recorded BEFORE the M2 template
# split; re-recorded once in M5 (2026-09-01) after the only intended template
# change since — removing the Short-Term Memory instruction block from
# operator.json (the <short_term_memory> relay was removed end to end per
# pro-context-memory-redesign §3.3). Any other drift here means the flag-off
# path is no longer byte-identical.
GOLDEN_EMPTY_PLAN = "c45aec7d67069dad3a195534dd3c97dadb48454ca186a40eb471d5f9136589fa"
GOLDEN_SENTINEL_PLAN = "670580efd48f237c3257d996a814145fbef4ec0bb467aa4dfa6eb2db4a613826"
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
async def _render_legacy_system(plan_and_history: str) -> str:
builder = PromptBuilder()
component = TemplatePromptComponent()
ctx = SimpleNamespace(execution_setup=None, actuator=None, data_engine=None)
state = SimpleNamespace(initial_goal="GOLDEN_FIXED_GOAL")
await component(
builder,
state,
ctx,
prompts=load_operator_prompts(),
template_name="main_template",
plan_and_history=plan_and_history,
)
return "".join(builder.system_parts)
@pytest.mark.asyncio
async def test_legacy_prompt_bytes_match_pre_split_golden():
empty = await _render_legacy_system("No plan or history yet.")
sentinel = await _render_legacy_system("PLAN_AND_HISTORY_SENTINEL\nline2")
assert hashlib.sha256(empty.encode("utf-8")).hexdigest() == GOLDEN_EMPTY_PLAN
assert hashlib.sha256(sentinel.encode("utf-8")).hexdigest() == GOLDEN_SENTINEL_PLAN
def test_main_template_contains_exactly_one_plan_history_section():
template = load_operator_prompts()["main_template"]
assert template.count(PLAN_HISTORY_TEMPLATE_SECTION) == 1
def test_transcript_static_system_is_stable_and_carries_no_history():
ctx = SimpleNamespace(execution_setup=None, actuator=None, data_engine=None)
state = SimpleNamespace(initial_goal="STATIC_GOAL")
prompts = load_operator_prompts()
first = render_transcript_static_system(prompts, ctx, state)
second = render_transcript_static_system(prompts, ctx, state)
assert first == second # byte-stable S region
assert "STATIC_GOAL" in first
# The volatile section is replaced by the static pointer...
assert PLAN_HISTORY_STATIC_POINTER.splitlines()[1] in first
# ...and no unrendered slot or trailing observation header survives.
assert "{{" not in first
assert not first.endswith("# CURRENT OBSERVATION\n")
def _transcript_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.execution_setup = None
ctx.actuator = None
ctx.data_engine = None
ctx.step_memory = StepMemoryService(ctx=None)
ctx.transcript_ledger = None
return ctx
def _transcript_state(**overrides):
state = MagicMock()
state.subagent_calls = []
state.initial_goal = "Transcript goal"
state.operator_feedback = None
state.injected_instruction = None
state.operator_tool_limit_exceeded = False
state.structured_decisions = None
state.last_execution_result = None
state.current_step_id = None
state.latest_ui_hierarchy = []
state.operator_raw_data = {
"screenshot_b64": SCREENSHOT_B64,
"xml_hierarchy": [],
"ocr_results": None,
"width": 1080,
"height": 2400,
}
for key, value in overrides.items():
setattr(state, key, value)
return state
def _no_action_llm(captured: list):
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.tool_calls = []
mock_response.content = "no action this turn"
async def mock_ainvoke(*args, **kwargs):
captured.append(list(args[0]))
return mock_response
mock_llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
mock_llm.bind_tools.return_value = mock_llm
return mock_llm
@pytest.mark.asyncio
async def test_transcript_mode_two_turns_build_four_regions(tmp_path):
ctx = _transcript_ctx()
# Regression data shape (M2 cold-start bug, on-device A/B 2026-09-01):
# turn 1 runs before any step record exists; by turn 2 the first step HAS
# been recorded, so `steps` is non-empty while the previous turn is still
# staged (turn_count == 0). The cold-start check must not read that
# live-session state as a process restart — seeding would be rejected by
# the empty-ledger invariant and silently demote every turn to the legacy
# 2-message path.
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
ctx.data_engine.global_base_dir = str(tmp_path)
ctx.data_engine.get_agent_friendly_steps.return_value = []
captured: list = []
mock_llm = _no_action_llm(captured)
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
# Turn 1
await node(_transcript_state())
ledger = ctx.transcript_ledger
assert isinstance(ledger, TranscriptLedger)
assert ledger.has_static_prefix
assert ledger.has_staged_turn
assert ledger.turn_count == 0
turn1_messages = captured[0]
assert isinstance(turn1_messages[0], SystemMessage)
assert isinstance(turn1_messages[-1], HumanMessage)
tail1_text = str(turn1_messages[-1].content)
assert "# CURRENT OBSERVATION [T+" in tail1_text
assert PLAN_RECITATION_MARKER in tail1_text
# Turn 2: step 1 is now recorded, and the previous turn is committed
# with its step id and result.
ctx.data_engine.get_agent_friendly_steps.return_value = [
{
"step_id": "step-1",
"step_number": 1,
"summary": "Step 1 summary",
"relative_time": "1.0s",
"extra_metadata": {"subgoal_hash": "default"},
}
]
await node(
_transcript_state(
current_step_id="step-1",
structured_decisions='[{"action": "tap"}]',
last_execution_result={"status": "success"},
)
)
# The staged-but-uncommitted turn 2 state never triggers cold-start
# seeding (the transcript path stayed live instead of falling back).
assert not ledger.has_restored_history
assert ledger.turn_count == 1
turn2_messages = captured[1]
# S region is the same object with the same bytes.
assert turn2_messages[0] is turn1_messages[0]
active = list(ledger.active_messages)
# Committed turn-1 observation followed by its validator result.
assert any(EXECUTION_RESULT_MARKER in str(m.content) for m in active)
committed_obs = active[0]
# Depth-1 scrub removed the old plan recitation copy.
assert PLAN_RECITATION_MARKER not in str(committed_obs.content)
# The live tail still recites the plan.
assert PLAN_RECITATION_MARKER in str(turn2_messages[-1].content)
# Message layout is S + A + tail.
assert turn2_messages[-1] is not turn1_messages[-1]
assert re.search(r"T\+\d{2,}:\d{2}", str(turn2_messages[-1].content))
@pytest.mark.asyncio
async def test_transcript_actionless_turn_commits_without_validator_message():
ctx = _transcript_ctx()
captured: list = []
mock_llm = _no_action_llm(captured)
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
await node(_transcript_state())
# Planner-rejected turns clear structured_decisions: no validator ran.
await node(
_transcript_state(
current_step_id="step-1",
structured_decisions="",
last_execution_result={"status": "success"},
)
)
ledger = ctx.transcript_ledger
assert ledger.turn_count == 1
assert not any(
EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages
)
@pytest.mark.asyncio
async def test_transcript_cold_start_builds_restored_history_block(tmp_path):
ctx = _transcript_ctx()
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
ctx.data_engine.global_base_dir = str(tmp_path)
ctx.data_engine.get_agent_friendly_steps.return_value = [
{
"step_id": f"step_{i}",
"step_number": i,
"summary": f"Step {i} summary",
"relative_time": f"{i}.0s",
"extra_metadata": {"subgoal_hash": "default"},
}
for i in range(1, 4)
]
captured: list = []
mock_llm = _no_action_llm(captured)
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
await node(_transcript_state())
ledger = ctx.transcript_ledger
assert ledger.has_restored_history
messages = captured[0]
restored_text = str(messages[1].content)
assert RESTORED_HISTORY_HEADER in restored_text
assert "--- Execution History ---" in restored_text
assert "Step 1 summary" in restored_text
# The live tail still follows normally and A stays appendable.
assert "# CURRENT OBSERVATION [T+" in str(messages[-1].content)
assert ledger.has_staged_turn
@pytest.mark.asyncio
async def test_flag_off_keeps_legacy_two_message_build_and_no_ledger():
ctx = _transcript_ctx()
captured: list = []
mock_llm = _no_action_llm(captured)
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=False))
await node(_transcript_state())
assert len(captured[0]) == 2
assert isinstance(captured[0][0], SystemMessage)
assert isinstance(captured[0][1], HumanMessage)
assert ctx.transcript_ledger is None
+3
View File
@@ -286,6 +286,9 @@ async def test_outputter_builds_concise_history(mock_get_llm):
}
]
mock_data_engine.get_agent_friendly_steps.return_value = steps
# M5: the transcript flag defaults on, so the outputter's compiled view
# consults the chunk store; no persisted chunks keeps the classic path.
mock_data_engine.get_history_chunks.return_value = []
mock_data_engine.base_dir = "/tmp/mock_session"
ctx.data_engine = mock_data_engine
+3 -3
View File
@@ -92,7 +92,7 @@ def test_full_toolset_is_the_default():
)
@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer",))
@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer", "recall_history"))
def test_removed_tool_leaves_no_reference(tool):
"""The executable definition of 'an absent tool costs the model nothing'."""
for name, template in load_operator_prompts().items():
@@ -154,8 +154,8 @@ def test_full_set_enum_slots_render_verbatim():
)
assert (
"(`ask_explorer`, `ask_diagnoser`, `video_analyzer`, `run_adb_command`,"
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`, and"
" `save_note`)" in out
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`,"
" `save_note`, and `recall_history`)" in out
)
+84 -148
View File
@@ -12,11 +12,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import MagicMock, Mock, patch
"""SummarizerNode tests (M2: visual-transition lens dispatch).
Per the history redesign §5 the Pro SummarizerNode no longer runs its own LLM
call to produce a text capsule: it dispatches the step to the shared
step-memory service (``ctx.step_memory``) with the pre/post screenshot bytes
pulled from the DataEngine, and degrades gracefully when images are missing.
"""
from types import SimpleNamespace
from unittest.mock import MagicMock, Mock
import pytest
from artemis.agents.summarizer.summarizer import SummarizerNode
from artemis.context import ArtemisContext
import pytest
class DummyState:
@@ -36,181 +46,107 @@ class DummyState:
@pytest.fixture
def mock_context():
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.device = None
ctx.data_engine = Mock()
ctx.background_tasks = []
ctx.step_memory = MagicMock()
pre_path = tmp_path / "pre.jpg"
post_path = tmp_path / "post.jpg"
pre_path.write_bytes(b"PRE_IMAGE")
post_path.write_bytes(b"POST_IMAGE")
engine = Mock()
engine.get_step_number.return_value = 7
engine.get_step_record.return_value = SimpleNamespace(
action_taken=[{"action": "tap", "coordinates": [500, 600], "target_text": "Search"}]
)
engine.get_step_image_path.side_effect = lambda step_number, which="pre": (
pre_path if which == "pre" else post_path
)
ctx.data_engine = engine
return ctx
@pytest.mark.asyncio
async def test_summarizer_success(mock_context):
"""Test that SummarizerNode successfully generates summary from raw thinking and execution result."""
decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}])
thinking = "The operator clicked the search button."
result = {
"status": "success",
"executed_actions": [{"action": "tap", "coordinates": [500, 600]}],
}
async def test_summarizer_dispatches_visual_lens(mock_context):
"""The node dispatches to the shared service instead of calling an LLM."""
state = DummyState(
structured_decisions=decisions,
operator_raw_thinking=thinking,
last_execution_result=result,
last_execution_result={"status": "success"},
current_step_id="12345678-1234-5678-1234-567812345678",
)
# Mock LLM response
mock_response = Mock()
mock_response.content = "Step completed successfully."
mock_llm = MagicMock()
node = SummarizerNode(mock_context)
result = await node(state)
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
with (
patch(
"artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm
) as mock_get_llm,
patch("asyncio.create_task") as mock_create_task,
):
# Run the node call
node = SummarizerNode(mock_context)
await node(state)
# Verify task was registered in context
assert len(mock_context.background_tasks) == 1
assert mock_context.background_tasks[0] == mock_create_task.return_value
# Verify background task was spawned
assert mock_create_task.called
background_coroutine = mock_create_task.call_args[0][0]
# Await the background task synchronously for test verification
await background_coroutine
# Verify LLM was invoked with proper structured messages
assert mock_llm.astream.called
invoked_messages = mock_llm.astream.call_args[0][0]
system_msg = invoked_messages[0].content
human_msg = invoked_messages[1].content
assert "Summarizer Agent" in system_msg
assert "Decisions made" in human_msg
assert "The operator clicked the search button." in human_msg
# Verify Data Engine updated the step summary
from uuid import UUID
mock_context.data_engine.update_step_summary.assert_called_once_with(
UUID("12345678-1234-5678-1234-567812345678"),
"Step completed successfully.",
assert result == {}
mock_context.step_memory.dispatch.assert_called_once_with(
step_number=7,
action_name="tap",
action_args={"coordinates": [500, 600], "target_text": "Search"},
pre_img_bytes=b"PRE_IMAGE",
post_img_bytes=b"POST_IMAGE",
exec_outcome="success",
data_engine_step_id="12345678-1234-5678-1234-567812345678",
)
# No direct capsule write from the node: the lens owns the versioned
# summary write when its background job completes.
mock_context.data_engine.update_step_summary.assert_not_called()
@pytest.mark.asyncio
async def test_summarizer_with_native_thinking(mock_context):
"""Test that SummarizerNode successfully generates summary from both explicit and native thoughts."""
decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}])
raw_thinking = "Explicit monologue."
native_thinking = "Silent core reasoning."
result = {"status": "success"}
async def test_summarizer_degrades_gracefully_without_images(mock_context):
"""Missing DataEngine screenshots dispatch with None bytes, no exception."""
mock_context.data_engine.get_step_image_path.side_effect = None
mock_context.data_engine.get_step_image_path.return_value = None
state = DummyState(
structured_decisions=decisions,
operator_raw_thinking=raw_thinking,
operator_native_thinking=native_thinking,
last_execution_result=result,
last_execution_result={"status": "failed", "error": "tap missed"},
current_step_id="12345678-1234-5678-1234-567812345678",
)
mock_response = Mock()
mock_response.content = "Summary result."
mock_llm = MagicMock()
node = SummarizerNode(mock_context)
await node(state)
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
with (
patch(
"artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm
) as mock_get_llm,
patch("asyncio.create_task") as mock_create_task,
):
node = SummarizerNode(mock_context)
await node(state)
# Verify task was registered in context
assert len(mock_context.background_tasks) == 1
assert mock_context.background_tasks[0] == mock_create_task.return_value
background_coroutine = mock_create_task.call_args[0][0]
await background_coroutine
assert mock_llm.astream.called
invoked_messages = mock_llm.astream.call_args[0][0]
human_msg = invoked_messages[1].content
assert "Operator explicit thoughts:" in human_msg
assert "Explicit monologue." in human_msg
assert "Operator native thoughts:" in human_msg
assert "Silent core reasoning." in human_msg
kwargs = mock_context.step_memory.dispatch.call_args.kwargs
assert kwargs["pre_img_bytes"] is None
assert kwargs["post_img_bytes"] is None
assert kwargs["exec_outcome"] == "Error: tap missed"
@pytest.mark.asyncio
async def test_summarizer_with_chronological_trace(mock_context):
"""Test that SummarizerNode generates a prompt containing the chronological step trace when available."""
step_id = "12345678-1234-5678-1234-567812345678"
async def test_summarizer_falls_back_to_structured_decisions(mock_context):
"""Without a step record the action comes from structured_decisions, and
extra actions ride along as additional_actions."""
mock_context.data_engine.get_step_record.return_value = None
state = DummyState(
current_step_id=step_id,
structured_decisions=(
'[{"action": "swipe", "coordinates": [1, 2, 3, 4]},'
' {"action": "tap", "coordinates": [9, 9]}]'
),
last_execution_result={"status": "success"},
current_step_id="12345678-1234-5678-1234-567812345678",
)
mock_step_data = {
"step_id": step_id,
"step_number": 1,
"relative_time": "5.0s",
"action_taken": [{"action": "click", "target_text": "Search"}],
"last_execution_result": {"status": "success"},
"interleaved_events": [
{"type": "thought", "content": "Let's click search."},
],
}
node = SummarizerNode(mock_context)
await node(state)
mock_context.data_engine.get_agent_friendly_steps.return_value = [mock_step_data]
kwargs = mock_context.step_memory.dispatch.call_args.kwargs
assert kwargs["action_name"] == "swipe"
assert kwargs["action_args"]["coordinates"] == [1, 2, 3, 4]
assert kwargs["action_args"]["additional_actions"] == [
{"action": "tap", "coordinates": [9, 9]}
]
# Mock LLM response
mock_response = Mock()
mock_response.content = "Summary using chronological trace."
mock_llm = MagicMock()
async def mock_astream(*args, **kwargs):
yield mock_response
@pytest.mark.asyncio
async def test_summarizer_noop_without_step_id(mock_context):
state = DummyState(current_step_id=None)
mock_llm.astream.side_effect = mock_astream
node = SummarizerNode(mock_context)
result = await node(state)
with (
patch("artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm),
patch("asyncio.create_task") as mock_create_task,
):
node = SummarizerNode(mock_context)
await node(state)
# Verify task was registered in context
assert len(mock_context.background_tasks) == 1
assert mock_context.background_tasks[0] == mock_create_task.return_value
background_coroutine = mock_create_task.call_args[0][0]
await background_coroutine
assert mock_llm.astream.called
invoked_messages = mock_llm.astream.call_args[0][0]
human_msg = invoked_messages[1].content
assert "--- Execution History ---" in human_msg
assert "Let's click search." in human_msg
assert result == {}
mock_context.step_memory.dispatch.assert_not_called()