merge(memory): integrate incremental transcript compaction

This commit is contained in:
somew1nd committed 2026-09-01 17:47:16 -07:00
commit 0bcddbf1d7
65 files changed
+9971 -815

No files matched your search

+325
View File
@@ -0,0 +1,325 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for the Flash scrub-edge compressor (history redesign §3.2, M1).
Covers the M1 acceptance items: the legacy-equivalent success-path product
(asserted directly since the legacy ``compress_flash_messages`` reference was
removed in M5), the three race regressions (grace recovery, grace exhaustion
without backfill, failed-summary placeholder), the freeze invariant (bytes
behind the scrub edge never change), and depth-1 XML stripping equivalence.
"""
import json
from unittest.mock import Mock
import pytest
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from artemis.agents.flash.context_compressor import ScrubEdgeCompressor
from artemis.agents.flash.summarizer import VisualStepSummarizer
from artemis.context import ArtemisContext
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.data_engine = None
return ctx
def _initial_observation():
return HumanMessage(
content=[
{"type": "text", "text": "Task: Search flights"},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_0"}},
{"type": "text", "text": "--- UI Element List ---\nTree 0"},
]
)
def _step(i: int) -> ToolMessage:
return ToolMessage(
tool_call_id=f"tc{i}",
name="click",
content=[
{"type": "text", "text": f"Action {i} completed."},
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG_{i}"}},
{"type": "text", "text": f"--- UI Element List ---\nTree {i}"},
],
)
def _simulate_turns(compressor, messages, new_messages):
"""Mimic FlashRunner's ordering: compress at turn start, then append."""
for msg in new_messages:
compressor.compress(messages)
messages.append(msg)
compressor.compress(messages)
def _content_fingerprint(msg) -> str:
return json.dumps(msg.content, sort_keys=True, default=str)
def _has_image(msg) -> bool:
return any(
isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content
)
def test_success_path_product_behind_the_scrub_edge(mock_context):
"""With summaries ready in time, every message behind the scrub edge is the
legacy-shaped product: action text kept, XML stripped, screenshot replaced
by the ``--- Historical Visual Transition ---`` summary block.
(Asserted directly; the legacy ``compress_flash_messages`` reference this
was originally diffed against was removed in M5.) Intended M1 timing
change: the swap happens at depth K, so the newest K-1 historical steps
still carry their screenshots.
"""
summarizer = VisualStepSummarizer(mock_context)
for i in range(1, 7):
summarizer._summaries[f"tc{i}"] = f"Visual summary {i}."
new_messages: list = []
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
prune_history_xml=True,
image_scrub_depth=3,
pending_grace_steps=3,
)
_simulate_turns(
compressor, new_messages, [_initial_observation()] + [_step(i) for i in range(1, 7)]
)
# Initial observation behind the edge: image silently dropped (no summary
# job), task text kept, XML stripped.
assert new_messages[0].content == [{"type": "text", "text": "Task: Search flights"}]
# Steps 1-4 behind the scrub edge: exact legacy-shaped swap product.
for i in range(1, 5):
assert new_messages[i].content == [
{"type": "text", "text": f"Action {i} completed."},
{
"type": "text",
"text": f"--- Historical Visual Transition ---\nVisual summary {i}.",
},
], f"message {i} diverged from the legacy-shaped product"
# The live observation is intact.
assert _has_image(new_messages[6])
assert "Tree 6" in str(new_messages[6].content)
# Intended timing change: step 5 (depth 2 < K) keeps its screenshot under
# the scrub edge; its XML was still stripped at depth 1.
assert _has_image(new_messages[5])
assert "Visual summary 5." not in str(new_messages[5].content)
assert "Tree 5" not in str(new_messages[5].content)
def test_race_pending_at_edge_recovers_within_grace(mock_context):
"""A step whose summary is late keeps its image at the scrub edge and is
swapped to the summary once it arrives within the grace window."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._step_inputs["tc1"] = {"step_number": 1} # job pending
messages: list = []
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
image_scrub_depth=3,
pending_grace_steps=3,
)
_simulate_turns(
compressor, messages, [_initial_observation()] + [_step(i) for i in range(1, 4)]
)
# Step 1 sits at the scrub edge with a pending summary: image retained.
assert _has_image(messages[1])
# Summary arrives within grace; the next pass applies it.
summarizer._summaries["tc1"] = "Recovered visual summary 1."
_simulate_turns(compressor, messages, [_step(4)])
assert not _has_image(messages[1])
assert {
"type": "text",
"text": "--- Historical Visual Transition ---\nRecovered visual summary 1.",
} in messages[1].content
def test_race_grace_exhausted_placeholder_never_backfilled(mock_context):
"""After the grace window a pending step becomes a placeholder, and a late
summary never mutates the frozen message again."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._step_inputs["tc1"] = {"step_number": 1} # pending forever (for now)
messages: list = []
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
image_scrub_depth=2,
pending_grace_steps=1,
)
_simulate_turns(compressor, messages, [_step(i) for i in range(1, 5)])
# Depth 4 > K + grace = 3: grace exhausted, placeholder applied.
step1_text = str(messages[0].content)
assert not _has_image(messages[0])
assert "[visual summary pending; evidence at DataEngine step 1]" in step1_text
frozen_fingerprint = _content_fingerprint(messages[0])
# The summary arriving later must NOT backfill the frozen message.
summarizer._summaries["tc1"] = "Too late summary."
_simulate_turns(compressor, messages, [_step(5)])
compressor.compress(messages)
assert _content_fingerprint(messages[0]) == frozen_fingerprint
assert "Too late summary." not in str(messages[0].content)
def test_race_failed_summary_becomes_unavailable_placeholder(mock_context):
"""A failed summary is replaced by an unavailable placeholder at the scrub
edge without waiting out the grace window."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._step_inputs["tc1"] = {"step_number": 1}
summarizer._failed.add("tc1")
messages: list = []
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
image_scrub_depth=2,
pending_grace_steps=5,
)
_simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)])
assert not _has_image(messages[0])
assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(
messages[0].content
)
def test_freeze_invariant_bytes_never_change(mock_context):
"""Once a message crosses the scrub edge its content bytes never change,
across many turns and late-arriving summaries."""
summarizer = VisualStepSummarizer(mock_context)
# Odd steps have summaries ready from the start; even steps stay pending.
for i in range(1, 11):
summarizer._step_inputs[f"tc{i}"] = {"step_number": i}
if i % 2 == 1:
summarizer._summaries[f"tc{i}"] = f"Ready summary {i}."
messages: list = []
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
image_scrub_depth=2,
pending_grace_steps=1,
)
frozen_fingerprints: dict[int, str] = {}
def check_frozen():
for idx in compressor._frozen:
fingerprint = _content_fingerprint(messages[idx])
if idx in frozen_fingerprints:
assert frozen_fingerprints[idx] == fingerprint, (
f"frozen message {idx} mutated"
)
else:
frozen_fingerprints[idx] = fingerprint
for i in range(1, 11):
compressor.compress(messages)
check_frozen()
messages.append(AIMessage(content=f"Thinking about step {i}."))
messages.append(_step(i))
if i == 6:
# Late summaries for already-frozen even steps: must be ignored.
summarizer._summaries["tc2"] = "Late summary 2."
summarizer._summaries["tc4"] = "Late summary 4."
compressor.compress(messages)
check_frozen()
assert len(frozen_fingerprints) >= 6
# A frozen pending step carries the placeholder, not the late summary.
assert "Late summary 2." not in " ".join(str(m.content) for m in messages)
def test_xml_depth1_strip_matches_legacy_semantics(mock_context):
"""Depth-1 XML stripping keeps combined-block prefixes and the live list."""
messages: list = []
compressor = ScrubEdgeCompressor(summarizer=None, prune_history_xml=True)
combined = ToolMessage(
tool_call_id="tc1",
name="click",
content=[
{
"type": "text",
"text": "Tapped Settings.\n--- UI Element List ---\n[huge tree]",
},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}},
],
)
_simulate_turns(compressor, messages, [combined, _step(2)])
# Historical combined block: prefix retained, tree stripped (the exact
# legacy ``compress_flash_messages`` product for this input).
assert messages[0].content[0] == {"type": "text", "text": "Tapped Settings."}
# Live observation keeps its UI list untouched.
assert "Tree 2" in str(messages[1].content)
def test_prune_history_xml_disabled_keeps_ui_lists(mock_context):
"""With the switch off, UI lists survive both depth-1 and freeze scrubs."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._summaries["tc1"] = "Summary 1."
messages: list = []
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
prune_history_xml=False,
image_scrub_depth=2,
pending_grace_steps=0,
)
_simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)])
# Step 1 froze (summary applied) but its UI list is preserved.
assert not _has_image(messages[0])
assert "Tree 1" in str(messages[0].content)
assert "Summary 1." in str(messages[0].content)
def test_untracked_messages_are_never_touched(mock_context):
"""Injected instructions and AI turns pass through untouched; a frozen
image-only tool message falls back to non-empty content."""
messages: list = [
HumanMessage(content="[REAL-TIME INJECTED INSTRUCTION from user]: stay put"),
]
compressor = ScrubEdgeCompressor(summarizer=None, image_scrub_depth=2)
image_only = ToolMessage(
tool_call_id="tc1",
name="click",
content=[{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}}],
)
_simulate_turns(
compressor,
messages,
[image_only, AIMessage(content="thinking"), _step(2), _step(3)],
)
assert messages[0].content == "[REAL-TIME INJECTED INSTRUCTION from user]: stay put"
assert messages[2].content == "thinking"
# Image-only tool message froze with the defensive fallback text.
assert messages[1].content == [{"type": "text", "text": "Action completed."}]
+333 -120
View File
@@ -14,12 +14,12 @@
"""Unit tests for VisualStepSummarizer and Context Compressor in Flash profile."""
from unittest.mock import AsyncMock, Mock, call
from unittest.mock import ANY, AsyncMock, Mock, call
from uuid import uuid4
import pytest
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from artemis.agents.flash.context_compressor import compress_flash_messages
from artemis.agents.flash.context_compressor import ScrubEdgeCompressor
from artemis.agents.flash.summarizer import VisualStepSummarizer
from artemis.context import ArtemisContext
from artemis.sdk.builders import Builders
@@ -100,8 +100,94 @@ async def test_summarizer_retries_independently(mock_context):
assert attempt_count == 2
@pytest.mark.asyncio
async def test_summarizer_bounded_retry_marks_failed(mock_context):
"""Persistent failures exit after 1 + retry_limit attempts with an explicit failed state."""
mock_context.data_engine = Mock()
summarizer = VisualStepSummarizer(mock_context, retry_limit=2)
summarizer._retry_delays = (0.0,)
attempt_count = 0
async def mock_ainvoke(messages):
nonlocal attempt_count
attempt_count += 1
raise TimeoutError("Simulated permanent failure")
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
step_id = uuid4()
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [100, 100]},
pre_img_bytes=b"pre1",
post_img_bytes=b"post1",
exec_outcome="Outcome 1",
action_key="tc-fail",
data_engine_step_id=step_id,
)
await summarizer.flush()
# Bounded exit: initial attempt + retry_limit retries, then explicit failure.
assert attempt_count == 3
assert summarizer.has_failed("tc-fail")
assert not summarizer.has_summary("tc-fail")
# Lossless semantics: a failed step still reads as pending so the
# compressor keeps its source image instead of dropping evidence.
assert summarizer.is_pending("tc-fail")
# DataEngine observability: pending on dispatch, failed on exhaustion.
mock_context.data_engine.update_step_summary.assert_has_calls(
[
call(step_id, None, status="pending", source="visual_transition", model=ANY),
call(step_id, None, status="failed", source="visual_transition", model=ANY),
]
)
@pytest.mark.asyncio
async def test_summarizer_success_within_retry_budget_not_failed(mock_context):
"""A summary that succeeds on a retry within the budget never enters failed state."""
summarizer = VisualStepSummarizer(mock_context, retry_limit=3)
summarizer._retry_delays = (0.0,)
attempt_count = 0
async def mock_ainvoke(messages):
nonlocal attempt_count
attempt_count += 1
if attempt_count < 3:
raise TimeoutError("Transient failure")
return AIMessage(content="[Step 1 State: Finally worked.]")
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [100, 100]},
pre_img_bytes=b"pre1",
post_img_bytes=b"post1",
exec_outcome="Outcome 1",
action_key="tc-ok",
)
await summarizer.flush()
assert attempt_count == 3
assert summarizer.has_summary("tc-ok")
assert not summarizer.has_failed("tc-ok")
def test_context_compressor_with_ready_summaries(mock_context):
"""Verify that ready summaries replace past image blocks and prune outdated XML."""
"""Ready summaries replace past image blocks and outdated XML is pruned.
Rewritten against ``ScrubEdgeCompressor`` (M5; the legacy
``compress_flash_messages`` was removed) with ``image_scrub_depth=1`` so
every historical message is scrubbed immediately, mirroring the legacy
timing this test was written for. Summaries are keyed by legacy ordinals
to keep the older-caller fallback path covered.
"""
summarizer = VisualStepSummarizer(mock_context)
summarizer._summaries[1] = "[Step 1 State: Tapped Search bar. Keyboard appeared.]"
summarizer._summaries[2] = "[Step 2 State: Typed 'flights'. Suggestions displayed.]"
@@ -134,7 +220,13 @@ def test_context_compressor_with_ready_summaries(mock_context):
),
]
compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
prune_history_xml=True,
image_scrub_depth=1,
pending_grace_steps=0,
)
compressor.compress(messages)
# Initial HumanMessage: Historical initial image pruned, task text preserved
assert len(messages[0].content) == 1
@@ -155,88 +247,15 @@ def test_context_compressor_with_ready_summaries(mock_context):
assert "Live Latest Tree 3" in messages[2].content[2]["text"]
def test_context_compressor_graceful_fallback(mock_context):
"""Verify that when a summary is not ready yet, it cleanly falls back to standard pruned state."""
summarizer = VisualStepSummarizer(mock_context)
# Step 1 has no summary in cache
messages = [
HumanMessage(
content=[
{"type": "text", "text": "Task: Search"},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
]
),
ToolMessage(
tool_call_id="tc1",
name="click",
content=[
{"type": "text", "text": "Action 'click' completed."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
],
),
]
compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
# Turn 1 should simply have image omitted (standard fallback)
assert len(messages[0].content) == 1
assert messages[0].content[0]["text"] == "Task: Search"
# Turn 2 (Latest) preserves image
assert len(messages[1].content) == 2
assert messages[1].content[1]["type"] == "image_url"
def test_context_compressor_backfills_late_summary_idempotently(mock_context):
"""A pending image remains until its late summary replaces it."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._step_inputs["action-1"] = {"step_number": 1}
messages = [
HumanMessage(
content=[
{"type": "text", "text": "Task: Search"},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
]
),
ToolMessage(
tool_call_id="action-1",
name="click",
content=[
{"type": "text", "text": "Tapped Search."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
],
),
ToolMessage(
tool_call_id="action-2",
name="input_text",
content=[
{"type": "text", "text": "Typed query."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_3"}},
],
),
]
compress_flash_messages(messages, summarizer=summarizer)
assert any(block["type"] == "image_url" for block in messages[1].content)
assert len(messages[1].content) == 2
summarizer._summaries["action-1"] = "Search field focused; keyboard appeared."
compress_flash_messages(messages, summarizer=summarizer)
compress_flash_messages(messages, summarizer=summarizer)
generated_blocks = [
block
for block in messages[1].content
if block.get("text", "").startswith("--- Historical Visual Transition ---")
]
assert len(generated_blocks) == 1
assert generated_blocks[0]["text"].endswith("Search field focused; keyboard appeared.")
assert all(block["type"] != "image_url" for block in messages[1].content)
def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context):
"""Generic tool messages cannot shift action summaries onto the wrong result."""
"""Generic tool messages cannot shift action summaries onto the wrong result.
(Rewritten against ``ScrubEdgeCompressor`` in M5. The legacy tests for
pending-summary fallback and late-summary backfill were removed with
``compress_flash_messages``: the scrub edge covers pending handling in
``test_flash_scrub_edge.py``, and late backfill is now forbidden by the
freeze invariant rather than supported.)
"""
summarizer = VisualStepSummarizer(mock_context)
summarizer._summaries["click-id"] = "Click-specific visual memory."
summarizer._summaries["type-id"] = "Typing-specific visual memory."
@@ -265,7 +284,12 @@ def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context)
),
]
compress_flash_messages(messages, summarizer=summarizer)
compressor = ScrubEdgeCompressor(
summarizer=summarizer,
image_scrub_depth=1,
pending_grace_steps=0,
)
compressor.compress(messages)
assert "Click-specific visual memory." in str(messages[0].content)
assert "Typing-specific visual memory." not in str(messages[0].content)
@@ -274,35 +298,6 @@ def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context)
assert messages[2].content[-1]["type"] == "image_url"
def test_context_compressor_preserves_text_before_embedded_ui_list(mock_context):
"""Pruning a combined XML block must retain its action result prefix."""
messages = [
ToolMessage(
tool_call_id="old",
name="click",
content=[
{
"type": "text",
"text": "Tapped Settings.\n--- UI Element List ---\n[huge tree]",
},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}},
],
),
ToolMessage(
tool_call_id="live",
name="click",
content=[
{"type": "text", "text": "Opened page."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,B"}},
],
),
]
compress_flash_messages(messages, prune_history_xml=True)
assert messages[0].content == [{"type": "text", "text": "Tapped Settings."}]
@pytest.mark.asyncio
async def test_summarizer_keys_actions_independently_of_turn_number(mock_context):
"""Multiple actions from one LLM turn keep independent inputs and summaries."""
@@ -344,17 +339,229 @@ async def test_summarizer_keys_actions_independently_of_turn_number(mock_context
assert summarizer.get_summary("tc-1") == "First action memory."
assert summarizer.get_summary("tc-2") == "Second action memory."
assert summarizer._step_inputs["tc-1"]["data_engine_step_id"] == first_step_id
assert summarizer._step_inputs["tc-2"]["data_engine_step_id"] == second_step_id
# M1 keying: jobs are canonical under the DataEngine step id; the
# tool_call_id stays queryable as an alias.
assert summarizer.get_job_payload("tc-1")["data_engine_step_id"] == first_step_id
assert summarizer.get_job_payload("tc-2")["data_engine_step_id"] == second_step_id
assert summarizer.resolve_key("tc-1") == str(first_step_id)
mock_context.data_engine.update_step_summary.assert_has_calls(
[
call(first_step_id, "First action memory."),
call(second_step_id, "Second action memory."),
call(
first_step_id,
"First action memory.",
status="ready",
source="visual_transition",
model=ANY,
),
call(
second_step_id,
"Second action memory.",
status="ready",
source="visual_transition",
model=ANY,
),
],
any_order=True,
)
@pytest.mark.asyncio
async def test_single_frame_dispatch_uses_single_image_prompt_variant(mock_context):
"""A missing after-frame (every Pro step, by design) selects the
single-image prompt: decision frame + red marker, no AFTER ACTION section."""
summarizer = VisualStepSummarizer(mock_context)
captured: list = []
async def mock_ainvoke(messages):
captured.append(messages)
return AIMessage(
content=(
"In Step 1, I tapped the 'Battery' row marked in red in the"
" Settings list, which showed 'Network & internet' and 'Display'."
)
)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [100, 200]},
pre_img_bytes=b"decision_frame",
post_img_bytes=None,
exec_outcome="executed",
)
await summarizer.flush()
assert summarizer.has_summary(1)
system_text = str(captured[0][0].content)
human_text = str(captured[0][1].content)
assert "Exactly ONE screenshot" in system_text
assert "no independent post-action evidence" in system_text
assert "DECISION FRAME" in human_text
assert "AFTER ACTION SCREEN" not in human_text
assert "--- [2]" not in human_text
@pytest.mark.asyncio
async def test_dual_frame_dispatch_keeps_transition_prompt(mock_context):
"""Both frames present (Flash path): the classic BEFORE/AFTER transition
prompt and section labels are unchanged."""
summarizer = VisualStepSummarizer(mock_context)
captured: list = []
async def mock_ainvoke(messages):
captured.append(messages)
return AIMessage(
content=(
"In Step 1, I tapped the 'Settings' gear icon marked in red;"
" the main Settings menu opened showing 'Apps'."
)
)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [100, 200]},
pre_img_bytes=b"pre",
post_img_bytes=b"post",
exec_outcome="executed",
)
await summarizer.flush()
assert summarizer.has_summary(1)
system_text = str(captured[0][0].content)
human_text = str(captured[0][1].content)
assert "Before/After screenshots" in system_text
assert "--- [1] BEFORE ACTION SCREEN (Action Marked Visually in Red) ---" in human_text
assert "--- [2] AFTER ACTION SCREEN ---" in human_text
assert "DECISION FRAME" not in human_text
@pytest.mark.asyncio
async def test_echo_output_fails_attempt_then_recovers_on_retry(mock_context):
"""An input-marker echo is a failed attempt (not a summary): the bounded
retry regenerates and the recovered text lands normally."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._retry_delays = (0.0,)
attempts = 0
async def mock_ainvoke(messages):
nonlocal attempts
attempts += 1
if attempts == 1:
return AIMessage(content="--- [2] AFTER ACTION SCREEN ---")
return AIMessage(
content=(
"In Step 1, I tapped the search field marked in red; the field"
" showed placeholder text 'Search here'."
)
)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [1, 2]},
pre_img_bytes=b"pre",
post_img_bytes=b"post",
exec_outcome="executed",
)
await summarizer.flush()
assert attempts == 2
assert summarizer.has_summary(1)
assert "Search here" in summarizer.get_summary(1)
@pytest.mark.asyncio
async def test_degenerate_outputs_exhaust_to_failed(mock_context):
"""Echo/too-short/too-long outputs never land as summaries; persistent
degeneration exhausts the bounded retries into the explicit failed state."""
summarizer = VisualStepSummarizer(mock_context, retry_limit=1)
summarizer._retry_delays = (0.0,)
async def mock_ainvoke(messages):
text = str(messages)
if "Step 1" in text:
return AIMessage(content="--- [1] DECISION FRAME: SCREEN AT ACTION TIME ---")
if "Step 2" in text:
return AIMessage(content="x" * 2000)
return AIMessage(content="ok.")
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
for step in (1, 2, 3):
summarizer.dispatch(
step_number=step,
action_name="click",
action_args={"target": [1, 2]},
pre_img_bytes=b"pre",
post_img_bytes=None,
exec_outcome="executed",
)
await summarizer.flush()
for step in (1, 2, 3):
assert summarizer.has_failed(step), f"step {step} should have failed"
assert not summarizer.has_summary(step)
@pytest.mark.asyncio
async def test_raw_lens_call_meters_llm_usage_without_touching_context_base(mock_context):
"""The raw-model lens bypass records an llm_usage trace, but its tiny
prompt never overwrites the session's last_prompt_tokens (the compaction
thresholds' live context base)."""
from artemis.services.token_meter import get_meter
engine = Mock()
engine.current_session_id = "lens-meter-session"
engine.current_step_id = None
mock_context.data_engine = engine
meter = get_meter("lens-meter-session")
meter.last_prompt_tokens = 777
summarizer = VisualStepSummarizer(mock_context)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(
return_value=AIMessage(
content=(
"In Step 1, I tapped the 'Battery' row marked in red; the list"
" showed 'Display' and 'Battery'."
),
usage_metadata={"input_tokens": 120, "output_tokens": 30, "total_tokens": 150},
)
)
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [1, 2]},
pre_img_bytes=b"pre",
post_img_bytes=b"post",
exec_outcome="executed",
)
await summarizer.flush()
usage_calls = [
c
for c in engine.record_trace.call_args_list
if c.kwargs.get("name") == "llm_usage"
]
assert usage_calls, "the raw lens bypass must record an llm_usage trace"
payload = usage_calls[0].kwargs["payload"]
assert payload["source"].startswith("lens:visual_transition:")
assert payload["prompt_tokens"] == 120
assert meter.last_prompt_tokens == 777
def test_flash_config_and_builder():
"""Verify Flash profile configuration model and SDK builder fluent API."""
cfg = Builders.AgentConfig.with_flash_config(
@@ -395,7 +602,13 @@ async def test_data_engine_integration_with_step_summarizer(mock_context):
)
await summarizer.flush()
mock_de.update_step_summary.assert_called_once_with(1, "[Step 1 State: Input text confirmed.]")
mock_de.update_step_summary.assert_called_once_with(
1,
"[Step 1 State: Input text confirmed.]",
status="ready",
source="visual_transition",
model=ANY,
)
def test_draw_action_overlay_rendering():
@@ -0,0 +1,201 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""HistoricalStateHintPromptComponent (M4): perceptual-hash hint + silence rules.
Division of labor under test: the hint fires only for matches *older* than the
recent window (returned-to-an-early-state regime); any match within the last 3
steps silences it entirely — that regime belongs to the pixel-level
ScreenshotSimilarityPromptComponent.
"""
import base64
import io
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from PIL import Image, ImageDraw
from artemis.agents.operator.prompts import HistoricalStateHintPromptComponent, PromptBuilder
from artemis.utils.image_hash import dhash_hex
def _jpeg(draw_fn) -> bytes:
img = Image.new("RGB", (270, 600), "white")
draw_fn(ImageDraw.Draw(img))
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=85)
return buf.getvalue()
def _screen_a(d):
d.rectangle([20, 40, 250, 90], fill="#3355ff")
d.rectangle([60, 450, 210, 510], fill="#22aa66")
def _screen_b(d):
for y in range(60, 560, 80):
d.rectangle([10, y, 260, y + 50], fill="#dddddd")
SCREEN_A = _jpeg(_screen_a)
SCREEN_B = _jpeg(_screen_b)
HASH_A = dhash_hex(SCREEN_A)
HASH_B = dhash_hex(SCREEN_B)
B64_A = base64.b64encode(SCREEN_A).decode("utf-8")
def _step(number: int, post_hash: str | None) -> dict:
meta = {"post_image_dhash": post_hash} if post_hash else {}
return {"step_number": number, "extra_metadata": meta}
def _cfg(similarity_hint=True, max_distance=8):
return SimpleNamespace(
memory=SimpleNamespace(
transcript=SimpleNamespace(
similarity_hint=similarity_hint,
similarity_max_distance=max_distance,
)
)
)
async def _run(steps, b64=B64_A, cfg=None):
builder = PromptBuilder()
component = HistoricalStateHintPromptComponent()
with patch("artemis.config.load_agent_config", return_value=cfg or _cfg()):
await component(
builder, MagicMock(), MagicMock(), latest_screenshot_b64=b64, steps=steps
)
return [p for p in builder.human_parts if isinstance(p, str)]
@pytest.mark.asyncio
async def test_hint_fires_on_old_matching_step():
steps = [
_step(1, HASH_B),
_step(2, HASH_A), # old match
_step(3, HASH_B),
_step(4, HASH_B),
_step(5, HASH_B),
_step(6, HASH_B),
]
parts = await _run(steps)
assert len(parts) == 1
assert "Historical state hint" in parts[0]
assert "Step 2" in parts[0]
assert "recall_history" in parts[0]
@pytest.mark.asyncio
async def test_silent_when_recent_step_also_matches():
steps = [
_step(1, HASH_A), # old match exists...
_step(2, HASH_B),
_step(3, HASH_B),
_step(4, HASH_B),
_step(5, HASH_A), # ...but a recent (last-3) match silences the hint
_step(6, HASH_B),
]
assert await _run(steps) == []
@pytest.mark.asyncio
async def test_silent_without_any_match():
steps = [_step(n, HASH_B) for n in range(1, 7)]
assert await _run(steps) == []
@pytest.mark.asyncio
async def test_silent_when_config_disabled():
steps = [
_step(1, HASH_A),
_step(2, HASH_B),
_step(3, HASH_B),
_step(4, HASH_B),
_step(5, HASH_B),
_step(6, HASH_B),
]
assert await _run(steps, cfg=_cfg(similarity_hint=False)) == []
@pytest.mark.asyncio
async def test_silent_with_too_few_steps():
# Everything within the recent window belongs to the pixel-level note.
assert await _run([_step(1, HASH_A), _step(2, HASH_A)]) == []
@pytest.mark.asyncio
async def test_tie_prefers_most_recent_old_step():
steps = [
_step(1, HASH_A),
_step(2, HASH_A), # same distance, more recent
_step(3, HASH_B),
_step(4, HASH_B),
_step(5, HASH_B),
_step(6, HASH_B),
]
parts = await _run(steps)
assert len(parts) == 1 and "Step 2" in parts[0]
@pytest.mark.asyncio
async def test_steps_without_hashes_are_skipped():
steps = [
_step(1, None),
_step(2, None),
_step(3, None),
_step(4, None),
_step(5, None),
_step(6, None),
]
assert await _run(steps) == []
def test_record_step_stamps_perceptual_hashes(tmp_path):
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
mock_ctx = MagicMock(spec=ArtemisContext)
setup = MagicMock()
setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = setup
mock_ctx.device = None
engine = DataEngine(mock_ctx)
engine.start_session("dhash stamping test")
engine.record_step(
pre_screenshot_bytes=SCREEN_A,
post_screenshot_bytes=SCREEN_B,
summary="two distinct screens",
)
# Same pre/post bytes: post_image_name is nulled but the post-action
# screen IS the pre screen, so the post hash mirrors the pre hash.
engine.record_step(
pre_screenshot_bytes=SCREEN_A,
post_screenshot_bytes=SCREEN_A,
summary="screen unchanged",
)
for t in list(engine._pending_threads):
t.join()
steps = engine.storage.get_steps(engine.current_session_id)
meta1 = steps[0].extra_metadata
assert meta1["pre_image_dhash"] == HASH_A
assert meta1["post_image_dhash"] == HASH_B
meta2 = steps[1].extra_metadata
assert meta2["pre_image_dhash"] == HASH_A
assert meta2["post_image_dhash"] == HASH_A
+8 -8
View File
@@ -344,7 +344,7 @@ async def test_history_analyzer_integration_with_task_tree():
mock_path.exists.return_value = True
mock_path.read_text.return_value = fake_plan
# Mock build_plan_and_history
# Mock the policy-table compiled-history renderer (M4)
with (
patch(
"artemis.agents.history_analyzer.history_analyzer.get_llm",
@@ -355,23 +355,23 @@ async def test_history_analyzer_integration_with_task_tree():
return_value=mock_path,
),
patch(
"artemis.agents.history_analyzer.history_analyzer.build_plan_and_history"
"artemis.agents.history_analyzer.history_analyzer.build_history_for"
) as mock_build,
):
mock_build.return_value = "Mocked operation history"
analyzer = HistoryAnalyzer(mock_ctx)
await analyzer.run("query")
# Verify build_plan_and_history call args
# Verify build_history_for call args
mock_build.assert_called_once()
args, kwargs = mock_build.call_args
assert args[0] == fake_plan
assert args[1] == steps
assert args[0] == "history_analyzer"
assert args[1] == fake_plan
assert args[2] == steps
import hashlib
expected_hash = hashlib.md5(b"Active Subgoal").hexdigest()
assert args[2] == expected_hash
assert args[3] == expected_hash
assert kwargs.get("last_n_detailed") == 1
assert kwargs.get("min_summaries") == len(steps)
assert kwargs.get("engine") is mock_ctx.data_engine
+84 -20
View File
@@ -16,9 +16,15 @@ import json
from unittest.mock import AsyncMock, MagicMock, Mock, patch
from artemis.agents.operator.operator import OperatorNode
from artemis.config.agent import MemoryTranscriptConfig
from artemis.context import ArtemisContext
import pytest
# These tests exercise the legacy 2-message prompt path. Since M5 the
# transcript path is the default, so flag-off semantics must be requested
# explicitly (never via the config default).
LEGACY_TRANSCRIPT = MemoryTranscriptConfig(enabled=False)
@pytest.mark.asyncio
async def test_operator_node_fast_path():
@@ -58,7 +64,7 @@ async def test_operator_node_fast_path():
# Patch dependencies
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
# Verify state was updated with action
@@ -77,7 +83,7 @@ async def test_perform_action_validation():
from artemis.graph.state import State
mock_ctx = MagicMock()
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
mock_state = MagicMock(spec=State)
mock_state.indexed_points = [[200, 300], [400, 500]]
@@ -114,6 +120,7 @@ async def test_perform_action_validation():
"target_bounds": "[100,100][300,500]",
"target_resource_id": "btn_1",
"target_class": "android.widget.Button",
"target_label_source": "index",
}
]
@@ -143,6 +150,7 @@ async def test_perform_action_validation():
"target_bounds": "[300,400][500,600]",
"target_resource_id": "input_1",
"target_class": "android.widget.EditText",
"target_label_source": "index",
}
# Test swipe direction
@@ -170,6 +178,7 @@ async def test_perform_action_validation():
"target_bounds": "[300,400][500,600]",
"target_resource_id": "input_1",
"target_class": "android.widget.EditText",
"target_label_source": "index",
}
]
@@ -222,6 +231,59 @@ async def test_perform_action_validation():
assert "Unsupported app management action" in err
@pytest.mark.asyncio
async def test_bare_coordinate_click_enriched_by_hit_test():
"""A coordinate click hit tests the pre-action frame for element semantics."""
from artemis.agents.operator.operator import OperatorNode
from unittest.mock import MagicMock
from artemis.graph.state import State
mock_ctx = MagicMock()
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
mock_state = MagicMock(spec=State)
mock_state.indexed_elements = [
{
"center": [550, 1440],
"text": "Confirm",
"bounds": [500, 1400, 600, 1480],
"class": "android.widget.Button",
"resource_id": "btn_confirm",
"is_ocr": False,
}
]
# Normalized (500, 600) on the default 1080x2400 device -> pixel (540, 1440),
# which falls inside the "Confirm" button bounds.
actions, err = node._translate_and_validate_tool(
{"name": "click", "args": {"target": [500, 600]}}, mock_state
)
assert err is None
assert actions == [
{
"action": "tap",
"coordinates": [540, 1440],
"normalized_coordinates": [500, 600],
"times": 1,
"delay_ms": 100,
"target_text": "Confirm",
"target_bounds": [500, 1400, 600, 1480],
"target_resource_id": "btn_confirm",
"target_class": "android.widget.Button",
"target_label_source": "hit_test",
}
]
# Graceful degradation: no perception data at all -> bare coordinates, "none".
mock_state.indexed_elements = []
actions, err = node._translate_and_validate_tool(
{"name": "click", "args": {"target": [500, 600]}}, mock_state
)
assert err is None
assert actions[0]["target_text"] is None
assert actions[0]["target_label_source"] == "none"
@pytest.mark.asyncio
async def test_operator_node_multiple_actions():
from artemis.agents.operator.operator import OperatorNode
@@ -258,7 +320,7 @@ async def test_operator_node_multiple_actions():
mock_llm.bind_tools.return_value = mock_llm
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
update_dict = node_update
@@ -302,7 +364,7 @@ async def test_operator_node_no_record_step():
mock_llm.bind_tools.return_value = mock_llm
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
mock_ctx.data_engine.record_step.assert_not_called()
@@ -352,7 +414,7 @@ async def test_operator_dynamic_prompt():
mock_llm.bind_tools.return_value = mock_llm
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx, prompt_components=[MockComponent()])
node = OperatorNode(mock_ctx, prompt_components=[MockComponent()], transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
@@ -488,7 +550,7 @@ async def test_operator_history_compression():
patch("pathlib.Path.exists", return_value=True),
patch("pathlib.Path.read_text", mock_read_text),
):
node = OperatorNode(mock_ctx, last_n_detailed=5)
node = OperatorNode(mock_ctx, last_n_detailed=5, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
@@ -506,7 +568,7 @@ async def test_operator_loop_detection():
mock_state.subagent_calls = ["ask_explorer", "ask_explorer", "ask_explorer"]
mock_state.initial_goal = "Test goal"
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
with pytest.raises(RuntimeError) as exc_info:
node_update = await node(mock_state)
@@ -568,7 +630,7 @@ async def test_operator_tracks_subagent_calls():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool])
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
update_dict = node_update
@@ -660,7 +722,7 @@ async def test_operator_defer_action_for_gathering():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool])
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
# Subagent tool should have been executed
@@ -683,7 +745,7 @@ async def test_wait_actions_translation():
from artemis.graph.state import State
mock_ctx = MagicMock()
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
mock_state = MagicMock(spec=State)
# 1. Test wait_for_delay translation
@@ -708,7 +770,7 @@ async def test_long_press_action_translation():
from artemis.graph.state import State
mock_ctx = MagicMock()
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
mock_state = MagicMock(spec=State)
mock_state.indexed_points = [[200, 300], [400, 500]]
mock_state.indexed_elements = [
@@ -744,6 +806,7 @@ async def test_long_press_action_translation():
"target_bounds": "[100,100][300,500]",
"target_resource_id": "btn_1",
"target_class": "android.widget.Button",
"target_label_source": "index",
}
]
@@ -765,6 +828,7 @@ async def test_long_press_action_translation():
"target_bounds": None,
"target_resource_id": None,
"target_class": None,
"target_label_source": "none",
}
]
@@ -844,7 +908,7 @@ async def test_operator_no_defer_for_note_updating():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_update_note_tool])
node = OperatorNode(mock_ctx, tools=[mock_update_note_tool], transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
# The update_note tool should have been executed immediately
@@ -934,7 +998,7 @@ async def test_operator_defer_for_note_reading():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_read_note_tool])
node = OperatorNode(mock_ctx, tools=[mock_read_note_tool], transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
# The read_note tool should have been executed
@@ -1022,7 +1086,7 @@ async def test_operator_no_defer_for_note_appending():
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[mock_append_note_tool])
node = OperatorNode(mock_ctx, tools=[mock_append_note_tool], transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
# The append_note tool should have been executed immediately
@@ -1111,7 +1175,7 @@ async def test_operator_background_tasks_prompt_injection():
side_effect=mock_invoke_llm_loop,
),
):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
# Assert that the captured messages (prompts) contain the background tasks section
@@ -1202,7 +1266,7 @@ async def test_operator_task_plan_warning_prompt_injection():
side_effect=mock_invoke_llm_loop_1,
),
):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
full_content_1 = ""
@@ -1255,7 +1319,7 @@ async def test_operator_task_plan_warning_prompt_injection():
side_effect=mock_invoke_llm_loop_2,
),
):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
full_content_2 = ""
@@ -1322,7 +1386,7 @@ async def test_operator_tool_limit_exceeded_warning():
side_effect=mock_invoke_llm_loop,
),
):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
full_content = ""
@@ -1380,7 +1444,7 @@ async def test_operator_fallback_function_call_parsing():
mock_llm.bind_tools.return_value = mock_llm
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
# Verify state was updated with parsed click action
@@ -1402,7 +1466,7 @@ async def test_operator_swipe_translation():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_state = MagicMock()
mock_state.operator_raw_data = {"width": 1080, "height": 2400}
node = OperatorNode(mock_ctx)
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
# 1. Directional swipe
actions, err = node._translate_and_validate_tool(
+1 -1
View File
@@ -24,7 +24,7 @@ def test_operator_prompt_contract_matches_note_runtime_semantics():
assert "write-through tools `update_note` and `append_note`" in prompt
assert "may accompany at most one Turn-Ending Action" in prompt
assert "`read_note`, `list_notes`, and `save_note`" in prompt
assert "`read_note`, `list_notes`, `save_note`, and `recall_history`" in prompt
assert "Do NOT submit a Turn-Ending Action at the same time" not in prompt
assert "memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`" not in prompt
@@ -0,0 +1,311 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Operator dual prompt path (M2): legacy golden parity and transcript mode.
Flag off (``agent.memory.transcript.enabled=false``): the legacy 2-message
build must stay byte-for-byte identical — pinned by SHA-256 goldens recorded
against the pre-split implementation.
Flag on: ``_build_prompt`` renders ``S + F + A + tail`` from the session
transcript ledger; the turn's messages are staged after the tool loop and
committed (with step key and validator result) at the next build.
"""
import base64
import hashlib
import re
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from langchain_core.messages import HumanMessage, SystemMessage
from artemis.agents.operator.operator import OperatorNode
from artemis.agents.operator.prompt_builder import load_operator_prompts
from artemis.agents.operator.prompts import (
PLAN_HISTORY_STATIC_POINTER,
PLAN_HISTORY_TEMPLATE_SECTION,
PromptBuilder,
TemplatePromptComponent,
render_transcript_static_system,
)
from artemis.config.agent import MemoryTranscriptConfig
from artemis.context import ArtemisContext
from artemis.memory.step_memory import StepMemoryService
from artemis.memory.transcript import (
EXECUTION_RESULT_MARKER,
PLAN_RECITATION_MARKER,
RESTORED_HISTORY_HEADER,
TranscriptLedger,
)
# SHA-256 of the legacy system message rendered by TemplatePromptComponent
# with the fixed inputs below. Originally recorded BEFORE the M2 template
# split; re-recorded once in M5 (2026-09-01) after the only intended template
# change since — removing the Short-Term Memory instruction block from
# operator.json (the <short_term_memory> relay was removed end to end per
# pro-context-memory-redesign §3.3). Any other drift here means the flag-off
# path is no longer byte-identical.
GOLDEN_EMPTY_PLAN = "c45aec7d67069dad3a195534dd3c97dadb48454ca186a40eb471d5f9136589fa"
GOLDEN_SENTINEL_PLAN = "670580efd48f237c3257d996a814145fbef4ec0bb467aa4dfa6eb2db4a613826"
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
async def _render_legacy_system(plan_and_history: str) -> str:
builder = PromptBuilder()
component = TemplatePromptComponent()
ctx = SimpleNamespace(execution_setup=None, actuator=None, data_engine=None)
state = SimpleNamespace(initial_goal="GOLDEN_FIXED_GOAL")
await component(
builder,
state,
ctx,
prompts=load_operator_prompts(),
template_name="main_template",
plan_and_history=plan_and_history,
)
return "".join(builder.system_parts)
@pytest.mark.asyncio
async def test_legacy_prompt_bytes_match_pre_split_golden():
empty = await _render_legacy_system("No plan or history yet.")
sentinel = await _render_legacy_system("PLAN_AND_HISTORY_SENTINEL\nline2")
assert hashlib.sha256(empty.encode("utf-8")).hexdigest() == GOLDEN_EMPTY_PLAN
assert hashlib.sha256(sentinel.encode("utf-8")).hexdigest() == GOLDEN_SENTINEL_PLAN
def test_main_template_contains_exactly_one_plan_history_section():
template = load_operator_prompts()["main_template"]
assert template.count(PLAN_HISTORY_TEMPLATE_SECTION) == 1
def test_transcript_static_system_is_stable_and_carries_no_history():
ctx = SimpleNamespace(execution_setup=None, actuator=None, data_engine=None)
state = SimpleNamespace(initial_goal="STATIC_GOAL")
prompts = load_operator_prompts()
first = render_transcript_static_system(prompts, ctx, state)
second = render_transcript_static_system(prompts, ctx, state)
assert first == second # byte-stable S region
assert "STATIC_GOAL" in first
# The volatile section is replaced by the static pointer...
assert PLAN_HISTORY_STATIC_POINTER.splitlines()[1] in first
# ...and no unrendered slot or trailing observation header survives.
assert "{{" not in first
assert not first.endswith("# CURRENT OBSERVATION\n")
def _transcript_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.execution_setup = None
ctx.actuator = None
ctx.data_engine = None
ctx.step_memory = StepMemoryService(ctx=None)
ctx.transcript_ledger = None
return ctx
def _transcript_state(**overrides):
state = MagicMock()
state.subagent_calls = []
state.initial_goal = "Transcript goal"
state.operator_feedback = None
state.injected_instruction = None
state.operator_tool_limit_exceeded = False
state.structured_decisions = None
state.last_execution_result = None
state.current_step_id = None
state.latest_ui_hierarchy = []
state.operator_raw_data = {
"screenshot_b64": SCREENSHOT_B64,
"xml_hierarchy": [],
"ocr_results": None,
"width": 1080,
"height": 2400,
}
for key, value in overrides.items():
setattr(state, key, value)
return state
def _no_action_llm(captured: list):
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.tool_calls = []
mock_response.content = "no action this turn"
async def mock_ainvoke(*args, **kwargs):
captured.append(list(args[0]))
return mock_response
mock_llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
mock_llm.bind_tools.return_value = mock_llm
return mock_llm
@pytest.mark.asyncio
async def test_transcript_mode_two_turns_build_four_regions(tmp_path):
ctx = _transcript_ctx()
# Regression data shape (M2 cold-start bug, on-device A/B 2026-09-01):
# turn 1 runs before any step record exists; by turn 2 the first step HAS
# been recorded, so `steps` is non-empty while the previous turn is still
# staged (turn_count == 0). The cold-start check must not read that
# live-session state as a process restart — seeding would be rejected by
# the empty-ledger invariant and silently demote every turn to the legacy
# 2-message path.
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
ctx.data_engine.global_base_dir = str(tmp_path)
ctx.data_engine.get_agent_friendly_steps.return_value = []
captured: list = []
mock_llm = _no_action_llm(captured)
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
# Turn 1
await node(_transcript_state())
ledger = ctx.transcript_ledger
assert isinstance(ledger, TranscriptLedger)
assert ledger.has_static_prefix
assert ledger.has_staged_turn
assert ledger.turn_count == 0
turn1_messages = captured[0]
assert isinstance(turn1_messages[0], SystemMessage)
assert isinstance(turn1_messages[-1], HumanMessage)
tail1_text = str(turn1_messages[-1].content)
assert "# CURRENT OBSERVATION [T+" in tail1_text
assert PLAN_RECITATION_MARKER in tail1_text
# Turn 2: step 1 is now recorded, and the previous turn is committed
# with its step id and result.
ctx.data_engine.get_agent_friendly_steps.return_value = [
{
"step_id": "step-1",
"step_number": 1,
"summary": "Step 1 summary",
"relative_time": "1.0s",
"extra_metadata": {"subgoal_hash": "default"},
}
]
await node(
_transcript_state(
current_step_id="step-1",
structured_decisions='[{"action": "tap"}]',
last_execution_result={"status": "success"},
)
)
# The staged-but-uncommitted turn 2 state never triggers cold-start
# seeding (the transcript path stayed live instead of falling back).
assert not ledger.has_restored_history
assert ledger.turn_count == 1
turn2_messages = captured[1]
# S region is the same object with the same bytes.
assert turn2_messages[0] is turn1_messages[0]
active = list(ledger.active_messages)
# Committed turn-1 observation followed by its validator result.
assert any(EXECUTION_RESULT_MARKER in str(m.content) for m in active)
committed_obs = active[0]
# Depth-1 scrub removed the old plan recitation copy.
assert PLAN_RECITATION_MARKER not in str(committed_obs.content)
# The live tail still recites the plan.
assert PLAN_RECITATION_MARKER in str(turn2_messages[-1].content)
# Message layout is S + A + tail.
assert turn2_messages[-1] is not turn1_messages[-1]
assert re.search(r"T\+\d{2,}:\d{2}", str(turn2_messages[-1].content))
@pytest.mark.asyncio
async def test_transcript_actionless_turn_commits_without_validator_message():
ctx = _transcript_ctx()
captured: list = []
mock_llm = _no_action_llm(captured)
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
await node(_transcript_state())
# Planner-rejected turns clear structured_decisions: no validator ran.
await node(
_transcript_state(
current_step_id="step-1",
structured_decisions="",
last_execution_result={"status": "success"},
)
)
ledger = ctx.transcript_ledger
assert ledger.turn_count == 1
assert not any(
EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages
)
@pytest.mark.asyncio
async def test_transcript_cold_start_builds_restored_history_block(tmp_path):
ctx = _transcript_ctx()
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
ctx.data_engine.global_base_dir = str(tmp_path)
ctx.data_engine.get_agent_friendly_steps.return_value = [
{
"step_id": f"step_{i}",
"step_number": i,
"summary": f"Step {i} summary",
"relative_time": f"{i}.0s",
"extra_metadata": {"subgoal_hash": "default"},
}
for i in range(1, 4)
]
captured: list = []
mock_llm = _no_action_llm(captured)
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
await node(_transcript_state())
ledger = ctx.transcript_ledger
assert ledger.has_restored_history
messages = captured[0]
restored_text = str(messages[1].content)
assert RESTORED_HISTORY_HEADER in restored_text
assert "--- Execution History ---" in restored_text
assert "Step 1 summary" in restored_text
# The live tail still follows normally and A stays appendable.
assert "# CURRENT OBSERVATION [T+" in str(messages[-1].content)
assert ledger.has_staged_turn
@pytest.mark.asyncio
async def test_flag_off_keeps_legacy_two_message_build_and_no_ledger():
ctx = _transcript_ctx()
captured: list = []
mock_llm = _no_action_llm(captured)
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=False))
await node(_transcript_state())
assert len(captured[0]) == 2
assert isinstance(captured[0][0], SystemMessage)
assert isinstance(captured[0][1], HumanMessage)
assert ctx.transcript_ledger is None
+3
View File
@@ -286,6 +286,9 @@ async def test_outputter_builds_concise_history(mock_get_llm):
}
]
mock_data_engine.get_agent_friendly_steps.return_value = steps
# M5: the transcript flag defaults on, so the outputter's compiled view
# consults the chunk store; no persisted chunks keeps the classic path.
mock_data_engine.get_history_chunks.return_value = []
mock_data_engine.base_dir = "/tmp/mock_session"
ctx.data_engine = mock_data_engine
+3 -3
View File
@@ -92,7 +92,7 @@ def test_full_toolset_is_the_default():
)
@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer",))
@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer", "recall_history"))
def test_removed_tool_leaves_no_reference(tool):
"""The executable definition of 'an absent tool costs the model nothing'."""
for name, template in load_operator_prompts().items():
@@ -154,8 +154,8 @@ def test_full_set_enum_slots_render_verbatim():
)
assert (
"(`ask_explorer`, `ask_diagnoser`, `video_analyzer`, `run_adb_command`,"
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`, and"
" `save_note`)" in out
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`,"
" `save_note`, and `recall_history`)" in out
)
+84 -148
View File
@@ -12,11 +12,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import MagicMock, Mock, patch
"""SummarizerNode tests (M2: visual-transition lens dispatch).
Per the history redesign §5 the Pro SummarizerNode no longer runs its own LLM
call to produce a text capsule: it dispatches the step to the shared
step-memory service (``ctx.step_memory``) with the pre/post screenshot bytes
pulled from the DataEngine, and degrades gracefully when images are missing.
"""
from types import SimpleNamespace
from unittest.mock import MagicMock, Mock
import pytest
from artemis.agents.summarizer.summarizer import SummarizerNode
from artemis.context import ArtemisContext
import pytest
class DummyState:
@@ -36,181 +46,107 @@ class DummyState:
@pytest.fixture
def mock_context():
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.device = None
ctx.data_engine = Mock()
ctx.background_tasks = []
ctx.step_memory = MagicMock()
pre_path = tmp_path / "pre.jpg"
post_path = tmp_path / "post.jpg"
pre_path.write_bytes(b"PRE_IMAGE")
post_path.write_bytes(b"POST_IMAGE")
engine = Mock()
engine.get_step_number.return_value = 7
engine.get_step_record.return_value = SimpleNamespace(
action_taken=[{"action": "tap", "coordinates": [500, 600], "target_text": "Search"}]
)
engine.get_step_image_path.side_effect = lambda step_number, which="pre": (
pre_path if which == "pre" else post_path
)
ctx.data_engine = engine
return ctx
@pytest.mark.asyncio
async def test_summarizer_success(mock_context):
"""Test that SummarizerNode successfully generates summary from raw thinking and execution result."""
decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}])
thinking = "The operator clicked the search button."
result = {
"status": "success",
"executed_actions": [{"action": "tap", "coordinates": [500, 600]}],
}
async def test_summarizer_dispatches_visual_lens(mock_context):
"""The node dispatches to the shared service instead of calling an LLM."""
state = DummyState(
structured_decisions=decisions,
operator_raw_thinking=thinking,
last_execution_result=result,
last_execution_result={"status": "success"},
current_step_id="12345678-1234-5678-1234-567812345678",
)
# Mock LLM response
mock_response = Mock()
mock_response.content = "Step completed successfully."
mock_llm = MagicMock()
node = SummarizerNode(mock_context)
result = await node(state)
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
with (
patch(
"artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm
) as mock_get_llm,
patch("asyncio.create_task") as mock_create_task,
):
# Run the node call
node = SummarizerNode(mock_context)
await node(state)
# Verify task was registered in context
assert len(mock_context.background_tasks) == 1
assert mock_context.background_tasks[0] == mock_create_task.return_value
# Verify background task was spawned
assert mock_create_task.called
background_coroutine = mock_create_task.call_args[0][0]
# Await the background task synchronously for test verification
await background_coroutine
# Verify LLM was invoked with proper structured messages
assert mock_llm.astream.called
invoked_messages = mock_llm.astream.call_args[0][0]
system_msg = invoked_messages[0].content
human_msg = invoked_messages[1].content
assert "Summarizer Agent" in system_msg
assert "Decisions made" in human_msg
assert "The operator clicked the search button." in human_msg
# Verify Data Engine updated the step summary
from uuid import UUID
mock_context.data_engine.update_step_summary.assert_called_once_with(
UUID("12345678-1234-5678-1234-567812345678"),
"Step completed successfully.",
assert result == {}
mock_context.step_memory.dispatch.assert_called_once_with(
step_number=7,
action_name="tap",
action_args={"coordinates": [500, 600], "target_text": "Search"},
pre_img_bytes=b"PRE_IMAGE",
post_img_bytes=b"POST_IMAGE",
exec_outcome="success",
data_engine_step_id="12345678-1234-5678-1234-567812345678",
)
# No direct capsule write from the node: the lens owns the versioned
# summary write when its background job completes.
mock_context.data_engine.update_step_summary.assert_not_called()
@pytest.mark.asyncio
async def test_summarizer_with_native_thinking(mock_context):
"""Test that SummarizerNode successfully generates summary from both explicit and native thoughts."""
decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}])
raw_thinking = "Explicit monologue."
native_thinking = "Silent core reasoning."
result = {"status": "success"}
async def test_summarizer_degrades_gracefully_without_images(mock_context):
"""Missing DataEngine screenshots dispatch with None bytes, no exception."""
mock_context.data_engine.get_step_image_path.side_effect = None
mock_context.data_engine.get_step_image_path.return_value = None
state = DummyState(
structured_decisions=decisions,
operator_raw_thinking=raw_thinking,
operator_native_thinking=native_thinking,
last_execution_result=result,
last_execution_result={"status": "failed", "error": "tap missed"},
current_step_id="12345678-1234-5678-1234-567812345678",
)
mock_response = Mock()
mock_response.content = "Summary result."
mock_llm = MagicMock()
node = SummarizerNode(mock_context)
await node(state)
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
with (
patch(
"artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm
) as mock_get_llm,
patch("asyncio.create_task") as mock_create_task,
):
node = SummarizerNode(mock_context)
await node(state)
# Verify task was registered in context
assert len(mock_context.background_tasks) == 1
assert mock_context.background_tasks[0] == mock_create_task.return_value
background_coroutine = mock_create_task.call_args[0][0]
await background_coroutine
assert mock_llm.astream.called
invoked_messages = mock_llm.astream.call_args[0][0]
human_msg = invoked_messages[1].content
assert "Operator explicit thoughts:" in human_msg
assert "Explicit monologue." in human_msg
assert "Operator native thoughts:" in human_msg
assert "Silent core reasoning." in human_msg
kwargs = mock_context.step_memory.dispatch.call_args.kwargs
assert kwargs["pre_img_bytes"] is None
assert kwargs["post_img_bytes"] is None
assert kwargs["exec_outcome"] == "Error: tap missed"
@pytest.mark.asyncio
async def test_summarizer_with_chronological_trace(mock_context):
"""Test that SummarizerNode generates a prompt containing the chronological step trace when available."""
step_id = "12345678-1234-5678-1234-567812345678"
async def test_summarizer_falls_back_to_structured_decisions(mock_context):
"""Without a step record the action comes from structured_decisions, and
extra actions ride along as additional_actions."""
mock_context.data_engine.get_step_record.return_value = None
state = DummyState(
current_step_id=step_id,
structured_decisions=(
'[{"action": "swipe", "coordinates": [1, 2, 3, 4]},'
' {"action": "tap", "coordinates": [9, 9]}]'
),
last_execution_result={"status": "success"},
current_step_id="12345678-1234-5678-1234-567812345678",
)
mock_step_data = {
"step_id": step_id,
"step_number": 1,
"relative_time": "5.0s",
"action_taken": [{"action": "click", "target_text": "Search"}],
"last_execution_result": {"status": "success"},
"interleaved_events": [
{"type": "thought", "content": "Let's click search."},
],
}
node = SummarizerNode(mock_context)
await node(state)
mock_context.data_engine.get_agent_friendly_steps.return_value = [mock_step_data]
kwargs = mock_context.step_memory.dispatch.call_args.kwargs
assert kwargs["action_name"] == "swipe"
assert kwargs["action_args"]["coordinates"] == [1, 2, 3, 4]
assert kwargs["action_args"]["additional_actions"] == [
{"action": "tap", "coordinates": [9, 9]}
]
# Mock LLM response
mock_response = Mock()
mock_response.content = "Summary using chronological trace."
mock_llm = MagicMock()
async def mock_astream(*args, **kwargs):
yield mock_response
@pytest.mark.asyncio
async def test_summarizer_noop_without_step_id(mock_context):
state = DummyState(current_step_id=None)
mock_llm.astream.side_effect = mock_astream
node = SummarizerNode(mock_context)
result = await node(state)
with (
patch("artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm),
patch("asyncio.create_task") as mock_create_task,
):
node = SummarizerNode(mock_context)
await node(state)
# Verify task was registered in context
assert len(mock_context.background_tasks) == 1
assert mock_context.background_tasks[0] == mock_create_task.return_value
background_coroutine = mock_create_task.call_args[0][0]
await background_coroutine
assert mock_llm.astream.called
invoked_messages = mock_llm.astream.call_args[0][0]
human_msg = invoked_messages[1].content
assert "--- Execution History ---" in human_msg
assert "Let's click search." in human_msg
assert result == {}
mock_context.step_memory.dispatch.assert_not_called()
+59
View File
@@ -380,3 +380,62 @@ def test_update_step_summary_includes_step_number_in_sse(tmp_path):
assert last_update["step_number"] == 1
def _foreground_engine(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
engine = DataEngine(mock_ctx)
engine.start_session("foreground_app persistence")
return engine
def test_record_step_persists_explicit_foreground_app(tmp_path):
"""M5: the historically dropped foreground_app parameter is persisted."""
engine = _foreground_engine(tmp_path)
engine.record_step(
foreground_app="com.android.settings",
ui_tree=[{"package": "com.other.app", "bounds": "[0,0][10,10]"}],
summary="Explicit parameter wins",
)
for t in list(engine._pending_threads):
t.join()
step = engine.storage.get_steps(engine.current_session_id)[0]
assert step.extra_metadata["foreground_app"] == "com.android.settings"
def test_record_step_derives_foreground_app_from_ui_tree(tmp_path):
"""Without the parameter, the dominant non-overlay package is stamped."""
engine = _foreground_engine(tmp_path)
engine.record_step(
ui_tree=[
{"package": "com.android.systemui", "bounds": "[0,0][10,10]"},
{"package": "com.google.android.deskclock"},
{
"packageName": "com.google.android.deskclock",
"children": [{"package": "com.google.android.deskclock"}],
},
],
summary="Derived from UI tree",
)
for t in list(engine._pending_threads):
t.join()
step = engine.storage.get_steps(engine.current_session_id)[0]
assert step.extra_metadata["foreground_app"] == "com.google.android.deskclock"
def test_record_step_without_app_data_stamps_nothing(tmp_path):
engine = _foreground_engine(tmp_path)
engine.record_step(
ui_tree=[{"package": "com.android.systemui"}],
summary="Only overlay packages",
)
for t in list(engine._pending_threads):
t.join()
step = engine.storage.get_steps(engine.current_session_id)[0]
assert "foreground_app" not in (step.extra_metadata or {})
@@ -0,0 +1,105 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""History-chunk storage (M3 §6.1): append-only versioned rows keyed by step range."""
from uuid import uuid4
import pytest
from artemis.data_engine.models import HistoryChunkRecord
from artemis.data_engine.storage import StorageManager
SESSION = "11111111-1111-1111-1111-111111111111"
@pytest.fixture
def storage(tmp_path):
return StorageManager(tmp_path / "test.db", tmp_path / "traces")
def _chunk(start: int, end: int, version: int = 1, **overrides) -> HistoryChunkRecord:
fields = {
"chunk_id": uuid4(),
"session_id": SESSION,
"start_step_id": f"sid-{start}",
"end_step_id": f"sid-{end}",
"start_step_number": start,
"end_step_number": end,
"source_step_ids": [f"sid-{n}" for n in range(start, end + 1)],
"subgoal_hash": "hash-a",
"version": version,
"status": "pending",
"band1": {},
"band2": None,
"band3": f"- Step {start} (T+00:10): Tapped 'x' at [1, 2] -> executed",
"rendered_text": "[Chunk 1 | ...]",
}
fields.update(overrides)
return HistoryChunkRecord(**fields)
def test_history_chunk_roundtrip(storage):
storage.create_history_chunk(_chunk(1, 4))
chunks = storage.get_history_chunks(SESSION)
assert len(chunks) == 1
rec = chunks[0]
assert (rec.start_step_number, rec.end_step_number) == (1, 4)
assert rec.source_step_ids == [f"sid-{n}" for n in range(1, 5)]
assert rec.subgoal_hash == "hash-a"
assert rec.status == "pending"
assert "T+00:10" in rec.band3
def test_history_chunk_versions_are_append_only_latest_wins(storage):
storage.create_history_chunk(_chunk(1, 4, version=1, status="pending"))
storage.create_history_chunk(
_chunk(
1,
4,
version=2,
status="ready",
band1={"doing": "d", "verified_facts": ["f1"]},
band2=" - Steps 1–4: did things",
)
)
latest = storage.get_history_chunks(SESSION)
assert len(latest) == 1
assert latest[0].version == 2
assert latest[0].status == "ready"
assert latest[0].band1["verified_facts"] == ["f1"]
trail = storage.get_history_chunks(SESSION, all_versions=True)
assert [r.version for r in trail] == [1, 2]
def test_history_chunks_ordered_by_step_range(storage):
storage.create_history_chunk(_chunk(9, 12))
storage.create_history_chunk(_chunk(1, 4))
storage.create_history_chunk(_chunk(5, 8))
latest = storage.get_history_chunks(SESSION)
assert [(r.start_step_number, r.end_step_number) for r in latest] == [
(1, 4),
(5, 8),
(9, 12),
]
def test_history_chunks_are_session_scoped(storage):
storage.create_history_chunk(_chunk(1, 4))
other = "22222222-2222-2222-2222-222222222222"
storage.create_history_chunk(_chunk(1, 4, session_id=other))
assert len(storage.get_history_chunks(SESSION)) == 1
assert len(storage.get_history_chunks(other)) == 1
@@ -0,0 +1,171 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for the versioned, status-carrying step summary write (M0 §6.1)."""
from unittest.mock import MagicMock
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
def _make_engine(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
engine = DataEngine(mock_ctx)
engine.start_session("summary versioning test")
return engine
def _flush(engine):
for t in list(engine._pending_threads):
t.join()
def _get_step(engine, step_id):
steps = engine.storage.get_steps(engine.current_session_id)
for step in steps:
if str(step.step_id) == str(step_id):
return step
return None
def test_ready_write_persists_summary_and_metadata(tmp_path):
engine = _make_engine(tmp_path)
step_id = engine.record_step(summary="initial")
_flush(engine)
applied = engine.storage.update_step_summary(
step_id,
"Tapped the login button.",
source="visual_transition",
model="gemini-2.5-flash-lite",
status="ready",
)
assert applied is True
step = _get_step(engine, step_id)
assert step.summary == "Tapped the login button."
assert step.extra_metadata["summary_status"] == "ready"
assert step.extra_metadata["summary_source"] == "visual_transition"
assert step.extra_metadata["summary_version"] == 1
assert step.extra_metadata["summary_model"] == "gemini-2.5-flash-lite"
def test_auto_version_increments_per_write(tmp_path):
engine = _make_engine(tmp_path)
step_id = engine.record_step(summary="initial")
_flush(engine)
engine.storage.update_step_summary(step_id, "v1", status="ready")
engine.storage.update_step_summary(step_id, "v2", status="ready")
step = _get_step(engine, step_id)
assert step.summary == "v2"
assert step.extra_metadata["summary_version"] == 2
def test_stale_explicit_version_is_dropped(tmp_path):
engine = _make_engine(tmp_path)
step_id = engine.record_step(summary="initial")
_flush(engine)
assert engine.storage.update_step_summary(step_id, "newer", version=5, status="ready")
applied = engine.storage.update_step_summary(step_id, "older", version=3, status="ready")
assert applied is False
step = _get_step(engine, step_id)
assert step.summary == "newer"
assert step.extra_metadata["summary_version"] == 5
def test_pending_does_not_downgrade_ready(tmp_path):
engine = _make_engine(tmp_path)
step_id = engine.record_step(summary="initial")
_flush(engine)
engine.storage.update_step_summary(step_id, "done", status="ready")
applied = engine.storage.update_step_summary(step_id, None, status="pending")
assert applied is False
step = _get_step(engine, step_id)
assert step.summary == "done"
assert step.extra_metadata["summary_status"] == "ready"
def test_failed_marks_status_without_touching_summary(tmp_path):
engine = _make_engine(tmp_path)
step_id = engine.record_step(summary="original text")
_flush(engine)
engine.storage.update_step_summary(step_id, None, status="pending", source="visual_transition")
applied = engine.storage.update_step_summary(
step_id, None, status="failed", source="visual_transition"
)
assert applied is True
step = _get_step(engine, step_id)
assert step.summary == "original text"
assert step.extra_metadata["summary_status"] == "failed"
# A failed write must never clobber an already-ready summary.
engine.storage.update_step_summary(step_id, "recovered", status="ready")
applied = engine.storage.update_step_summary(step_id, None, status="failed")
assert applied is False
step = _get_step(engine, step_id)
assert step.extra_metadata["summary_status"] == "ready"
def test_missing_step_returns_false(tmp_path):
from uuid import uuid4
engine = _make_engine(tmp_path)
assert engine.storage.update_step_summary(uuid4(), "text", status="ready") is False
def test_engine_level_write_carries_metadata_and_publishes_ready_only(tmp_path):
engine = _make_engine(tmp_path)
published = []
engine.subscribe(lambda event_type, data: published.append((event_type, data)))
step_id = engine.record_step(summary="initial")
_flush(engine)
published.clear()
# Non-ready statuses persist metadata but publish no SSE.
engine.update_step_summary(step_id, None, status="pending", source="visual_transition")
_flush(engine)
assert not [e for e in published if e[0] == "step_updated"]
step = _get_step(engine, step_id)
assert step.extra_metadata["summary_status"] == "pending"
engine.update_step_summary(
step_id,
"Ready text.",
status="ready",
source="visual_transition",
model="test-model",
)
_flush(engine)
ready_events = [e for e in published if e[0] == "step_updated"]
assert len(ready_events) == 1
assert ready_events[0][1]["summary"] == "Ready text."
step = _get_step(engine, step_id)
assert step.summary == "Ready text."
assert step.extra_metadata["summary_status"] == "ready"
assert step.extra_metadata["summary_model"] == "test-model"
@@ -0,0 +1,158 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for record-time target semantics enrichment in McpActionExecutor."""
from unittest.mock import Mock
from artemis.mcp.action_executor import McpActionExecutor
def _make_executor(width=1080, height=2400):
ctx = Mock()
ctx.device.device_width = width
ctx.device.device_height = height
actuator = Mock()
actuator.controller = Mock()
return McpActionExecutor(ctx, actuator=actuator)
def _state_with_elements(elements):
state = Mock()
state.indexed_elements = elements
return state
def test_click_target_semantics_from_pre_action_frame():
executor = _make_executor()
# Normalized (500, 600) on a 1080x2400 device -> pixel (540, 1440).
state = _state_with_elements(
[
{
"text": "Confirm",
"bounds": [500, 1400, 600, 1480],
"class": "android.widget.Button",
"resource_id": "btn_confirm",
"is_ocr": False,
}
]
)
semantics = executor._target_semantics("click", {"target": [500, 600]}, state)
assert semantics == {
"target_label_source": "hit_test",
"target_text": "Confirm",
"target_class": "android.widget.Button",
"target_resource_id": "btn_confirm",
}
def test_semantics_degrade_to_none_without_element_data():
executor = _make_executor()
# Empty perception data
semantics = executor._target_semantics(
"click", {"target": [500, 600]}, _state_with_elements([])
)
assert semantics == {"target_label_source": "none"}
# No state at all
semantics = executor._target_semantics("click", {"target": [500, 600]}, None)
assert semantics == {"target_label_source": "none"}
def test_non_targeted_actions_and_missing_targets_skip_enrichment():
executor = _make_executor()
state = _state_with_elements([{"text": "X", "bounds": [0, 0, 10, 10]}])
assert executor._target_semantics("press_key", {"key": "BACK"}, state) is None
# input_text without a coordinate target (focused field typing)
assert executor._target_semantics("input_text", {"text": "hi", "target": None}, state) is None
# Multi-point actions without resolvable points skip enrichment too.
assert executor._target_semantics("click_sequence", {"sequence": []}, state) is None
assert executor._target_semantics("swipe", {"start": None, "end": None}, state) is None
def test_click_sequence_hit_tests_every_point_first_point_is_main_label():
"""M5: multi-point actions get per-point best-effort semantics; the first
point's fields are hoisted as the action's main label."""
executor = _make_executor()
state = _state_with_elements(
[
{
"text": "Digit 1",
"bounds": [500, 1400, 600, 1480],
"class": "android.widget.Button",
"is_ocr": False,
},
{
"text": "Digit 2",
"bounds": [700, 1400, 800, 1480],
"class": "android.widget.Button",
"is_ocr": False,
},
]
)
# Normalized points -> pixels: (500,600)->(540,1440) hits Digit 1;
# (690,600)->(745,1440) hits Digit 2; (10,10)->(10,24) hits nothing.
semantics = executor._target_semantics(
"click_sequence",
{"sequence": [[500, 600], [690, 600], [10, 10]], "delay_ms": 50},
state,
)
assert semantics["target_text"] == "Digit 1"
assert semantics["target_label_source"] == "hit_test"
per_point = semantics["points_semantics"]
assert len(per_point) == 3
assert per_point[0]["target_text"] == "Digit 1"
assert per_point[1]["target_text"] == "Digit 2"
assert per_point[2] == {"target_label_source": "none"}
def test_swipe_hit_tests_start_and_end_start_is_main_label():
executor = _make_executor()
state = _state_with_elements(
[
{
"text": "Brightness slider",
"bounds": [100, 1400, 900, 1480],
"class": "android.widget.SeekBar",
"is_ocr": False,
}
]
)
semantics = executor._target_semantics(
"swipe", {"start": [200, 600], "end": [800, 600], "duration_ms": 400}, state
)
assert semantics["target_text"] == "Brightness slider"
assert semantics["start_semantics"]["target_text"] == "Brightness slider"
assert semantics["end_semantics"]["target_text"] == "Brightness slider"
assert semantics["target_label_source"] == "hit_test"
def test_long_press_uses_ocr_fallback():
executor = _make_executor()
state = _state_with_elements(
[
{
"text": "OCR Word",
"bounds": [520, 1420, 560, 1460],
"class": None,
"resource_id": None,
"is_ocr": True,
}
]
)
semantics = executor._target_semantics("long_press", {"target": [500, 600]}, state)
assert semantics["target_label_source"] == "ocr"
assert semantics["target_text"] == "OCR Word"
+304
View File
@@ -0,0 +1,304 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""ContextPolicy table (M4): byte-exact parity with the eight historical call
sites, config overrides, and the chunk hand-off to the compiled view."""
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from artemis.memory.context_policy import (
CONTEXT_POLICIES,
DIGEST_LEDGER_MARKER_TEMPLATE,
build_history_for,
load_chunk_blocks,
resolve_policy,
)
from artemis.utils.task_tree import build_plan_and_history
HASH_A = "hash-a"
HASH_B = "hash-b"
TASK_PLAN = f"- [x] First milestone <!-- {HASH_A} -->\n- [/] Second milestone <!-- {HASH_B} -->"
def _step(number: int, subgoal_hash: str, summary: str | None = None) -> dict:
return {
"step_id": f"id-{number}",
"step_number": number,
"relative_time": f"{number * 10}.0s",
"summary": summary or f"Step {number} summary text.",
"action_taken": {"action": "click", "target_text": f"button {number}"},
"operator_raw_thinking": f"thinking {number}",
"last_execution_result": {"status": "success"},
"extra_metadata": {"subgoal_hash": subgoal_hash},
}
STEPS = [
_step(1, HASH_A),
_step(2, HASH_A),
_step(3, HASH_A),
_step(4, HASH_B),
_step(5, HASH_B),
_step(6, HASH_B),
_step(7, HASH_B),
]
# ---------------------------------------------------------------------------
# Golden parity: the policy table must reproduce every historical hard-coded
# call byte for byte (kwargs recorded verbatim from the 2026-09-01 call sites).
# ---------------------------------------------------------------------------
GOLDEN_CALLS = {
"operator": lambda: build_plan_and_history(
TASK_PLAN,
STEPS,
HASH_B,
last_n_detailed=1,
strict_milestone_pruning=True,
recent_window_size=3,
),
"operator_cold_start": lambda: build_plan_and_history(
TASK_PLAN,
STEPS,
HASH_B,
min_summaries=len(STEPS),
last_n_detailed=1,
),
"failure_analyzer": lambda: build_plan_and_history(
TASK_PLAN,
STEPS,
HASH_B,
last_n_detailed=1,
strict_milestone_pruning=True,
recent_window_size=3,
for_failure_analyzer=True,
),
"planner": lambda: build_plan_and_history(
"",
STEPS,
"default",
last_n_detailed=2,
strict_milestone_pruning=True,
recent_window_size=5,
),
"outputter": lambda: build_plan_and_history(
TASK_PLAN,
STEPS,
HASH_B,
last_n_detailed=0,
min_summaries=len(STEPS),
strict_milestone_pruning=False,
),
"history_analyzer": lambda: build_plan_and_history(
TASK_PLAN,
STEPS,
HASH_B,
last_n_detailed=1,
min_summaries=len(STEPS),
),
"diagnoser": lambda: build_plan_and_history(
TASK_PLAN,
STEPS,
HASH_B,
keep_subgoal_hashes={HASH_B},
),
"committee": lambda: build_plan_and_history(
TASK_PLAN,
STEPS,
HASH_B,
keep_subgoal_hashes={HASH_B},
last_n_detailed=3,
),
}
POLICY_CALLS = {
"operator": lambda: build_history_for(
"operator", TASK_PLAN, STEPS, HASH_B, last_n_detailed=1
),
"operator_cold_start": lambda: build_history_for(
"operator_cold_start", TASK_PLAN, STEPS, HASH_B
),
"failure_analyzer": lambda: build_history_for(
"failure_analyzer", TASK_PLAN, STEPS, HASH_B
),
"planner": lambda: build_history_for("planner", "", STEPS, "default"),
"outputter": lambda: build_history_for("outputter", TASK_PLAN, STEPS, HASH_B),
"history_analyzer": lambda: build_history_for(
"history_analyzer", TASK_PLAN, STEPS, HASH_B
),
"diagnoser": lambda: build_history_for(
"diagnoser", TASK_PLAN, STEPS, HASH_B, keep_subgoal_hashes={HASH_B}
),
"committee": lambda: build_history_for(
"committee", TASK_PLAN, STEPS, HASH_B, keep_subgoal_hashes={HASH_B}
),
}
@pytest.mark.parametrize("agent", sorted(GOLDEN_CALLS))
def test_policy_table_reproduces_call_site_bytes(agent):
assert POLICY_CALLS[agent]() == GOLDEN_CALLS[agent]()
def test_every_declared_policy_is_golden_tested():
assert set(GOLDEN_CALLS) == set(CONTEXT_POLICIES)
# ---------------------------------------------------------------------------
# Config overrides
# ---------------------------------------------------------------------------
def _config_with_policies(policies):
return SimpleNamespace(memory=SimpleNamespace(policies=policies))
def test_resolve_policy_applies_config_override():
cfg = _config_with_policies({"planner": {"last_n_detailed": 3, "bogus_field": 1}})
with patch("artemis.config.load_agent_config", return_value=cfg):
policy = resolve_policy("planner")
assert policy.last_n_detailed == 3
assert policy.strict_milestone_pruning is True # untouched fields survive
def test_resolve_policy_unknown_agent_raises():
with pytest.raises(KeyError):
resolve_policy("no_such_agent")
# ---------------------------------------------------------------------------
# Chunk hand-off (§10 decision 4)
# ---------------------------------------------------------------------------
RENDERED_CHUNK = (
"[Chunk 1 | Steps 1–3 | T+00:10 → T+00:30]\n"
"\n"
"① Synopsis & effects\n"
" What this segment was doing: warming up.\n"
"\n"
"② Compressed step summary\n"
" - Steps 1–3: opened the app and logged in\n"
"\n"
"③ Step action ledger\n"
"- Step 1 (T+00:10): click['button 1'] -> executed\n"
"- Step 2 (T+00:20): click['button 2'] -> executed\n"
"- Step 3 (T+00:30): click['button 3'] -> executed"
)
class _FakeEngine:
def __init__(self, rows):
self._rows = rows
def get_history_chunks(self, **kwargs):
return self._rows
def _chunk_row():
return SimpleNamespace(
start_step_number=1,
end_step_number=3,
rendered_text=RENDERED_CHUNK,
status="ready",
)
def _transcript_cfg(enabled: bool):
return SimpleNamespace(
memory=SimpleNamespace(
transcript=SimpleNamespace(enabled=enabled), policies={}
)
)
def test_flag_off_output_is_byte_identical_even_with_engine():
engine = _FakeEngine([_chunk_row()])
with patch("artemis.config.load_agent_config", return_value=_transcript_cfg(False)):
with_engine = build_history_for(
"outputter", TASK_PLAN, STEPS, HASH_B, engine=engine
)
without_engine = build_history_for("outputter", TASK_PLAN, STEPS, HASH_B)
assert with_engine == without_engine
assert "[Chunk 1" not in with_engine
def test_flag_on_full_view_renders_chunk_block_and_drops_covered_steps():
engine = _FakeEngine([_chunk_row()])
with patch("artemis.config.load_agent_config", return_value=_transcript_cfg(True)):
out = build_history_for("outputter", TASK_PLAN, STEPS, HASH_B, engine=engine)
# Full view: the whole three-band block including the ③ ledger.
assert "[Chunk 1 | Steps 1–3" in out
assert "③ Step action ledger" in out
assert "- Step 2 (T+00:20)" in out
# Covered per-step summary lines are replaced by the chunk block.
assert "*Step 2 (Start: 20.0s)" not in out
# Steps outside the chunk range still render.
assert "Step 5" in out
def test_flag_on_digest_view_replaces_ledger_with_recall_marker():
engine = _FakeEngine([_chunk_row()])
cfg = _transcript_cfg(True)
with patch("artemis.config.load_agent_config", return_value=cfg):
out = build_history_for(
"diagnoser",
TASK_PLAN,
STEPS,
HASH_B,
keep_subgoal_hashes={HASH_A, HASH_B},
engine=engine,
)
assert "[Chunk 1 | Steps 1–3" in out
assert "② Compressed step summary" in out
marker = DIGEST_LEDGER_MARKER_TEMPLATE.format(start=1, end=3)
assert marker in out
assert "- Step 2 (T+00:20)" not in out # digest carries no ③ rows
def test_chunk_view_none_never_loads_chunks():
engine = _FakeEngine([_chunk_row()])
with patch("artemis.config.load_agent_config", return_value=_transcript_cfg(True)):
assert load_chunk_blocks(engine, None) is None
# The cold-start policy declares chunk_view=None: even with an engine
# and the flag on, its output carries no chunk blocks (the restored
# F region must stay a pure step rendering).
out = build_history_for(
"operator_cold_start", TASK_PLAN, STEPS, HASH_B, engine=engine
)
assert CONTEXT_POLICIES["operator_cold_start"].chunk_view is None
assert "[Chunk 1" not in out
def test_most_recent_step_is_never_suppressed_by_a_chunk():
engine = _FakeEngine(
[
SimpleNamespace(
start_step_number=1,
end_step_number=7, # covers every step incl. the newest
rendered_text=RENDERED_CHUNK,
status="ready",
)
]
)
with patch("artemis.config.load_agent_config", return_value=_transcript_cfg(True)):
out = build_history_for("outputter", TASK_PLAN, STEPS, HASH_B, engine=engine)
# The most-recent step must keep its own (detailed) rendering.
assert "Step 7" in out
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,187 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for the shared StepMemoryService scheduling runtime (M1)."""
import asyncio
from unittest.mock import Mock
import pytest
from artemis.memory.step_memory import StepMemoryService
class _ScriptedService(StepMemoryService):
"""StepMemoryService with a scriptable attempt function and status log."""
def __init__(self, attempt_fn, **kwargs):
super().__init__(Mock(), **kwargs)
self._attempt_fn = attempt_fn
self.status_events: list[tuple] = []
async def _attempt(self, key):
return await self._attempt_fn(self, key)
def _on_status(self, key, status):
self.status_events.append((key, status))
@pytest.mark.asyncio
async def test_zero_blocking_submit_and_flush():
"""submit() returns immediately; flush() drains the background job."""
release = asyncio.Event()
async def attempt(svc, key):
await release.wait()
svc._summaries[key] = f"summary-for-{key}"
return True
service = _ScriptedService(attempt)
service.submit("step-1", {"step_number": 1})
# Zero-blocking: the job cannot have completed yet (its gate is closed).
assert service.is_pending("step-1")
assert not service.has_summary("step-1")
release.set()
await service.flush()
assert service.get_summary("step-1") == "summary-for-step-1"
assert not service.is_pending("step-1")
@pytest.mark.asyncio
async def test_bounded_retry_enters_failed_state():
"""A permanently failing job stops after 1 + retry_limit attempts."""
attempts = 0
async def attempt(svc, key):
nonlocal attempts
attempts += 1
return False
service = _ScriptedService(attempt, retry_limit=2)
service._retry_delays = (0.0,)
service.submit("step-x", {"step_number": 7})
await service.flush()
assert attempts == 3
assert service.has_failed("step-x")
assert not service.has_summary("step-x")
# The failed job still reads as pending (lossless semantics downstream).
assert service.is_pending("step-x")
assert service.status_events == [("step-x", "pending"), ("step-x", "failed")]
@pytest.mark.asyncio
async def test_step_id_keying_with_tool_call_alias():
"""Jobs keyed by step id remain queryable through tool_call_id aliases."""
async def attempt(svc, key):
svc._summaries[key] = "canonical summary"
return True
service = _ScriptedService(attempt)
service.submit("uuid-123", {"step_number": 4}, aliases=("tc-abc",))
await service.flush()
assert service.resolve_key("tc-abc") == "uuid-123"
assert service.has_job("tc-abc")
assert service.has_summary("tc-abc")
assert service.get_summary("tc-abc") == "canonical summary"
assert service.get_summary("uuid-123") == "canonical summary"
assert service.get_job_payload("tc-abc") == {"step_number": 4}
assert not service.has_failed("tc-abc")
@pytest.mark.asyncio
async def test_max_concurrency_bounds_parallel_attempts():
"""With max_concurrency=1, two jobs never run their attempts concurrently."""
active = 0
max_active = 0
async def attempt(svc, key):
nonlocal active, max_active
active += 1
max_active = max(max_active, active)
await asyncio.sleep(0.01)
active -= 1
svc._summaries[key] = "done"
return True
service = _ScriptedService(attempt, max_concurrency=1)
service.submit("a", {"step_number": 1})
service.submit("b", {"step_number": 2})
await service.flush()
assert service.has_summary("a") and service.has_summary("b")
assert max_active == 1
@pytest.mark.asyncio
async def test_flush_timeout_cancels_stragglers():
"""flush() cancels jobs that outlive the timeout without failing them."""
async def attempt(svc, key):
await asyncio.sleep(30)
return True
service = _ScriptedService(attempt)
service.submit("slow", {"step_number": 1})
await service.flush(timeout_seconds=0.05)
assert service.is_pending("slow")
assert not service.has_failed("slow")
assert service._pending_tasks["slow"].done()
@pytest.mark.asyncio
async def test_flush_uses_configured_default_timeout():
"""flush() with no argument honors the constructor's flush_timeout_s."""
async def attempt(svc, key):
await asyncio.sleep(30)
return True
service = _ScriptedService(attempt, flush_timeout_s=0.05)
service.submit("slow", {"step_number": 1})
await asyncio.wait_for(service.flush(), timeout=5)
assert service._pending_tasks["slow"].done()
@pytest.mark.asyncio
async def test_resubmit_clears_failed_state():
"""Re-submitting a failed job gives it a fresh retry budget."""
behaviors = {"fail": True}
async def attempt(svc, key):
if behaviors["fail"]:
return False
svc._summaries[key] = "second time lucky"
return True
service = _ScriptedService(attempt, retry_limit=0)
service._retry_delays = (0.0,)
service.submit("job", {"step_number": 1})
await service.flush()
assert service.has_failed("job")
behaviors["fail"] = False
service._retry_counts.clear()
service.submit("job", {"step_number": 1})
await service.flush()
assert not service.has_failed("job")
assert service.get_summary("job") == "second time lucky"
+298
View File
@@ -0,0 +1,298 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for the Pro session transcript ledger (history redesign §3.2, M2).
Covers the four-region discipline: S-region byte stability across turns, the
append-only active region, depth-1 stripping of old UI lists and plan
recitations, the depth-K screenshot scrub in the Pro message shape (grace /
placeholder / freeze — the M1 race regressions re-run against HumanMessage
observations keyed by step id), ``T+mm:ss`` session-offset timestamps, the
tool-call/response pairing invariant, and the cold-start restored-history
block.
"""
import json
import re
import pytest
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
from artemis.memory.step_memory import StepMemoryService
from artemis.memory.transcript import (
EXECUTION_RESULT_MARKER,
PLAN_RECITATION_MARKER,
PRO_UI_LIST_MARKER,
RESTORED_HISTORY_HEADER,
TranscriptLedger,
format_session_offset,
)
OFFSET_RE = re.compile(r"T\+\d{2,}:\d{2}")
def _service() -> StepMemoryService:
return StepMemoryService(ctx=None)
def _observation(i: int) -> HumanMessage:
return HumanMessage(
content=[
{"type": "text", "text": f"# CURRENT OBSERVATION [T+00:0{i % 10}]"},
{"type": "text", "text": f"{PLAN_RECITATION_MARKER}\n- [/] milestone {i}"},
{"type": "text", "text": "--- Current Screenshot ---"},
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG_{i}"}},
{"type": "text", "text": f"{PRO_UI_LIST_MARKER}\n[1] button {i}"},
]
)
def _turn(i: int, tool_call_count: int = 1) -> list:
messages: list = [_observation(i)]
if tool_call_count:
tool_calls = [
{"name": "click", "args": {"target": n + 1}, "id": f"tc{i}-{n}", "type": "tool_call"}
for n in range(tool_call_count)
]
messages.append(AIMessage(content=f"thinking {i}", tool_calls=tool_calls))
for tc in tool_calls:
messages.append(ToolMessage(tool_call_id=tc["id"], content="Action Recorded"))
return messages
def _play_turn(ledger: TranscriptLedger, i: int, *, prev_key=None, prev_result=None, **turn_kwargs):
"""Mimic the operator's real ordering: commit previous, render, stage."""
ledger.commit_staged(step_key=prev_key, validator_result=prev_result)
turn = _turn(i, **turn_kwargs)
rendered = ledger.render([turn[0]])
ledger.stage_turn(turn)
return rendered
def _fingerprint(msg) -> str:
return json.dumps(msg.content, sort_keys=True, default=str)
def _has_image(msg) -> bool:
return any(
isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content
)
def test_format_session_offset():
assert format_session_offset(0) == "T+00:00"
assert format_session_offset(83) == "T+01:23"
# Minutes never wrap into hours: byte-stable monotonic labels.
assert format_session_offset(3725) == "T+62:05"
assert format_session_offset(-5) == "T+00:00"
def test_static_prefix_set_once_and_byte_stable():
ledger = TranscriptLedger(step_memory=_service())
system = SystemMessage(content="STATIC SYSTEM PROMPT")
ledger.set_static_prefix([system])
with pytest.raises(RuntimeError):
ledger.set_static_prefix([SystemMessage(content="other")])
fingerprints = set()
for i in range(1, 6):
rendered = _play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
assert rendered[0] is system
fingerprints.add(rendered[0].content)
assert fingerprints == {"STATIC SYSTEM PROMPT"}
def test_active_region_is_append_only():
ledger = TranscriptLedger(step_memory=_service())
seen_ids: list[int] = []
for i in range(1, 5):
_play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
active = ledger.active_messages
# The previously observed prefix is unchanged in identity and order.
assert [id(m) for m in active[: len(seen_ids)]] == seen_ids
seen_ids = [id(m) for m in active]
assert ledger.turn_count == 3 # 3 committed, 1 still staged
def test_validator_result_message_carries_session_offset():
ledger = TranscriptLedger(step_memory=_service())
_play_turn(ledger, 1)
_play_turn(
ledger,
2,
prev_key="step-1",
prev_result={"status": "success", "execution": [{"action": "tap", "attempts": ["ok"]}]},
)
result_messages = [
m
for m in ledger.active_messages
if isinstance(m, HumanMessage)
and str(m.content).find(EXECUTION_RESULT_MARKER) >= 0
]
assert len(result_messages) == 1
text = result_messages[0].content[0]["text"]
assert OFFSET_RE.search(text), text
assert "Status: success" in text
assert "ago" not in text
def test_validator_result_skipped_for_actionless_turn():
ledger = TranscriptLedger(step_memory=_service())
_play_turn(ledger, 1)
_play_turn(ledger, 2, prev_key="step-1", prev_result=None)
assert not any(
EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages
)
def test_old_turn_recitation_and_ui_list_stripped_at_depth_1():
ledger = TranscriptLedger(step_memory=_service())
_play_turn(ledger, 1)
rendered = _play_turn(ledger, 2, prev_key="step-1")
committed_obs = ledger.active_messages[0]
committed_text = str(committed_obs.content)
# Depth-1 scrub: the previous turn's plan recitation and UI list are gone.
assert PLAN_RECITATION_MARKER not in committed_text
assert PRO_UI_LIST_MARKER not in committed_text
# Its screenshot is still inside the K-depth active window.
assert _has_image(committed_obs)
# The live tail keeps both.
tail_text = str(rendered[-1].content)
assert PLAN_RECITATION_MARKER in tail_text
assert PRO_UI_LIST_MARKER in tail_text
def test_depth_k_image_resolved_to_ready_visual_summary():
service = _service()
service._step_inputs["step-1"] = {"step_number": 1}
service._summaries["step-1"] = "Objective visual transition 1."
ledger = TranscriptLedger(step_memory=service, image_scrub_depth=3)
for i in range(1, 5):
_play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
first_obs = ledger.active_messages[0]
assert not _has_image(first_obs)
assert {
"type": "text",
"text": "--- Historical Visual Transition ---\nObjective visual transition 1.",
} in first_obs.content
def test_pending_grace_then_placeholder_never_backfilled():
service = _service()
service._step_inputs["step-1"] = {"step_number": 1} # pending job, no summary
ledger = TranscriptLedger(
step_memory=service, image_scrub_depth=2, pending_grace_steps=1
)
# Enough turns to push turn 1 past K + grace.
for i in range(1, 7):
_play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
first_obs = ledger.active_messages[0]
assert not _has_image(first_obs)
assert "[visual summary pending; evidence at DataEngine step 1]" in str(
first_obs.content
)
frozen = _fingerprint(first_obs)
# A late summary must never mutate the frozen message.
service._summaries["step-1"] = "Too late."
_play_turn(ledger, 7, prev_key="step-6")
assert _fingerprint(ledger.active_messages[0]) == frozen
def test_failed_summary_becomes_unavailable_placeholder():
service = _service()
service._step_inputs["step-1"] = {"step_number": 1}
service._failed.add("step-1")
ledger = TranscriptLedger(
step_memory=service, image_scrub_depth=2, pending_grace_steps=5
)
for i in range(1, 4):
_play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
first_obs = ledger.active_messages[0]
assert not _has_image(first_obs)
assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(
first_obs.content
)
def test_tool_call_response_pairs_are_never_split():
ledger = TranscriptLedger(step_memory=_service(), image_scrub_depth=2)
for i in range(1, 7):
_play_turn(
ledger,
i,
prev_key=f"step-{i - 1}" if i > 1 else None,
prev_result={"status": "success"} if i > 1 else None,
tool_call_count=2,
)
active = list(ledger.active_messages)
for idx, msg in enumerate(active):
if isinstance(msg, AIMessage) and msg.tool_calls:
following = active[idx + 1 : idx + 1 + len(msg.tool_calls)]
assert [getattr(m, "tool_call_id", None) for m in following] == [
tc["id"] for tc in msg.tool_calls
], f"tool-call pairing split at active index {idx}"
def test_restored_history_only_seeds_an_empty_ledger():
ledger = TranscriptLedger(step_memory=_service())
ledger.set_restored_history(f"{RESTORED_HISTORY_HEADER} steps 1-9 ...")
assert ledger.has_restored_history
with pytest.raises(RuntimeError):
ledger.set_restored_history("again")
ledger.set_static_prefix([SystemMessage(content="S")])
rendered = _play_turn(ledger, 1)
# Order: S region, then the frozen restored block, then the live tail.
assert rendered[0].content == "S"
assert RESTORED_HISTORY_HEADER in str(rendered[1].content)
assert "# CURRENT OBSERVATION" in str(rendered[-1].content)
ledger2 = TranscriptLedger(step_memory=_service())
_play_turn(ledger2, 1)
ledger2.commit_staged(step_key="step-1")
with pytest.raises(RuntimeError):
ledger2.set_restored_history("late")
def test_stage_twice_commits_the_forgotten_turn():
ledger = TranscriptLedger(step_memory=_service())
ledger.stage_turn(_turn(1))
ledger.stage_turn(_turn(2))
assert ledger.turn_count == 1
assert ledger.has_staged_turn
def test_no_ago_wording_in_ledger_output():
ledger = TranscriptLedger(step_memory=_service())
for i in range(1, 4):
_play_turn(
ledger,
i,
prev_key=f"step-{i - 1}" if i > 1 else None,
prev_result={"status": "success"} if i > 1 else None,
)
blob = " ".join(str(m.content) for m in ledger.active_messages)
assert " ago" not in blob
+134
View File
@@ -0,0 +1,134 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for the agent.memory configuration block (M1/M2)."""
from artemis.config import AgentGlobalConfig
def test_memory_defaults():
cfg = AgentGlobalConfig.model_validate({})
# 2 per the 2026-09-01 on-device baseline (serial queue at 1 pushed
# summary-ready P90 past the scrub-edge grace window).
assert cfg.memory.runtime.max_concurrency == 2
assert cfg.memory.runtime.retry_limit == 3
assert cfg.memory.runtime.flush_timeout_s == 30.0
# M5 (2026-09-01): the transcript path ships ON by default; explicit
# false is the byte-for-byte rollback switch (see the flip test below).
assert cfg.memory.transcript.enabled is True
assert cfg.memory.transcript.image_scrub_depth == 3
assert cfg.memory.transcript.pending_grace_steps == 3
assert cfg.memory.transcript.xml_scrub_depth == 1
def test_memory_transcript_flag_explicit_disable_is_rollback_switch():
cfg = AgentGlobalConfig.model_validate(
{"memory": {"transcript": {"enabled": False}}}
)
assert cfg.memory.transcript.enabled is False
# Sibling scrub defaults are unaffected by flipping the flag.
assert cfg.memory.transcript.image_scrub_depth == 3
def test_memory_explicit_values():
cfg = AgentGlobalConfig.model_validate(
{
"memory": {
"runtime": {"max_concurrency": 4, "retry_limit": 5, "flush_timeout_s": 10},
"transcript": {
"image_scrub_depth": 5,
"pending_grace_steps": 1,
"xml_scrub_depth": 2,
},
}
}
)
assert cfg.memory.runtime.max_concurrency == 4
assert cfg.memory.runtime.retry_limit == 5
assert cfg.memory.runtime.flush_timeout_s == 10.0
assert cfg.memory.transcript.image_scrub_depth == 5
assert cfg.memory.transcript.pending_grace_steps == 1
assert cfg.memory.transcript.xml_scrub_depth == 2
def test_legacy_step_summarizer_retry_limit_seeds_memory_runtime():
"""An explicit legacy flash.step_summarizer.retry_limit keeps working."""
cfg = AgentGlobalConfig.model_validate(
{"flash": {"step_summarizer": {"retry_limit": 7}}}
)
assert cfg.flash.step_summarizer.retry_limit == 7
assert cfg.memory.runtime.retry_limit == 7
def test_explicit_memory_runtime_wins_over_legacy_alias():
cfg = AgentGlobalConfig.model_validate(
{
"flash": {"step_summarizer": {"retry_limit": 7}},
"memory": {"runtime": {"retry_limit": 2}},
}
)
assert cfg.memory.runtime.retry_limit == 2
# The legacy key itself is untouched for back-compat readers.
assert cfg.flash.step_summarizer.retry_limit == 7
def test_legacy_default_does_not_override_memory_default():
"""A step_summarizer block without retry_limit must not seed anything."""
cfg = AgentGlobalConfig.model_validate(
{"flash": {"step_summarizer": {"enabled": True}}}
)
assert cfg.memory.runtime.retry_limit == 3
def test_recall_and_similarity_defaults():
"""M4 defaults: recall block on, bounded; similarity hint on. The distance
threshold was calibrated to 5 in M5 (2026-09-01 on-device dHash data:
same screen <=4, different screens >=7, valley at 5-6)."""
cfg = AgentGlobalConfig()
assert cfg.memory.recall.enabled is True
assert cfg.memory.recall.max_results == 5
assert cfg.memory.recall.max_text_tokens == 2000
assert cfg.memory.recall.max_image_steps == 1
assert cfg.memory.transcript.similarity_hint is True
assert cfg.memory.transcript.similarity_max_distance == 5
assert cfg.memory.policies == {}
def test_chunking_era_cap_follows_max_chunks_by_default():
"""M5: max_eras is an independent key; None (default) follows max_chunks."""
cfg = AgentGlobalConfig()
assert cfg.memory.chunking.max_chunks == 8
assert cfg.memory.chunking.max_eras is None
overridden = AgentGlobalConfig.model_validate(
{"memory": {"chunking": {"max_chunks": 12, "max_eras": 4}}}
)
assert overridden.memory.chunking.max_chunks == 12
assert overridden.memory.chunking.max_eras == 4
def test_recall_and_policies_overridable():
cfg = AgentGlobalConfig.model_validate(
{
"memory": {
"recall": {"enabled": False, "max_results": 3},
"transcript": {"similarity_hint": False},
"policies": {"planner": {"last_n_detailed": 3}},
}
}
)
assert cfg.memory.recall.enabled is False
assert cfg.memory.recall.max_results == 3
assert cfg.memory.transcript.similarity_hint is False
assert cfg.memory.policies == {"planner": {"last_n_detailed": 3}}
+173
View File
@@ -0,0 +1,173 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for the M0 session token metering pipeline (pure recording)."""
from unittest.mock import Mock
from uuid import uuid4
from langchain_core.messages import AIMessage
from artemis.services.token_meter import (
SessionTokenMeter,
extract_usage,
get_meter,
record_llm_usage,
)
def _msg_with_usage(prompt=1000, completion=50, cached=0):
usage = {
"input_tokens": prompt,
"output_tokens": completion,
"total_tokens": prompt + completion,
}
if cached:
usage["input_token_details"] = {"cache_read": cached}
return AIMessage(content="ok", usage_metadata=usage)
def test_extract_usage_from_usage_metadata():
usage = extract_usage(_msg_with_usage(prompt=1200, completion=80, cached=900))
assert usage == {
"prompt_tokens": 1200,
"completion_tokens": 80,
"total_tokens": 1280,
"cached_tokens": 900,
}
def test_extract_usage_from_response_metadata_fallback():
msg = AIMessage(content="ok")
msg.response_metadata = {
"usage_metadata": {
"prompt_token_count": 500,
"candidates_token_count": 25,
"total_token_count": 525,
"cached_content_token_count": 400,
}
}
usage = extract_usage(msg)
assert usage == {
"prompt_tokens": 500,
"completion_tokens": 25,
"total_tokens": 525,
"cached_tokens": 400,
}
def test_extract_usage_absent_returns_none():
assert extract_usage(AIMessage(content="no usage")) is None
assert extract_usage(object()) is None
def test_meter_accumulates_session_totals():
meter = SessionTokenMeter("s1")
meter.record({"prompt_tokens": 100, "completion_tokens": 10, "cached_tokens": 0})
snapshot = meter.record(
{"prompt_tokens": 300, "completion_tokens": 20, "cached_tokens": 250}
)
assert snapshot == {
"session_llm_calls": 2,
"session_prompt_tokens": 400,
"session_completion_tokens": 30,
"session_cached_tokens": 250,
"session_cache_hit_calls": 1,
}
assert meter.last_prompt_tokens == 300
def test_record_llm_usage_records_trace_with_context_base():
engine = Mock()
engine.current_session_id = uuid4()
engine.current_step_id = uuid4()
payload = record_llm_usage(
engine, _msg_with_usage(prompt=2000, completion=100, cached=1500), source="google:test"
)
assert payload is not None
assert payload["context_base_tokens"] == 2000
assert payload["cached_tokens"] == 1500
assert payload["source"] == "google:test"
engine.record_trace.assert_called_once()
kwargs = engine.record_trace.call_args.kwargs
assert kwargs["type"] == "llm_call"
assert kwargs["name"] == "llm_usage"
assert kwargs["step_id"] == engine.current_step_id
assert kwargs["payload"]["prompt_tokens"] == 2000
def test_record_llm_usage_accumulates_across_calls_same_session():
engine = Mock()
engine.current_session_id = uuid4()
engine.current_step_id = None
record_llm_usage(engine, _msg_with_usage(prompt=100, completion=5))
payload = record_llm_usage(engine, _msg_with_usage(prompt=250, completion=10, cached=90))
assert payload["session_llm_calls"] == 2
assert payload["session_prompt_tokens"] == 350
assert payload["session_cached_tokens"] == 90
assert payload["session_cache_hit_calls"] == 1
meter = get_meter(engine.current_session_id)
assert meter.last_prompt_tokens == 250
def test_record_llm_usage_never_raises_and_skips_gracefully():
# No engine
assert record_llm_usage(None, _msg_with_usage()) is None
# Engine without a session
engine = Mock()
engine.current_session_id = None
assert record_llm_usage(engine, _msg_with_usage()) is None
engine.record_trace.assert_not_called()
# Response without usage
engine2 = Mock()
engine2.current_session_id = uuid4()
assert record_llm_usage(engine2, AIMessage(content="nope")) is None
engine2.record_trace.assert_not_called()
# record_trace blowing up must not propagate
engine3 = Mock()
engine3.current_session_id = uuid4()
engine3.record_trace.side_effect = RuntimeError("boom")
assert record_llm_usage(engine3, _msg_with_usage()) is None
def test_record_llm_usage_lens_calls_do_not_touch_last_prompt_tokens():
"""Background lens calls accumulate totals but never overwrite the live
context base (last_prompt_tokens) consumed by the compaction thresholds."""
engine = Mock()
engine.current_session_id = uuid4()
engine.current_step_id = None
record_llm_usage(engine, _msg_with_usage(prompt=15000))
meter = get_meter(engine.current_session_id)
assert meter.last_prompt_tokens == 15000
payload = record_llm_usage(
engine,
_msg_with_usage(prompt=300, completion=40),
source="lens:visual_transition:test-model",
update_last_prompt=False,
)
assert payload["source"] == "lens:visual_transition:test-model"
assert meter.last_prompt_tokens == 15000 # untouched by the lens call
assert meter.llm_calls == 2
assert meter.prompt_tokens == 15300
+265
View File
@@ -0,0 +1,265 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""recall_history boundaries and search correctness (M4, design §10).
Hard boundaries under test: bounded result count, bounded response tokens,
image recall limited to the real stored screenshots of a single step, and
every result carrying a step number. Plus: keyword/step-range/notes filter
correctness and the era-marker re-entry (a step_range returns the full-width
``build_action_ledger`` rows for that range).
"""
import io
import re
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from PIL import Image
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
from artemis.tools.history_recall import (
RecallHistoryTool,
recall_history,
search_history,
)
from artemis.utils.notes import save_note_content
def _jpeg(color: str) -> bytes:
buf = io.BytesIO()
Image.new("RGB", (64, 128), color).save(buf, format="JPEG")
return buf.getvalue()
def _make_engine(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
engine = DataEngine(mock_ctx)
engine.start_session("recall test session")
return engine
def _flush(engine):
for t in list(engine._pending_threads):
t.join()
def _cfg(**overrides):
values = {"enabled": True, "max_results": 5, "max_text_tokens": 2000, "max_image_steps": 1}
values.update(overrides)
return SimpleNamespace(**values)
@pytest.fixture
def engine(tmp_path):
engine = _make_engine(tmp_path)
engine.record_step(
pre_screenshot_bytes=_jpeg("red"),
summary="Opened the login page.",
action_taken={"action": "click", "target_text": "Login entry"},
last_execution_result={"status": "success"},
)
engine.record_step(
pre_screenshot_bytes=_jpeg("green"),
post_screenshot_bytes=_jpeg("blue"),
summary="Typed the verification code 4711 into the input.",
action_taken={"action": "input_text", "text": "4711"},
last_execution_result={"status": "success"},
operator_raw_thinking="The code arrived via SMS; entering 4711 now.",
)
engine.record_step(
summary="A promo popup appeared and was dismissed.",
action_taken={"action": "click", "target_text": "Close popup"},
last_execution_result={"status": "failed", "error": "element vanished"},
)
_flush(engine)
return engine
def _result_blocks(text: str) -> list[str]:
return [b for b in text.split("\n") if b.startswith("[")]
def test_keyword_search_finds_step_and_carries_step_number(engine):
out = search_history(engine, query="verification code", recall_config=_cfg())
assert isinstance(out, str)
assert "Step 2" in out
assert "4711" in out
# Every result header names a step / step range / step anchor.
for header in _result_blocks(out):
assert re.search(r"Step[s]? \d", header), header
def test_step_range_filters_matches(engine):
out = search_history(
engine, query="popup", step_range=[1, 2], recall_config=_cfg()
)
# Step 3 (the popup step) is outside the range: only the range ledger and
# a no-match line may appear.
assert "A promo popup appeared" not in out
assert "No matches" in out
def test_step_range_returns_full_width_ledger_rows(engine):
"""The recall-only era period paragraph's '(Step-level ledger via
recall_history for steps a–b)' re-entry point."""
out = search_history(engine, query="", step_range=[1, 3], recall_config=_cfg())
assert "Action ledger for Steps 1–3" in out
# Full-width build_action_ledger rows: step number + T+mm:ss offset +
# semantic action + result phrase.
assert re.search(r"- Step 1 \(T\+\d{2}:\d{2}\): .*Login entry", out)
assert re.search(r"- Step 2 \(T\+\d{2}:\d{2}\): ", out)
assert re.search(r"- Step 3 \(T\+\d{2}:\d{2}\): ", out)
def test_result_count_is_clamped_by_config(engine):
for i in range(8):
engine.record_step(
summary=f"Scrolled the anchorterm feed page {i}.",
action_taken={"action": "swipe"},
last_execution_result={"status": "success"},
)
_flush(engine)
out = search_history(
engine, query="anchorterm", max_results=50, recall_config=_cfg(max_results=3)
)
assert out.count("[Step ") == 3
assert "more not shown" in out
def test_response_is_truncated_at_token_budget(engine):
engine.record_step(
summary="needleword " + "filler content " * 400,
action_taken={"action": "click"},
last_execution_result={"status": "success"},
)
_flush(engine)
out = search_history(
engine,
query="needleword",
include_details=True,
recall_config=_cfg(max_text_tokens=150),
)
assert len(out) <= 150 * 4 + 200 # budget + truncation notice
assert "truncated at the recall token budget" in out
def test_notes_are_searchable_and_anchored_to_a_step(engine):
save_note_content(engine.base_dir, "login_flow", "The OTP entry lives behind the SMS tab.")
out = search_history(engine, query="OTP entry", recall_config=_cfg())
assert "[Note 'login_flow'" in out
assert re.search(r"as of Step \d|last written at Step \d", out)
def test_chunk_rows_are_searchable(engine):
engine.record_history_chunk(
start_step_number=1,
end_step_number=2,
version=1,
status="ready",
band2=" - Steps 1–2: walked the chunkneedle flow",
band3="- Step 1 (T+00:01): click -> executed",
rendered_text=(
"[Chunk 1 | Steps 1–2]\n② Compressed step summary\n"
" - Steps 1–2: walked the chunkneedle flow\n"
"③ Step action ledger\n- Step 1 (T+00:01): click -> executed"
),
)
_flush(engine)
out = search_history(engine, query="chunkneedle", recall_config=_cfg())
assert "[History chunk | Steps 1–2" in out
def test_images_only_for_a_single_step_and_only_real_files(engine):
out = search_history(
engine,
query="verification code",
include_images=True,
recall_config=_cfg(),
)
assert isinstance(out, list)
images = [b for b in out if b.get("type") == "image_url"]
# Step 2 has a real pre and post screenshot on disk — but never more than
# one step's worth of images.
assert 1 <= len(images) <= 2
labels = [b["text"] for b in out if b.get("type") == "text" and "screenshot of Step" in b.get("text", "")]
assert all("Step 2" in label for label in labels)
for block in images:
assert block["image_url"]["url"].startswith("data:image/jpeg;base64,")
def test_images_disabled_by_config_cap(engine):
out = search_history(
engine,
query="verification code",
include_images=True,
recall_config=_cfg(max_image_steps=0),
)
assert isinstance(out, str)
def test_missing_image_files_yield_note_not_blocks(engine):
# Step 3 recorded no screenshots at all.
out = search_history(
engine,
query="promo popup",
include_images=True,
recall_config=_cfg(),
)
assert isinstance(out, list)
assert not [b for b in out if b.get("type") == "image_url"]
assert any(
"no stored screenshots exist" in b.get("text", "") for b in out if b.get("type") == "text"
)
@pytest.mark.asyncio
async def test_execute_requires_query_or_range(engine):
tool = RecallHistoryTool()
ctx = SimpleNamespace(data_engine=engine)
out = await tool.execute(ctx=ctx, query="", step_range=None)
assert "needs a query and/or a step_range" in out
@pytest.mark.asyncio
async def test_execute_without_engine_degrades():
tool = RecallHistoryTool()
out = await tool.execute(ctx=SimpleNamespace(data_engine=None), query="x")
assert "no active DataEngine session" in out
def test_availability_requires_engine_and_config():
assert recall_history.is_available(None) is False
assert recall_history.is_available(SimpleNamespace(data_engine=None)) is False
assert recall_history.is_available(SimpleNamespace(data_engine=object())) is True
def test_foreground_app_stamp_joins_search_surface(tmp_path):
"""M5: a package-name query hits the record_step foreground_app stamp."""
engine = _make_engine(tmp_path)
engine.record_step(
summary="Opened the alarms list.",
action_taken={"action": "click", "target_text": "Alarm"},
ui_tree=[{"package": "com.google.android.deskclock", "bounds": "[0,0][9,9]"}],
)
_flush(engine)
out = search_history(engine, query="deskclock", recall_config=_cfg())
assert "Step 1" in out
assert "No matches" not in out
+116
View File
@@ -0,0 +1,116 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for record-time action semantics enrichment (element hit testing)."""
from artemis.utils.element_hit_test import find_element_at_point, hit_test_semantics
def _xml_el(text, bounds, resource_id=None, cls="android.widget.TextView"):
return {
"text": text,
"bounds": bounds,
"class": cls,
"resource_id": resource_id,
"is_ocr": False,
}
def test_smallest_covering_element_wins():
elements = [
_xml_el("Whole screen", [0, 0, 1080, 2400]),
_xml_el("Card", [100, 100, 500, 500]),
_xml_el("Button", [200, 200, 300, 260], resource_id="btn_ok"),
]
el, source = find_element_at_point(elements, 250, 230)
assert source == "hit_test"
assert el["text"] == "Button"
semantics = hit_test_semantics(elements, 250, 230)
assert semantics == {
"target_label_source": "hit_test",
"target_text": "Button",
"target_class": "android.widget.TextView",
"target_resource_id": "btn_ok",
}
def test_xml_element_preferred_over_smaller_ocr_element():
elements = [
_xml_el("Login", [100, 100, 500, 300]),
{
"text": "Login OCR",
"bounds": [200, 150, 260, 180],
"class": None,
"resource_id": None,
"is_ocr": True,
},
]
el, source = find_element_at_point(elements, 220, 160)
assert source == "hit_test"
assert el["text"] == "Login"
def test_ocr_fallback_when_no_xml_element_covers_point():
elements = [
_xml_el("Elsewhere", [800, 800, 900, 900]),
{
"text": "OCR Label",
"bounds": [100, 100, 300, 200],
"class": None,
"resource_id": None,
"is_ocr": True,
},
]
semantics = hit_test_semantics(elements, 150, 150)
assert semantics["target_label_source"] == "ocr"
assert semantics["target_text"] == "OCR Label"
assert "target_class" not in semantics
assert "target_resource_id" not in semantics
def test_string_bounds_are_parsed():
elements = [_xml_el("Item", "[100,100][300,500]")]
el, source = find_element_at_point(elements, 200, 300)
assert source == "hit_test"
assert el["text"] == "Item"
def test_graceful_degradation_to_none():
# No elements at all
assert hit_test_semantics(None, 100, 100) == {"target_label_source": "none"}
assert hit_test_semantics([], 100, 100) == {"target_label_source": "none"}
# Point outside every element
elements = [_xml_el("Item", [0, 0, 50, 50])]
assert hit_test_semantics(elements, 500, 500) == {"target_label_source": "none"}
# Malformed element data must not raise
malformed = [
{"text": "no bounds"},
{"text": "bad bounds", "bounds": "garbage"},
{"text": "short bounds", "bounds": [1, 2]},
{"text": "non-numeric", "bounds": ["a", "b", "c", "d"]},
"not-a-dict",
None,
]
assert hit_test_semantics(malformed, 10, 10) == {"target_label_source": "none"}
def test_element_without_optional_fields_only_reports_source():
elements = [
{"text": "", "bounds": [0, 0, 100, 100], "class": None, "resource_id": None}
]
semantics = hit_test_semantics(elements, 50, 50)
assert semantics == {"target_label_source": "hit_test"}
+81
View File
@@ -0,0 +1,81 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Perceptual-hash stability and distance behavior (M4 similarity hint)."""
import io
from PIL import Image, ImageDraw
from artemis.utils.image_hash import HASH_SIZE, dhash_hex, hamming_distance_hex
def _screen(draw_fn, size=(270, 600), quality=85) -> bytes:
img = Image.new("RGB", size, "white")
draw_fn(ImageDraw.Draw(img))
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=quality)
return buf.getvalue()
def _login_screen(d):
d.rectangle([20, 40, 250, 90], fill="#3355ff")
d.rectangle([30, 200, 240, 250], outline="#333333", width=4)
d.rectangle([30, 300, 240, 350], outline="#333333", width=4)
d.rectangle([60, 450, 210, 510], fill="#22aa66")
def _settings_screen(d):
for y in range(60, 560, 80):
d.rectangle([10, y, 260, y + 50], fill="#dddddd")
d.ellipse([200, 20, 250, 45], fill="#aa2222")
def test_dhash_is_stable_for_identical_bytes():
img = _screen(_login_screen)
assert dhash_hex(img) == dhash_hex(img)
assert len(dhash_hex(img)) == HASH_SIZE * HASH_SIZE // 4
def test_dhash_survives_jpeg_reencoding_with_small_distance():
original = _screen(_login_screen, quality=95)
with Image.open(io.BytesIO(original)) as img:
buf = io.BytesIO()
img.save(buf, format="JPEG", quality=60)
reencoded = buf.getvalue()
distance = hamming_distance_hex(dhash_hex(original), dhash_hex(reencoded))
assert distance is not None and distance <= 4
def test_dhash_separates_different_screens():
distance = hamming_distance_hex(
dhash_hex(_screen(_login_screen)), dhash_hex(_screen(_settings_screen))
)
assert distance is not None and distance > 8
def test_dhash_best_effort_on_garbage():
assert dhash_hex(None) is None
assert dhash_hex(b"") is None
assert dhash_hex(b"not an image at all") is None
def test_hamming_distance_edge_cases():
assert hamming_distance_hex(None, "ff") is None
assert hamming_distance_hex("ff", None) is None
assert hamming_distance_hex("ff", "ffff") is None # length mismatch
assert hamming_distance_hex("zz", "aa") is None # not hex
assert hamming_distance_hex("00ff", "00ff") == 0
assert hamming_distance_hex("0000", "000f") == 4
+3 -7
View File
@@ -365,7 +365,7 @@ def test_build_plan_and_history_concatenates_thoughts():
assert " - Raw thought" in output
def test_build_plan_and_history_chronological_last_step():
def test_build_plan_and_history_interleaved_decision_loop():
plan = "- [ ] Subgoal 1"
steps = [
{
@@ -390,9 +390,7 @@ def test_build_plan_and_history_chronological_last_step():
],
}
]
output = build_plan_and_history(
plan, steps, "default", last_n_detailed=1, chronological_last_step=True
)
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
assert plan in output
assert "--- Execution History ---" in output
@@ -464,9 +462,7 @@ def test_build_plan_and_history_safety_net_and_failure_analyzer():
],
}
]
output = build_plan_and_history(
plan, steps, "default", last_n_detailed=1, chronological_last_step=True
)
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
assert "- **Step 1 (Most Recent Step, Start: 5.0s)**" in output
assert "* [Operator Decision Loop]:" in output