feat(memory): add incremental transcript compaction
This commit is contained in:
1 parent
cbe8f4f0c6
commit
41ae386a3a
65 files changed
+9971
-815
No files matched your search
@@ -0,0 +1,325 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for the Flash scrub-edge compressor (history redesign §3.2, M1).
|
||||
|
||||
Covers the M1 acceptance items: the legacy-equivalent success-path product
|
||||
(asserted directly since the legacy ``compress_flash_messages`` reference was
|
||||
removed in M5), the three race regressions (grace recovery, grace exhaustion
|
||||
without backfill, failed-summary placeholder), the freeze invariant (bytes
|
||||
behind the scrub edge never change), and depth-1 XML stripping equivalence.
|
||||
"""
|
||||
|
||||
import json
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
|
||||
from artemis.agents.flash.context_compressor import ScrubEdgeCompressor
|
||||
from artemis.agents.flash.summarizer import VisualStepSummarizer
|
||||
from artemis.context import ArtemisContext
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.data_engine = None
|
||||
return ctx
|
||||
|
||||
|
||||
def _initial_observation():
|
||||
return HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "Task: Search flights"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_0"}},
|
||||
{"type": "text", "text": "--- UI Element List ---\nTree 0"},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _step(i: int) -> ToolMessage:
|
||||
return ToolMessage(
|
||||
tool_call_id=f"tc{i}",
|
||||
name="click",
|
||||
content=[
|
||||
{"type": "text", "text": f"Action {i} completed."},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG_{i}"}},
|
||||
{"type": "text", "text": f"--- UI Element List ---\nTree {i}"},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _simulate_turns(compressor, messages, new_messages):
|
||||
"""Mimic FlashRunner's ordering: compress at turn start, then append."""
|
||||
for msg in new_messages:
|
||||
compressor.compress(messages)
|
||||
messages.append(msg)
|
||||
compressor.compress(messages)
|
||||
|
||||
|
||||
def _content_fingerprint(msg) -> str:
|
||||
return json.dumps(msg.content, sort_keys=True, default=str)
|
||||
|
||||
|
||||
def _has_image(msg) -> bool:
|
||||
return any(
|
||||
isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content
|
||||
)
|
||||
|
||||
|
||||
def test_success_path_product_behind_the_scrub_edge(mock_context):
|
||||
"""With summaries ready in time, every message behind the scrub edge is the
|
||||
legacy-shaped product: action text kept, XML stripped, screenshot replaced
|
||||
by the ``--- Historical Visual Transition ---`` summary block.
|
||||
|
||||
(Asserted directly; the legacy ``compress_flash_messages`` reference this
|
||||
was originally diffed against was removed in M5.) Intended M1 timing
|
||||
change: the swap happens at depth K, so the newest K-1 historical steps
|
||||
still carry their screenshots.
|
||||
"""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
for i in range(1, 7):
|
||||
summarizer._summaries[f"tc{i}"] = f"Visual summary {i}."
|
||||
|
||||
new_messages: list = []
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
prune_history_xml=True,
|
||||
image_scrub_depth=3,
|
||||
pending_grace_steps=3,
|
||||
)
|
||||
_simulate_turns(
|
||||
compressor, new_messages, [_initial_observation()] + [_step(i) for i in range(1, 7)]
|
||||
)
|
||||
|
||||
# Initial observation behind the edge: image silently dropped (no summary
|
||||
# job), task text kept, XML stripped.
|
||||
assert new_messages[0].content == [{"type": "text", "text": "Task: Search flights"}]
|
||||
|
||||
# Steps 1-4 behind the scrub edge: exact legacy-shaped swap product.
|
||||
for i in range(1, 5):
|
||||
assert new_messages[i].content == [
|
||||
{"type": "text", "text": f"Action {i} completed."},
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"--- Historical Visual Transition ---\nVisual summary {i}.",
|
||||
},
|
||||
], f"message {i} diverged from the legacy-shaped product"
|
||||
|
||||
# The live observation is intact.
|
||||
assert _has_image(new_messages[6])
|
||||
assert "Tree 6" in str(new_messages[6].content)
|
||||
|
||||
# Intended timing change: step 5 (depth 2 < K) keeps its screenshot under
|
||||
# the scrub edge; its XML was still stripped at depth 1.
|
||||
assert _has_image(new_messages[5])
|
||||
assert "Visual summary 5." not in str(new_messages[5].content)
|
||||
assert "Tree 5" not in str(new_messages[5].content)
|
||||
|
||||
|
||||
def test_race_pending_at_edge_recovers_within_grace(mock_context):
|
||||
"""A step whose summary is late keeps its image at the scrub edge and is
|
||||
swapped to the summary once it arrives within the grace window."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._step_inputs["tc1"] = {"step_number": 1} # job pending
|
||||
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
image_scrub_depth=3,
|
||||
pending_grace_steps=3,
|
||||
)
|
||||
_simulate_turns(
|
||||
compressor, messages, [_initial_observation()] + [_step(i) for i in range(1, 4)]
|
||||
)
|
||||
|
||||
# Step 1 sits at the scrub edge with a pending summary: image retained.
|
||||
assert _has_image(messages[1])
|
||||
|
||||
# Summary arrives within grace; the next pass applies it.
|
||||
summarizer._summaries["tc1"] = "Recovered visual summary 1."
|
||||
_simulate_turns(compressor, messages, [_step(4)])
|
||||
|
||||
assert not _has_image(messages[1])
|
||||
assert {
|
||||
"type": "text",
|
||||
"text": "--- Historical Visual Transition ---\nRecovered visual summary 1.",
|
||||
} in messages[1].content
|
||||
|
||||
|
||||
def test_race_grace_exhausted_placeholder_never_backfilled(mock_context):
|
||||
"""After the grace window a pending step becomes a placeholder, and a late
|
||||
summary never mutates the frozen message again."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._step_inputs["tc1"] = {"step_number": 1} # pending forever (for now)
|
||||
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
image_scrub_depth=2,
|
||||
pending_grace_steps=1,
|
||||
)
|
||||
_simulate_turns(compressor, messages, [_step(i) for i in range(1, 5)])
|
||||
|
||||
# Depth 4 > K + grace = 3: grace exhausted, placeholder applied.
|
||||
step1_text = str(messages[0].content)
|
||||
assert not _has_image(messages[0])
|
||||
assert "[visual summary pending; evidence at DataEngine step 1]" in step1_text
|
||||
frozen_fingerprint = _content_fingerprint(messages[0])
|
||||
|
||||
# The summary arriving later must NOT backfill the frozen message.
|
||||
summarizer._summaries["tc1"] = "Too late summary."
|
||||
_simulate_turns(compressor, messages, [_step(5)])
|
||||
compressor.compress(messages)
|
||||
|
||||
assert _content_fingerprint(messages[0]) == frozen_fingerprint
|
||||
assert "Too late summary." not in str(messages[0].content)
|
||||
|
||||
|
||||
def test_race_failed_summary_becomes_unavailable_placeholder(mock_context):
|
||||
"""A failed summary is replaced by an unavailable placeholder at the scrub
|
||||
edge without waiting out the grace window."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._step_inputs["tc1"] = {"step_number": 1}
|
||||
summarizer._failed.add("tc1")
|
||||
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
image_scrub_depth=2,
|
||||
pending_grace_steps=5,
|
||||
)
|
||||
_simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)])
|
||||
|
||||
assert not _has_image(messages[0])
|
||||
assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(
|
||||
messages[0].content
|
||||
)
|
||||
|
||||
|
||||
def test_freeze_invariant_bytes_never_change(mock_context):
|
||||
"""Once a message crosses the scrub edge its content bytes never change,
|
||||
across many turns and late-arriving summaries."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
# Odd steps have summaries ready from the start; even steps stay pending.
|
||||
for i in range(1, 11):
|
||||
summarizer._step_inputs[f"tc{i}"] = {"step_number": i}
|
||||
if i % 2 == 1:
|
||||
summarizer._summaries[f"tc{i}"] = f"Ready summary {i}."
|
||||
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
image_scrub_depth=2,
|
||||
pending_grace_steps=1,
|
||||
)
|
||||
|
||||
frozen_fingerprints: dict[int, str] = {}
|
||||
|
||||
def check_frozen():
|
||||
for idx in compressor._frozen:
|
||||
fingerprint = _content_fingerprint(messages[idx])
|
||||
if idx in frozen_fingerprints:
|
||||
assert frozen_fingerprints[idx] == fingerprint, (
|
||||
f"frozen message {idx} mutated"
|
||||
)
|
||||
else:
|
||||
frozen_fingerprints[idx] = fingerprint
|
||||
|
||||
for i in range(1, 11):
|
||||
compressor.compress(messages)
|
||||
check_frozen()
|
||||
messages.append(AIMessage(content=f"Thinking about step {i}."))
|
||||
messages.append(_step(i))
|
||||
if i == 6:
|
||||
# Late summaries for already-frozen even steps: must be ignored.
|
||||
summarizer._summaries["tc2"] = "Late summary 2."
|
||||
summarizer._summaries["tc4"] = "Late summary 4."
|
||||
compressor.compress(messages)
|
||||
check_frozen()
|
||||
|
||||
assert len(frozen_fingerprints) >= 6
|
||||
# A frozen pending step carries the placeholder, not the late summary.
|
||||
assert "Late summary 2." not in " ".join(str(m.content) for m in messages)
|
||||
|
||||
|
||||
def test_xml_depth1_strip_matches_legacy_semantics(mock_context):
|
||||
"""Depth-1 XML stripping keeps combined-block prefixes and the live list."""
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(summarizer=None, prune_history_xml=True)
|
||||
|
||||
combined = ToolMessage(
|
||||
tool_call_id="tc1",
|
||||
name="click",
|
||||
content=[
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Tapped Settings.\n--- UI Element List ---\n[huge tree]",
|
||||
},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}},
|
||||
],
|
||||
)
|
||||
_simulate_turns(compressor, messages, [combined, _step(2)])
|
||||
|
||||
# Historical combined block: prefix retained, tree stripped (the exact
|
||||
# legacy ``compress_flash_messages`` product for this input).
|
||||
assert messages[0].content[0] == {"type": "text", "text": "Tapped Settings."}
|
||||
# Live observation keeps its UI list untouched.
|
||||
assert "Tree 2" in str(messages[1].content)
|
||||
|
||||
|
||||
def test_prune_history_xml_disabled_keeps_ui_lists(mock_context):
|
||||
"""With the switch off, UI lists survive both depth-1 and freeze scrubs."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._summaries["tc1"] = "Summary 1."
|
||||
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
prune_history_xml=False,
|
||||
image_scrub_depth=2,
|
||||
pending_grace_steps=0,
|
||||
)
|
||||
_simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)])
|
||||
|
||||
# Step 1 froze (summary applied) but its UI list is preserved.
|
||||
assert not _has_image(messages[0])
|
||||
assert "Tree 1" in str(messages[0].content)
|
||||
assert "Summary 1." in str(messages[0].content)
|
||||
|
||||
|
||||
def test_untracked_messages_are_never_touched(mock_context):
|
||||
"""Injected instructions and AI turns pass through untouched; a frozen
|
||||
image-only tool message falls back to non-empty content."""
|
||||
messages: list = [
|
||||
HumanMessage(content="[REAL-TIME INJECTED INSTRUCTION from user]: stay put"),
|
||||
]
|
||||
compressor = ScrubEdgeCompressor(summarizer=None, image_scrub_depth=2)
|
||||
|
||||
image_only = ToolMessage(
|
||||
tool_call_id="tc1",
|
||||
name="click",
|
||||
content=[{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}}],
|
||||
)
|
||||
_simulate_turns(
|
||||
compressor,
|
||||
messages,
|
||||
[image_only, AIMessage(content="thinking"), _step(2), _step(3)],
|
||||
)
|
||||
|
||||
assert messages[0].content == "[REAL-TIME INJECTED INSTRUCTION from user]: stay put"
|
||||
assert messages[2].content == "thinking"
|
||||
# Image-only tool message froze with the defensive fallback text.
|
||||
assert messages[1].content == [{"type": "text", "text": "Action completed."}]
|
||||
@@ -14,12 +14,12 @@
|
||||
|
||||
"""Unit tests for VisualStepSummarizer and Context Compressor in Flash profile."""
|
||||
|
||||
from unittest.mock import AsyncMock, Mock, call
|
||||
from unittest.mock import ANY, AsyncMock, Mock, call
|
||||
from uuid import uuid4
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
|
||||
from artemis.agents.flash.context_compressor import compress_flash_messages
|
||||
from artemis.agents.flash.context_compressor import ScrubEdgeCompressor
|
||||
from artemis.agents.flash.summarizer import VisualStepSummarizer
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.sdk.builders import Builders
|
||||
@@ -100,8 +100,94 @@ async def test_summarizer_retries_independently(mock_context):
|
||||
assert attempt_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_bounded_retry_marks_failed(mock_context):
|
||||
"""Persistent failures exit after 1 + retry_limit attempts with an explicit failed state."""
|
||||
mock_context.data_engine = Mock()
|
||||
summarizer = VisualStepSummarizer(mock_context, retry_limit=2)
|
||||
summarizer._retry_delays = (0.0,)
|
||||
attempt_count = 0
|
||||
|
||||
async def mock_ainvoke(messages):
|
||||
nonlocal attempt_count
|
||||
attempt_count += 1
|
||||
raise TimeoutError("Simulated permanent failure")
|
||||
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
|
||||
step_id = uuid4()
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="click",
|
||||
action_args={"target": [100, 100]},
|
||||
pre_img_bytes=b"pre1",
|
||||
post_img_bytes=b"post1",
|
||||
exec_outcome="Outcome 1",
|
||||
action_key="tc-fail",
|
||||
data_engine_step_id=step_id,
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
# Bounded exit: initial attempt + retry_limit retries, then explicit failure.
|
||||
assert attempt_count == 3
|
||||
assert summarizer.has_failed("tc-fail")
|
||||
assert not summarizer.has_summary("tc-fail")
|
||||
# Lossless semantics: a failed step still reads as pending so the
|
||||
# compressor keeps its source image instead of dropping evidence.
|
||||
assert summarizer.is_pending("tc-fail")
|
||||
|
||||
# DataEngine observability: pending on dispatch, failed on exhaustion.
|
||||
mock_context.data_engine.update_step_summary.assert_has_calls(
|
||||
[
|
||||
call(step_id, None, status="pending", source="visual_transition", model=ANY),
|
||||
call(step_id, None, status="failed", source="visual_transition", model=ANY),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_success_within_retry_budget_not_failed(mock_context):
|
||||
"""A summary that succeeds on a retry within the budget never enters failed state."""
|
||||
summarizer = VisualStepSummarizer(mock_context, retry_limit=3)
|
||||
summarizer._retry_delays = (0.0,)
|
||||
attempt_count = 0
|
||||
|
||||
async def mock_ainvoke(messages):
|
||||
nonlocal attempt_count
|
||||
attempt_count += 1
|
||||
if attempt_count < 3:
|
||||
raise TimeoutError("Transient failure")
|
||||
return AIMessage(content="[Step 1 State: Finally worked.]")
|
||||
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="click",
|
||||
action_args={"target": [100, 100]},
|
||||
pre_img_bytes=b"pre1",
|
||||
post_img_bytes=b"post1",
|
||||
exec_outcome="Outcome 1",
|
||||
action_key="tc-ok",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
assert attempt_count == 3
|
||||
assert summarizer.has_summary("tc-ok")
|
||||
assert not summarizer.has_failed("tc-ok")
|
||||
|
||||
|
||||
def test_context_compressor_with_ready_summaries(mock_context):
|
||||
"""Verify that ready summaries replace past image blocks and prune outdated XML."""
|
||||
"""Ready summaries replace past image blocks and outdated XML is pruned.
|
||||
|
||||
Rewritten against ``ScrubEdgeCompressor`` (M5; the legacy
|
||||
``compress_flash_messages`` was removed) with ``image_scrub_depth=1`` so
|
||||
every historical message is scrubbed immediately, mirroring the legacy
|
||||
timing this test was written for. Summaries are keyed by legacy ordinals
|
||||
to keep the older-caller fallback path covered.
|
||||
"""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._summaries[1] = "[Step 1 State: Tapped Search bar. Keyboard appeared.]"
|
||||
summarizer._summaries[2] = "[Step 2 State: Typed 'flights'. Suggestions displayed.]"
|
||||
@@ -134,7 +220,13 @@ def test_context_compressor_with_ready_summaries(mock_context):
|
||||
),
|
||||
]
|
||||
|
||||
compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
prune_history_xml=True,
|
||||
image_scrub_depth=1,
|
||||
pending_grace_steps=0,
|
||||
)
|
||||
compressor.compress(messages)
|
||||
|
||||
# Initial HumanMessage: Historical initial image pruned, task text preserved
|
||||
assert len(messages[0].content) == 1
|
||||
@@ -155,88 +247,15 @@ def test_context_compressor_with_ready_summaries(mock_context):
|
||||
assert "Live Latest Tree 3" in messages[2].content[2]["text"]
|
||||
|
||||
|
||||
def test_context_compressor_graceful_fallback(mock_context):
|
||||
"""Verify that when a summary is not ready yet, it cleanly falls back to standard pruned state."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
# Step 1 has no summary in cache
|
||||
|
||||
messages = [
|
||||
HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "Task: Search"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
|
||||
]
|
||||
),
|
||||
ToolMessage(
|
||||
tool_call_id="tc1",
|
||||
name="click",
|
||||
content=[
|
||||
{"type": "text", "text": "Action 'click' completed."},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
|
||||
|
||||
# Turn 1 should simply have image omitted (standard fallback)
|
||||
assert len(messages[0].content) == 1
|
||||
assert messages[0].content[0]["text"] == "Task: Search"
|
||||
|
||||
# Turn 2 (Latest) preserves image
|
||||
assert len(messages[1].content) == 2
|
||||
assert messages[1].content[1]["type"] == "image_url"
|
||||
|
||||
|
||||
def test_context_compressor_backfills_late_summary_idempotently(mock_context):
|
||||
"""A pending image remains until its late summary replaces it."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._step_inputs["action-1"] = {"step_number": 1}
|
||||
messages = [
|
||||
HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "Task: Search"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
|
||||
]
|
||||
),
|
||||
ToolMessage(
|
||||
tool_call_id="action-1",
|
||||
name="click",
|
||||
content=[
|
||||
{"type": "text", "text": "Tapped Search."},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
|
||||
],
|
||||
),
|
||||
ToolMessage(
|
||||
tool_call_id="action-2",
|
||||
name="input_text",
|
||||
content=[
|
||||
{"type": "text", "text": "Typed query."},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_3"}},
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
compress_flash_messages(messages, summarizer=summarizer)
|
||||
assert any(block["type"] == "image_url" for block in messages[1].content)
|
||||
assert len(messages[1].content) == 2
|
||||
|
||||
summarizer._summaries["action-1"] = "Search field focused; keyboard appeared."
|
||||
compress_flash_messages(messages, summarizer=summarizer)
|
||||
compress_flash_messages(messages, summarizer=summarizer)
|
||||
|
||||
generated_blocks = [
|
||||
block
|
||||
for block in messages[1].content
|
||||
if block.get("text", "").startswith("--- Historical Visual Transition ---")
|
||||
]
|
||||
assert len(generated_blocks) == 1
|
||||
assert generated_blocks[0]["text"].endswith("Search field focused; keyboard appeared.")
|
||||
assert all(block["type"] != "image_url" for block in messages[1].content)
|
||||
|
||||
|
||||
def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context):
|
||||
"""Generic tool messages cannot shift action summaries onto the wrong result."""
|
||||
"""Generic tool messages cannot shift action summaries onto the wrong result.
|
||||
|
||||
(Rewritten against ``ScrubEdgeCompressor`` in M5. The legacy tests for
|
||||
pending-summary fallback and late-summary backfill were removed with
|
||||
``compress_flash_messages``: the scrub edge covers pending handling in
|
||||
``test_flash_scrub_edge.py``, and late backfill is now forbidden by the
|
||||
freeze invariant rather than supported.)
|
||||
"""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._summaries["click-id"] = "Click-specific visual memory."
|
||||
summarizer._summaries["type-id"] = "Typing-specific visual memory."
|
||||
@@ -265,7 +284,12 @@ def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context)
|
||||
),
|
||||
]
|
||||
|
||||
compress_flash_messages(messages, summarizer=summarizer)
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer,
|
||||
image_scrub_depth=1,
|
||||
pending_grace_steps=0,
|
||||
)
|
||||
compressor.compress(messages)
|
||||
|
||||
assert "Click-specific visual memory." in str(messages[0].content)
|
||||
assert "Typing-specific visual memory." not in str(messages[0].content)
|
||||
@@ -274,35 +298,6 @@ def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context)
|
||||
assert messages[2].content[-1]["type"] == "image_url"
|
||||
|
||||
|
||||
def test_context_compressor_preserves_text_before_embedded_ui_list(mock_context):
|
||||
"""Pruning a combined XML block must retain its action result prefix."""
|
||||
messages = [
|
||||
ToolMessage(
|
||||
tool_call_id="old",
|
||||
name="click",
|
||||
content=[
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Tapped Settings.\n--- UI Element List ---\n[huge tree]",
|
||||
},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}},
|
||||
],
|
||||
),
|
||||
ToolMessage(
|
||||
tool_call_id="live",
|
||||
name="click",
|
||||
content=[
|
||||
{"type": "text", "text": "Opened page."},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,B"}},
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
compress_flash_messages(messages, prune_history_xml=True)
|
||||
|
||||
assert messages[0].content == [{"type": "text", "text": "Tapped Settings."}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_keys_actions_independently_of_turn_number(mock_context):
|
||||
"""Multiple actions from one LLM turn keep independent inputs and summaries."""
|
||||
@@ -344,17 +339,229 @@ async def test_summarizer_keys_actions_independently_of_turn_number(mock_context
|
||||
|
||||
assert summarizer.get_summary("tc-1") == "First action memory."
|
||||
assert summarizer.get_summary("tc-2") == "Second action memory."
|
||||
assert summarizer._step_inputs["tc-1"]["data_engine_step_id"] == first_step_id
|
||||
assert summarizer._step_inputs["tc-2"]["data_engine_step_id"] == second_step_id
|
||||
# M1 keying: jobs are canonical under the DataEngine step id; the
|
||||
# tool_call_id stays queryable as an alias.
|
||||
assert summarizer.get_job_payload("tc-1")["data_engine_step_id"] == first_step_id
|
||||
assert summarizer.get_job_payload("tc-2")["data_engine_step_id"] == second_step_id
|
||||
assert summarizer.resolve_key("tc-1") == str(first_step_id)
|
||||
mock_context.data_engine.update_step_summary.assert_has_calls(
|
||||
[
|
||||
call(first_step_id, "First action memory."),
|
||||
call(second_step_id, "Second action memory."),
|
||||
call(
|
||||
first_step_id,
|
||||
"First action memory.",
|
||||
status="ready",
|
||||
source="visual_transition",
|
||||
model=ANY,
|
||||
),
|
||||
call(
|
||||
second_step_id,
|
||||
"Second action memory.",
|
||||
status="ready",
|
||||
source="visual_transition",
|
||||
model=ANY,
|
||||
),
|
||||
],
|
||||
any_order=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_frame_dispatch_uses_single_image_prompt_variant(mock_context):
|
||||
"""A missing after-frame (every Pro step, by design) selects the
|
||||
single-image prompt: decision frame + red marker, no AFTER ACTION section."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
captured: list = []
|
||||
|
||||
async def mock_ainvoke(messages):
|
||||
captured.append(messages)
|
||||
return AIMessage(
|
||||
content=(
|
||||
"In Step 1, I tapped the 'Battery' row marked in red in the"
|
||||
" Settings list, which showed 'Network & internet' and 'Display'."
|
||||
)
|
||||
)
|
||||
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="click",
|
||||
action_args={"target": [100, 200]},
|
||||
pre_img_bytes=b"decision_frame",
|
||||
post_img_bytes=None,
|
||||
exec_outcome="executed",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
assert summarizer.has_summary(1)
|
||||
system_text = str(captured[0][0].content)
|
||||
human_text = str(captured[0][1].content)
|
||||
assert "Exactly ONE screenshot" in system_text
|
||||
assert "no independent post-action evidence" in system_text
|
||||
assert "DECISION FRAME" in human_text
|
||||
assert "AFTER ACTION SCREEN" not in human_text
|
||||
assert "--- [2]" not in human_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dual_frame_dispatch_keeps_transition_prompt(mock_context):
|
||||
"""Both frames present (Flash path): the classic BEFORE/AFTER transition
|
||||
prompt and section labels are unchanged."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
captured: list = []
|
||||
|
||||
async def mock_ainvoke(messages):
|
||||
captured.append(messages)
|
||||
return AIMessage(
|
||||
content=(
|
||||
"In Step 1, I tapped the 'Settings' gear icon marked in red;"
|
||||
" the main Settings menu opened showing 'Apps'."
|
||||
)
|
||||
)
|
||||
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="click",
|
||||
action_args={"target": [100, 200]},
|
||||
pre_img_bytes=b"pre",
|
||||
post_img_bytes=b"post",
|
||||
exec_outcome="executed",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
assert summarizer.has_summary(1)
|
||||
system_text = str(captured[0][0].content)
|
||||
human_text = str(captured[0][1].content)
|
||||
assert "Before/After screenshots" in system_text
|
||||
assert "--- [1] BEFORE ACTION SCREEN (Action Marked Visually in Red) ---" in human_text
|
||||
assert "--- [2] AFTER ACTION SCREEN ---" in human_text
|
||||
assert "DECISION FRAME" not in human_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_echo_output_fails_attempt_then_recovers_on_retry(mock_context):
|
||||
"""An input-marker echo is a failed attempt (not a summary): the bounded
|
||||
retry regenerates and the recovered text lands normally."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._retry_delays = (0.0,)
|
||||
attempts = 0
|
||||
|
||||
async def mock_ainvoke(messages):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
if attempts == 1:
|
||||
return AIMessage(content="--- [2] AFTER ACTION SCREEN ---")
|
||||
return AIMessage(
|
||||
content=(
|
||||
"In Step 1, I tapped the search field marked in red; the field"
|
||||
" showed placeholder text 'Search here'."
|
||||
)
|
||||
)
|
||||
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="click",
|
||||
action_args={"target": [1, 2]},
|
||||
pre_img_bytes=b"pre",
|
||||
post_img_bytes=b"post",
|
||||
exec_outcome="executed",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
assert attempts == 2
|
||||
assert summarizer.has_summary(1)
|
||||
assert "Search here" in summarizer.get_summary(1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_degenerate_outputs_exhaust_to_failed(mock_context):
|
||||
"""Echo/too-short/too-long outputs never land as summaries; persistent
|
||||
degeneration exhausts the bounded retries into the explicit failed state."""
|
||||
summarizer = VisualStepSummarizer(mock_context, retry_limit=1)
|
||||
summarizer._retry_delays = (0.0,)
|
||||
|
||||
async def mock_ainvoke(messages):
|
||||
text = str(messages)
|
||||
if "Step 1" in text:
|
||||
return AIMessage(content="--- [1] DECISION FRAME: SCREEN AT ACTION TIME ---")
|
||||
if "Step 2" in text:
|
||||
return AIMessage(content="x" * 2000)
|
||||
return AIMessage(content="ok.")
|
||||
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
|
||||
for step in (1, 2, 3):
|
||||
summarizer.dispatch(
|
||||
step_number=step,
|
||||
action_name="click",
|
||||
action_args={"target": [1, 2]},
|
||||
pre_img_bytes=b"pre",
|
||||
post_img_bytes=None,
|
||||
exec_outcome="executed",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
for step in (1, 2, 3):
|
||||
assert summarizer.has_failed(step), f"step {step} should have failed"
|
||||
assert not summarizer.has_summary(step)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_lens_call_meters_llm_usage_without_touching_context_base(mock_context):
|
||||
"""The raw-model lens bypass records an llm_usage trace, but its tiny
|
||||
prompt never overwrites the session's last_prompt_tokens (the compaction
|
||||
thresholds' live context base)."""
|
||||
from artemis.services.token_meter import get_meter
|
||||
|
||||
engine = Mock()
|
||||
engine.current_session_id = "lens-meter-session"
|
||||
engine.current_step_id = None
|
||||
mock_context.data_engine = engine
|
||||
meter = get_meter("lens-meter-session")
|
||||
meter.last_prompt_tokens = 777
|
||||
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(
|
||||
return_value=AIMessage(
|
||||
content=(
|
||||
"In Step 1, I tapped the 'Battery' row marked in red; the list"
|
||||
" showed 'Display' and 'Battery'."
|
||||
),
|
||||
usage_metadata={"input_tokens": 120, "output_tokens": 30, "total_tokens": 150},
|
||||
)
|
||||
)
|
||||
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="click",
|
||||
action_args={"target": [1, 2]},
|
||||
pre_img_bytes=b"pre",
|
||||
post_img_bytes=b"post",
|
||||
exec_outcome="executed",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
usage_calls = [
|
||||
c
|
||||
for c in engine.record_trace.call_args_list
|
||||
if c.kwargs.get("name") == "llm_usage"
|
||||
]
|
||||
assert usage_calls, "the raw lens bypass must record an llm_usage trace"
|
||||
payload = usage_calls[0].kwargs["payload"]
|
||||
assert payload["source"].startswith("lens:visual_transition:")
|
||||
assert payload["prompt_tokens"] == 120
|
||||
assert meter.last_prompt_tokens == 777
|
||||
|
||||
|
||||
def test_flash_config_and_builder():
|
||||
"""Verify Flash profile configuration model and SDK builder fluent API."""
|
||||
cfg = Builders.AgentConfig.with_flash_config(
|
||||
@@ -395,7 +602,13 @@ async def test_data_engine_integration_with_step_summarizer(mock_context):
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
mock_de.update_step_summary.assert_called_once_with(1, "[Step 1 State: Input text confirmed.]")
|
||||
mock_de.update_step_summary.assert_called_once_with(
|
||||
1,
|
||||
"[Step 1 State: Input text confirmed.]",
|
||||
status="ready",
|
||||
source="visual_transition",
|
||||
model=ANY,
|
||||
)
|
||||
|
||||
|
||||
def test_draw_action_overlay_rendering():
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""HistoricalStateHintPromptComponent (M4): perceptual-hash hint + silence rules.
|
||||
|
||||
Division of labor under test: the hint fires only for matches *older* than the
|
||||
recent window (returned-to-an-early-state regime); any match within the last 3
|
||||
steps silences it entirely — that regime belongs to the pixel-level
|
||||
ScreenshotSimilarityPromptComponent.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import io
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
from artemis.agents.operator.prompts import HistoricalStateHintPromptComponent, PromptBuilder
|
||||
from artemis.utils.image_hash import dhash_hex
|
||||
|
||||
|
||||
def _jpeg(draw_fn) -> bytes:
|
||||
img = Image.new("RGB", (270, 600), "white")
|
||||
draw_fn(ImageDraw.Draw(img))
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=85)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _screen_a(d):
|
||||
d.rectangle([20, 40, 250, 90], fill="#3355ff")
|
||||
d.rectangle([60, 450, 210, 510], fill="#22aa66")
|
||||
|
||||
|
||||
def _screen_b(d):
|
||||
for y in range(60, 560, 80):
|
||||
d.rectangle([10, y, 260, y + 50], fill="#dddddd")
|
||||
|
||||
|
||||
SCREEN_A = _jpeg(_screen_a)
|
||||
SCREEN_B = _jpeg(_screen_b)
|
||||
HASH_A = dhash_hex(SCREEN_A)
|
||||
HASH_B = dhash_hex(SCREEN_B)
|
||||
B64_A = base64.b64encode(SCREEN_A).decode("utf-8")
|
||||
|
||||
|
||||
def _step(number: int, post_hash: str | None) -> dict:
|
||||
meta = {"post_image_dhash": post_hash} if post_hash else {}
|
||||
return {"step_number": number, "extra_metadata": meta}
|
||||
|
||||
|
||||
def _cfg(similarity_hint=True, max_distance=8):
|
||||
return SimpleNamespace(
|
||||
memory=SimpleNamespace(
|
||||
transcript=SimpleNamespace(
|
||||
similarity_hint=similarity_hint,
|
||||
similarity_max_distance=max_distance,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _run(steps, b64=B64_A, cfg=None):
|
||||
builder = PromptBuilder()
|
||||
component = HistoricalStateHintPromptComponent()
|
||||
with patch("artemis.config.load_agent_config", return_value=cfg or _cfg()):
|
||||
await component(
|
||||
builder, MagicMock(), MagicMock(), latest_screenshot_b64=b64, steps=steps
|
||||
)
|
||||
return [p for p in builder.human_parts if isinstance(p, str)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hint_fires_on_old_matching_step():
|
||||
steps = [
|
||||
_step(1, HASH_B),
|
||||
_step(2, HASH_A), # old match
|
||||
_step(3, HASH_B),
|
||||
_step(4, HASH_B),
|
||||
_step(5, HASH_B),
|
||||
_step(6, HASH_B),
|
||||
]
|
||||
parts = await _run(steps)
|
||||
assert len(parts) == 1
|
||||
assert "Historical state hint" in parts[0]
|
||||
assert "Step 2" in parts[0]
|
||||
assert "recall_history" in parts[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_silent_when_recent_step_also_matches():
|
||||
steps = [
|
||||
_step(1, HASH_A), # old match exists...
|
||||
_step(2, HASH_B),
|
||||
_step(3, HASH_B),
|
||||
_step(4, HASH_B),
|
||||
_step(5, HASH_A), # ...but a recent (last-3) match silences the hint
|
||||
_step(6, HASH_B),
|
||||
]
|
||||
assert await _run(steps) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_silent_without_any_match():
|
||||
steps = [_step(n, HASH_B) for n in range(1, 7)]
|
||||
assert await _run(steps) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_silent_when_config_disabled():
|
||||
steps = [
|
||||
_step(1, HASH_A),
|
||||
_step(2, HASH_B),
|
||||
_step(3, HASH_B),
|
||||
_step(4, HASH_B),
|
||||
_step(5, HASH_B),
|
||||
_step(6, HASH_B),
|
||||
]
|
||||
assert await _run(steps, cfg=_cfg(similarity_hint=False)) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_silent_with_too_few_steps():
|
||||
# Everything within the recent window belongs to the pixel-level note.
|
||||
assert await _run([_step(1, HASH_A), _step(2, HASH_A)]) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tie_prefers_most_recent_old_step():
|
||||
steps = [
|
||||
_step(1, HASH_A),
|
||||
_step(2, HASH_A), # same distance, more recent
|
||||
_step(3, HASH_B),
|
||||
_step(4, HASH_B),
|
||||
_step(5, HASH_B),
|
||||
_step(6, HASH_B),
|
||||
]
|
||||
parts = await _run(steps)
|
||||
assert len(parts) == 1 and "Step 2" in parts[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_steps_without_hashes_are_skipped():
|
||||
steps = [
|
||||
_step(1, None),
|
||||
_step(2, None),
|
||||
_step(3, None),
|
||||
_step(4, None),
|
||||
_step(5, None),
|
||||
_step(6, None),
|
||||
]
|
||||
assert await _run(steps) == []
|
||||
|
||||
|
||||
def test_record_step_stamps_perceptual_hashes(tmp_path):
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
setup = MagicMock()
|
||||
setup.traces_path = str(tmp_path)
|
||||
mock_ctx.execution_setup = setup
|
||||
mock_ctx.device = None
|
||||
engine = DataEngine(mock_ctx)
|
||||
engine.start_session("dhash stamping test")
|
||||
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=SCREEN_A,
|
||||
post_screenshot_bytes=SCREEN_B,
|
||||
summary="two distinct screens",
|
||||
)
|
||||
# Same pre/post bytes: post_image_name is nulled but the post-action
|
||||
# screen IS the pre screen, so the post hash mirrors the pre hash.
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=SCREEN_A,
|
||||
post_screenshot_bytes=SCREEN_A,
|
||||
summary="screen unchanged",
|
||||
)
|
||||
for t in list(engine._pending_threads):
|
||||
t.join()
|
||||
|
||||
steps = engine.storage.get_steps(engine.current_session_id)
|
||||
meta1 = steps[0].extra_metadata
|
||||
assert meta1["pre_image_dhash"] == HASH_A
|
||||
assert meta1["post_image_dhash"] == HASH_B
|
||||
meta2 = steps[1].extra_metadata
|
||||
assert meta2["pre_image_dhash"] == HASH_A
|
||||
assert meta2["post_image_dhash"] == HASH_A
|
||||
@@ -344,7 +344,7 @@ async def test_history_analyzer_integration_with_task_tree():
|
||||
mock_path.exists.return_value = True
|
||||
mock_path.read_text.return_value = fake_plan
|
||||
|
||||
# Mock build_plan_and_history
|
||||
# Mock the policy-table compiled-history renderer (M4)
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.history_analyzer.history_analyzer.get_llm",
|
||||
@@ -355,23 +355,23 @@ async def test_history_analyzer_integration_with_task_tree():
|
||||
return_value=mock_path,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.history_analyzer.history_analyzer.build_plan_and_history"
|
||||
"artemis.agents.history_analyzer.history_analyzer.build_history_for"
|
||||
) as mock_build,
|
||||
):
|
||||
mock_build.return_value = "Mocked operation history"
|
||||
analyzer = HistoryAnalyzer(mock_ctx)
|
||||
await analyzer.run("query")
|
||||
|
||||
# Verify build_plan_and_history call args
|
||||
# Verify build_history_for call args
|
||||
mock_build.assert_called_once()
|
||||
args, kwargs = mock_build.call_args
|
||||
assert args[0] == fake_plan
|
||||
assert args[1] == steps
|
||||
assert args[0] == "history_analyzer"
|
||||
assert args[1] == fake_plan
|
||||
assert args[2] == steps
|
||||
|
||||
import hashlib
|
||||
|
||||
expected_hash = hashlib.md5(b"Active Subgoal").hexdigest()
|
||||
assert args[2] == expected_hash
|
||||
assert args[3] == expected_hash
|
||||
|
||||
assert kwargs.get("last_n_detailed") == 1
|
||||
assert kwargs.get("min_summaries") == len(steps)
|
||||
assert kwargs.get("engine") is mock_ctx.data_engine
|
||||
@@ -16,9 +16,15 @@ import json
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.config.agent import MemoryTranscriptConfig
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
# These tests exercise the legacy 2-message prompt path. Since M5 the
|
||||
# transcript path is the default, so flag-off semantics must be requested
|
||||
# explicitly (never via the config default).
|
||||
LEGACY_TRANSCRIPT = MemoryTranscriptConfig(enabled=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_node_fast_path():
|
||||
@@ -58,7 +64,7 @@ async def test_operator_node_fast_path():
|
||||
|
||||
# Patch dependencies
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# Verify state was updated with action
|
||||
@@ -77,7 +83,7 @@ async def test_perform_action_validation():
|
||||
from artemis.graph.state import State
|
||||
|
||||
mock_ctx = MagicMock()
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.indexed_points = [[200, 300], [400, 500]]
|
||||
@@ -114,6 +120,7 @@ async def test_perform_action_validation():
|
||||
"target_bounds": "[100,100][300,500]",
|
||||
"target_resource_id": "btn_1",
|
||||
"target_class": "android.widget.Button",
|
||||
"target_label_source": "index",
|
||||
}
|
||||
]
|
||||
|
||||
@@ -143,6 +150,7 @@ async def test_perform_action_validation():
|
||||
"target_bounds": "[300,400][500,600]",
|
||||
"target_resource_id": "input_1",
|
||||
"target_class": "android.widget.EditText",
|
||||
"target_label_source": "index",
|
||||
}
|
||||
|
||||
# Test swipe direction
|
||||
@@ -170,6 +178,7 @@ async def test_perform_action_validation():
|
||||
"target_bounds": "[300,400][500,600]",
|
||||
"target_resource_id": "input_1",
|
||||
"target_class": "android.widget.EditText",
|
||||
"target_label_source": "index",
|
||||
}
|
||||
]
|
||||
|
||||
@@ -222,6 +231,59 @@ async def test_perform_action_validation():
|
||||
assert "Unsupported app management action" in err
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bare_coordinate_click_enriched_by_hit_test():
|
||||
"""A coordinate click hit tests the pre-action frame for element semantics."""
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from unittest.mock import MagicMock
|
||||
from artemis.graph.state import State
|
||||
|
||||
mock_ctx = MagicMock()
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.indexed_elements = [
|
||||
{
|
||||
"center": [550, 1440],
|
||||
"text": "Confirm",
|
||||
"bounds": [500, 1400, 600, 1480],
|
||||
"class": "android.widget.Button",
|
||||
"resource_id": "btn_confirm",
|
||||
"is_ocr": False,
|
||||
}
|
||||
]
|
||||
|
||||
# Normalized (500, 600) on the default 1080x2400 device -> pixel (540, 1440),
|
||||
# which falls inside the "Confirm" button bounds.
|
||||
actions, err = node._translate_and_validate_tool(
|
||||
{"name": "click", "args": {"target": [500, 600]}}, mock_state
|
||||
)
|
||||
assert err is None
|
||||
assert actions == [
|
||||
{
|
||||
"action": "tap",
|
||||
"coordinates": [540, 1440],
|
||||
"normalized_coordinates": [500, 600],
|
||||
"times": 1,
|
||||
"delay_ms": 100,
|
||||
"target_text": "Confirm",
|
||||
"target_bounds": [500, 1400, 600, 1480],
|
||||
"target_resource_id": "btn_confirm",
|
||||
"target_class": "android.widget.Button",
|
||||
"target_label_source": "hit_test",
|
||||
}
|
||||
]
|
||||
|
||||
# Graceful degradation: no perception data at all -> bare coordinates, "none".
|
||||
mock_state.indexed_elements = []
|
||||
actions, err = node._translate_and_validate_tool(
|
||||
{"name": "click", "args": {"target": [500, 600]}}, mock_state
|
||||
)
|
||||
assert err is None
|
||||
assert actions[0]["target_text"] is None
|
||||
assert actions[0]["target_label_source"] == "none"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_node_multiple_actions():
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
@@ -258,7 +320,7 @@ async def test_operator_node_multiple_actions():
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
update_dict = node_update
|
||||
@@ -302,7 +364,7 @@ async def test_operator_node_no_record_step():
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
mock_ctx.data_engine.record_step.assert_not_called()
|
||||
@@ -352,7 +414,7 @@ async def test_operator_dynamic_prompt():
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx, prompt_components=[MockComponent()])
|
||||
node = OperatorNode(mock_ctx, prompt_components=[MockComponent()], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
|
||||
@@ -488,7 +550,7 @@ async def test_operator_history_compression():
|
||||
patch("pathlib.Path.exists", return_value=True),
|
||||
patch("pathlib.Path.read_text", mock_read_text),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, last_n_detailed=5)
|
||||
node = OperatorNode(mock_ctx, last_n_detailed=5, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
|
||||
@@ -506,7 +568,7 @@ async def test_operator_loop_detection():
|
||||
mock_state.subagent_calls = ["ask_explorer", "ask_explorer", "ask_explorer"]
|
||||
mock_state.initial_goal = "Test goal"
|
||||
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
node_update = await node(mock_state)
|
||||
@@ -568,7 +630,7 @@ async def test_operator_tracks_subagent_calls():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool])
|
||||
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
update_dict = node_update
|
||||
@@ -660,7 +722,7 @@ async def test_operator_defer_action_for_gathering():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool])
|
||||
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# Subagent tool should have been executed
|
||||
@@ -683,7 +745,7 @@ async def test_wait_actions_translation():
|
||||
from artemis.graph.state import State
|
||||
|
||||
mock_ctx = MagicMock()
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
mock_state = MagicMock(spec=State)
|
||||
|
||||
# 1. Test wait_for_delay translation
|
||||
@@ -708,7 +770,7 @@ async def test_long_press_action_translation():
|
||||
from artemis.graph.state import State
|
||||
|
||||
mock_ctx = MagicMock()
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.indexed_points = [[200, 300], [400, 500]]
|
||||
mock_state.indexed_elements = [
|
||||
@@ -744,6 +806,7 @@ async def test_long_press_action_translation():
|
||||
"target_bounds": "[100,100][300,500]",
|
||||
"target_resource_id": "btn_1",
|
||||
"target_class": "android.widget.Button",
|
||||
"target_label_source": "index",
|
||||
}
|
||||
]
|
||||
|
||||
@@ -765,6 +828,7 @@ async def test_long_press_action_translation():
|
||||
"target_bounds": None,
|
||||
"target_resource_id": None,
|
||||
"target_class": None,
|
||||
"target_label_source": "none",
|
||||
}
|
||||
]
|
||||
|
||||
@@ -844,7 +908,7 @@ async def test_operator_no_defer_for_note_updating():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_update_note_tool])
|
||||
node = OperatorNode(mock_ctx, tools=[mock_update_note_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# The update_note tool should have been executed immediately
|
||||
@@ -934,7 +998,7 @@ async def test_operator_defer_for_note_reading():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_read_note_tool])
|
||||
node = OperatorNode(mock_ctx, tools=[mock_read_note_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# The read_note tool should have been executed
|
||||
@@ -1022,7 +1086,7 @@ async def test_operator_no_defer_for_note_appending():
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_append_note_tool])
|
||||
node = OperatorNode(mock_ctx, tools=[mock_append_note_tool], transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# The append_note tool should have been executed immediately
|
||||
@@ -1111,7 +1175,7 @@ async def test_operator_background_tasks_prompt_injection():
|
||||
side_effect=mock_invoke_llm_loop,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# Assert that the captured messages (prompts) contain the background tasks section
|
||||
@@ -1202,7 +1266,7 @@ async def test_operator_task_plan_warning_prompt_injection():
|
||||
side_effect=mock_invoke_llm_loop_1,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
full_content_1 = ""
|
||||
@@ -1255,7 +1319,7 @@ async def test_operator_task_plan_warning_prompt_injection():
|
||||
side_effect=mock_invoke_llm_loop_2,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
full_content_2 = ""
|
||||
@@ -1322,7 +1386,7 @@ async def test_operator_tool_limit_exceeded_warning():
|
||||
side_effect=mock_invoke_llm_loop,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
full_content = ""
|
||||
@@ -1380,7 +1444,7 @@ async def test_operator_fallback_function_call_parsing():
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# Verify state was updated with parsed click action
|
||||
@@ -1402,7 +1466,7 @@ async def test_operator_swipe_translation():
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_state = MagicMock()
|
||||
mock_state.operator_raw_data = {"width": 1080, "height": 2400}
|
||||
node = OperatorNode(mock_ctx)
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
|
||||
# 1. Directional swipe
|
||||
actions, err = node._translate_and_validate_tool(
|
||||
|
||||
@@ -24,7 +24,7 @@ def test_operator_prompt_contract_matches_note_runtime_semantics():
|
||||
|
||||
assert "write-through tools `update_note` and `append_note`" in prompt
|
||||
assert "may accompany at most one Turn-Ending Action" in prompt
|
||||
assert "`read_note`, `list_notes`, and `save_note`" in prompt
|
||||
assert "`read_note`, `list_notes`, `save_note`, and `recall_history`" in prompt
|
||||
assert "Do NOT submit a Turn-Ending Action at the same time" not in prompt
|
||||
assert "memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`" not in prompt
|
||||
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Operator dual prompt path (M2): legacy golden parity and transcript mode.
|
||||
|
||||
Flag off (``agent.memory.transcript.enabled=false``): the legacy 2-message
|
||||
build must stay byte-for-byte identical — pinned by SHA-256 goldens recorded
|
||||
against the pre-split implementation.
|
||||
|
||||
Flag on: ``_build_prompt`` renders ``S + F + A + tail`` from the session
|
||||
transcript ledger; the turn's messages are staged after the tool loop and
|
||||
committed (with step key and validator result) at the next build.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import re
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.agents.operator.prompt_builder import load_operator_prompts
|
||||
from artemis.agents.operator.prompts import (
|
||||
PLAN_HISTORY_STATIC_POINTER,
|
||||
PLAN_HISTORY_TEMPLATE_SECTION,
|
||||
PromptBuilder,
|
||||
TemplatePromptComponent,
|
||||
render_transcript_static_system,
|
||||
)
|
||||
from artemis.config.agent import MemoryTranscriptConfig
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.memory.step_memory import StepMemoryService
|
||||
from artemis.memory.transcript import (
|
||||
EXECUTION_RESULT_MARKER,
|
||||
PLAN_RECITATION_MARKER,
|
||||
RESTORED_HISTORY_HEADER,
|
||||
TranscriptLedger,
|
||||
)
|
||||
|
||||
# SHA-256 of the legacy system message rendered by TemplatePromptComponent
|
||||
# with the fixed inputs below. Originally recorded BEFORE the M2 template
|
||||
# split; re-recorded once in M5 (2026-09-01) after the only intended template
|
||||
# change since — removing the Short-Term Memory instruction block from
|
||||
# operator.json (the <short_term_memory> relay was removed end to end per
|
||||
# pro-context-memory-redesign §3.3). Any other drift here means the flag-off
|
||||
# path is no longer byte-identical.
|
||||
GOLDEN_EMPTY_PLAN = "c45aec7d67069dad3a195534dd3c97dadb48454ca186a40eb471d5f9136589fa"
|
||||
GOLDEN_SENTINEL_PLAN = "670580efd48f237c3257d996a814145fbef4ec0bb467aa4dfa6eb2db4a613826"
|
||||
|
||||
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
|
||||
|
||||
|
||||
async def _render_legacy_system(plan_and_history: str) -> str:
|
||||
builder = PromptBuilder()
|
||||
component = TemplatePromptComponent()
|
||||
ctx = SimpleNamespace(execution_setup=None, actuator=None, data_engine=None)
|
||||
state = SimpleNamespace(initial_goal="GOLDEN_FIXED_GOAL")
|
||||
await component(
|
||||
builder,
|
||||
state,
|
||||
ctx,
|
||||
prompts=load_operator_prompts(),
|
||||
template_name="main_template",
|
||||
plan_and_history=plan_and_history,
|
||||
)
|
||||
return "".join(builder.system_parts)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_legacy_prompt_bytes_match_pre_split_golden():
|
||||
empty = await _render_legacy_system("No plan or history yet.")
|
||||
sentinel = await _render_legacy_system("PLAN_AND_HISTORY_SENTINEL\nline2")
|
||||
assert hashlib.sha256(empty.encode("utf-8")).hexdigest() == GOLDEN_EMPTY_PLAN
|
||||
assert hashlib.sha256(sentinel.encode("utf-8")).hexdigest() == GOLDEN_SENTINEL_PLAN
|
||||
|
||||
|
||||
def test_main_template_contains_exactly_one_plan_history_section():
|
||||
template = load_operator_prompts()["main_template"]
|
||||
assert template.count(PLAN_HISTORY_TEMPLATE_SECTION) == 1
|
||||
|
||||
|
||||
def test_transcript_static_system_is_stable_and_carries_no_history():
|
||||
ctx = SimpleNamespace(execution_setup=None, actuator=None, data_engine=None)
|
||||
state = SimpleNamespace(initial_goal="STATIC_GOAL")
|
||||
prompts = load_operator_prompts()
|
||||
|
||||
first = render_transcript_static_system(prompts, ctx, state)
|
||||
second = render_transcript_static_system(prompts, ctx, state)
|
||||
assert first == second # byte-stable S region
|
||||
|
||||
assert "STATIC_GOAL" in first
|
||||
# The volatile section is replaced by the static pointer...
|
||||
assert PLAN_HISTORY_STATIC_POINTER.splitlines()[1] in first
|
||||
# ...and no unrendered slot or trailing observation header survives.
|
||||
assert "{{" not in first
|
||||
assert not first.endswith("# CURRENT OBSERVATION\n")
|
||||
|
||||
|
||||
def _transcript_ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.execution_setup = None
|
||||
ctx.actuator = None
|
||||
ctx.data_engine = None
|
||||
ctx.step_memory = StepMemoryService(ctx=None)
|
||||
ctx.transcript_ledger = None
|
||||
return ctx
|
||||
|
||||
|
||||
def _transcript_state(**overrides):
|
||||
state = MagicMock()
|
||||
state.subagent_calls = []
|
||||
state.initial_goal = "Transcript goal"
|
||||
state.operator_feedback = None
|
||||
state.injected_instruction = None
|
||||
state.operator_tool_limit_exceeded = False
|
||||
state.structured_decisions = None
|
||||
state.last_execution_result = None
|
||||
state.current_step_id = None
|
||||
state.latest_ui_hierarchy = []
|
||||
state.operator_raw_data = {
|
||||
"screenshot_b64": SCREENSHOT_B64,
|
||||
"xml_hierarchy": [],
|
||||
"ocr_results": None,
|
||||
"width": 1080,
|
||||
"height": 2400,
|
||||
}
|
||||
for key, value in overrides.items():
|
||||
setattr(state, key, value)
|
||||
return state
|
||||
|
||||
|
||||
def _no_action_llm(captured: list):
|
||||
mock_llm = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.tool_calls = []
|
||||
mock_response.content = "no action this turn"
|
||||
|
||||
async def mock_ainvoke(*args, **kwargs):
|
||||
captured.append(list(args[0]))
|
||||
return mock_response
|
||||
|
||||
mock_llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
return mock_llm
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_mode_two_turns_build_four_regions(tmp_path):
|
||||
ctx = _transcript_ctx()
|
||||
# Regression data shape (M2 cold-start bug, on-device A/B 2026-09-01):
|
||||
# turn 1 runs before any step record exists; by turn 2 the first step HAS
|
||||
# been recorded, so `steps` is non-empty while the previous turn is still
|
||||
# staged (turn_count == 0). The cold-start check must not read that
|
||||
# live-session state as a process restart — seeding would be rejected by
|
||||
# the empty-ledger invariant and silently demote every turn to the legacy
|
||||
# 2-message path.
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
ctx.data_engine.global_base_dir = str(tmp_path)
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
captured: list = []
|
||||
mock_llm = _no_action_llm(captured)
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
|
||||
|
||||
# Turn 1
|
||||
await node(_transcript_state())
|
||||
ledger = ctx.transcript_ledger
|
||||
assert isinstance(ledger, TranscriptLedger)
|
||||
assert ledger.has_static_prefix
|
||||
assert ledger.has_staged_turn
|
||||
assert ledger.turn_count == 0
|
||||
|
||||
turn1_messages = captured[0]
|
||||
assert isinstance(turn1_messages[0], SystemMessage)
|
||||
assert isinstance(turn1_messages[-1], HumanMessage)
|
||||
tail1_text = str(turn1_messages[-1].content)
|
||||
assert "# CURRENT OBSERVATION [T+" in tail1_text
|
||||
assert PLAN_RECITATION_MARKER in tail1_text
|
||||
|
||||
# Turn 2: step 1 is now recorded, and the previous turn is committed
|
||||
# with its step id and result.
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = [
|
||||
{
|
||||
"step_id": "step-1",
|
||||
"step_number": 1,
|
||||
"summary": "Step 1 summary",
|
||||
"relative_time": "1.0s",
|
||||
"extra_metadata": {"subgoal_hash": "default"},
|
||||
}
|
||||
]
|
||||
await node(
|
||||
_transcript_state(
|
||||
current_step_id="step-1",
|
||||
structured_decisions='[{"action": "tap"}]',
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
)
|
||||
|
||||
# The staged-but-uncommitted turn 2 state never triggers cold-start
|
||||
# seeding (the transcript path stayed live instead of falling back).
|
||||
assert not ledger.has_restored_history
|
||||
assert ledger.turn_count == 1
|
||||
turn2_messages = captured[1]
|
||||
# S region is the same object with the same bytes.
|
||||
assert turn2_messages[0] is turn1_messages[0]
|
||||
|
||||
active = list(ledger.active_messages)
|
||||
# Committed turn-1 observation followed by its validator result.
|
||||
assert any(EXECUTION_RESULT_MARKER in str(m.content) for m in active)
|
||||
committed_obs = active[0]
|
||||
# Depth-1 scrub removed the old plan recitation copy.
|
||||
assert PLAN_RECITATION_MARKER not in str(committed_obs.content)
|
||||
# The live tail still recites the plan.
|
||||
assert PLAN_RECITATION_MARKER in str(turn2_messages[-1].content)
|
||||
# Message layout is S + A + tail.
|
||||
assert turn2_messages[-1] is not turn1_messages[-1]
|
||||
assert re.search(r"T\+\d{2,}:\d{2}", str(turn2_messages[-1].content))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_actionless_turn_commits_without_validator_message():
|
||||
ctx = _transcript_ctx()
|
||||
captured: list = []
|
||||
mock_llm = _no_action_llm(captured)
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
|
||||
await node(_transcript_state())
|
||||
# Planner-rejected turns clear structured_decisions: no validator ran.
|
||||
await node(
|
||||
_transcript_state(
|
||||
current_step_id="step-1",
|
||||
structured_decisions="",
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
)
|
||||
|
||||
ledger = ctx.transcript_ledger
|
||||
assert ledger.turn_count == 1
|
||||
assert not any(
|
||||
EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_cold_start_builds_restored_history_block(tmp_path):
|
||||
ctx = _transcript_ctx()
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
ctx.data_engine.global_base_dir = str(tmp_path)
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = [
|
||||
{
|
||||
"step_id": f"step_{i}",
|
||||
"step_number": i,
|
||||
"summary": f"Step {i} summary",
|
||||
"relative_time": f"{i}.0s",
|
||||
"extra_metadata": {"subgoal_hash": "default"},
|
||||
}
|
||||
for i in range(1, 4)
|
||||
]
|
||||
|
||||
captured: list = []
|
||||
mock_llm = _no_action_llm(captured)
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
|
||||
await node(_transcript_state())
|
||||
|
||||
ledger = ctx.transcript_ledger
|
||||
assert ledger.has_restored_history
|
||||
|
||||
messages = captured[0]
|
||||
restored_text = str(messages[1].content)
|
||||
assert RESTORED_HISTORY_HEADER in restored_text
|
||||
assert "--- Execution History ---" in restored_text
|
||||
assert "Step 1 summary" in restored_text
|
||||
# The live tail still follows normally and A stays appendable.
|
||||
assert "# CURRENT OBSERVATION [T+" in str(messages[-1].content)
|
||||
assert ledger.has_staged_turn
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flag_off_keeps_legacy_two_message_build_and_no_ledger():
|
||||
ctx = _transcript_ctx()
|
||||
captured: list = []
|
||||
mock_llm = _no_action_llm(captured)
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=False))
|
||||
await node(_transcript_state())
|
||||
|
||||
assert len(captured[0]) == 2
|
||||
assert isinstance(captured[0][0], SystemMessage)
|
||||
assert isinstance(captured[0][1], HumanMessage)
|
||||
assert ctx.transcript_ledger is None
|
||||
@@ -286,6 +286,9 @@ async def test_outputter_builds_concise_history(mock_get_llm):
|
||||
}
|
||||
]
|
||||
mock_data_engine.get_agent_friendly_steps.return_value = steps
|
||||
# M5: the transcript flag defaults on, so the outputter's compiled view
|
||||
# consults the chunk store; no persisted chunks keeps the classic path.
|
||||
mock_data_engine.get_history_chunks.return_value = []
|
||||
mock_data_engine.base_dir = "/tmp/mock_session"
|
||||
ctx.data_engine = mock_data_engine
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ def test_full_toolset_is_the_default():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer",))
|
||||
@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer", "recall_history"))
|
||||
def test_removed_tool_leaves_no_reference(tool):
|
||||
"""The executable definition of 'an absent tool costs the model nothing'."""
|
||||
for name, template in load_operator_prompts().items():
|
||||
@@ -154,8 +154,8 @@ def test_full_set_enum_slots_render_verbatim():
|
||||
)
|
||||
assert (
|
||||
"(`ask_explorer`, `ask_diagnoser`, `video_analyzer`, `run_adb_command`,"
|
||||
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`, and"
|
||||
" `save_note`)" in out
|
||||
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`,"
|
||||
" `save_note`, and `recall_history`)" in out
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -12,11 +12,21 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
"""SummarizerNode tests (M2: visual-transition lens dispatch).
|
||||
|
||||
Per the history redesign §5 the Pro SummarizerNode no longer runs its own LLM
|
||||
call to produce a text capsule: it dispatches the step to the shared
|
||||
step-memory service (``ctx.step_memory``) with the pre/post screenshot bytes
|
||||
pulled from the DataEngine, and degrades gracefully when images are missing.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.agents.summarizer.summarizer import SummarizerNode
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
|
||||
class DummyState:
|
||||
@@ -36,181 +46,107 @@ class DummyState:
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
def mock_context(tmp_path):
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.device = None
|
||||
ctx.data_engine = Mock()
|
||||
ctx.background_tasks = []
|
||||
ctx.step_memory = MagicMock()
|
||||
|
||||
pre_path = tmp_path / "pre.jpg"
|
||||
post_path = tmp_path / "post.jpg"
|
||||
pre_path.write_bytes(b"PRE_IMAGE")
|
||||
post_path.write_bytes(b"POST_IMAGE")
|
||||
|
||||
engine = Mock()
|
||||
engine.get_step_number.return_value = 7
|
||||
engine.get_step_record.return_value = SimpleNamespace(
|
||||
action_taken=[{"action": "tap", "coordinates": [500, 600], "target_text": "Search"}]
|
||||
)
|
||||
engine.get_step_image_path.side_effect = lambda step_number, which="pre": (
|
||||
pre_path if which == "pre" else post_path
|
||||
)
|
||||
ctx.data_engine = engine
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_success(mock_context):
|
||||
"""Test that SummarizerNode successfully generates summary from raw thinking and execution result."""
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}])
|
||||
thinking = "The operator clicked the search button."
|
||||
result = {
|
||||
"status": "success",
|
||||
"executed_actions": [{"action": "tap", "coordinates": [500, 600]}],
|
||||
}
|
||||
|
||||
async def test_summarizer_dispatches_visual_lens(mock_context):
|
||||
"""The node dispatches to the shared service instead of calling an LLM."""
|
||||
state = DummyState(
|
||||
structured_decisions=decisions,
|
||||
operator_raw_thinking=thinking,
|
||||
last_execution_result=result,
|
||||
last_execution_result={"status": "success"},
|
||||
current_step_id="12345678-1234-5678-1234-567812345678",
|
||||
)
|
||||
|
||||
# Mock LLM response
|
||||
mock_response = Mock()
|
||||
mock_response.content = "Step completed successfully."
|
||||
mock_llm = MagicMock()
|
||||
node = SummarizerNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm
|
||||
) as mock_get_llm,
|
||||
patch("asyncio.create_task") as mock_create_task,
|
||||
):
|
||||
# Run the node call
|
||||
node = SummarizerNode(mock_context)
|
||||
await node(state)
|
||||
|
||||
# Verify task was registered in context
|
||||
assert len(mock_context.background_tasks) == 1
|
||||
assert mock_context.background_tasks[0] == mock_create_task.return_value
|
||||
|
||||
# Verify background task was spawned
|
||||
assert mock_create_task.called
|
||||
background_coroutine = mock_create_task.call_args[0][0]
|
||||
|
||||
# Await the background task synchronously for test verification
|
||||
await background_coroutine
|
||||
|
||||
# Verify LLM was invoked with proper structured messages
|
||||
assert mock_llm.astream.called
|
||||
invoked_messages = mock_llm.astream.call_args[0][0]
|
||||
|
||||
system_msg = invoked_messages[0].content
|
||||
human_msg = invoked_messages[1].content
|
||||
|
||||
assert "Summarizer Agent" in system_msg
|
||||
assert "Decisions made" in human_msg
|
||||
assert "The operator clicked the search button." in human_msg
|
||||
|
||||
# Verify Data Engine updated the step summary
|
||||
from uuid import UUID
|
||||
|
||||
mock_context.data_engine.update_step_summary.assert_called_once_with(
|
||||
UUID("12345678-1234-5678-1234-567812345678"),
|
||||
"Step completed successfully.",
|
||||
assert result == {}
|
||||
mock_context.step_memory.dispatch.assert_called_once_with(
|
||||
step_number=7,
|
||||
action_name="tap",
|
||||
action_args={"coordinates": [500, 600], "target_text": "Search"},
|
||||
pre_img_bytes=b"PRE_IMAGE",
|
||||
post_img_bytes=b"POST_IMAGE",
|
||||
exec_outcome="success",
|
||||
data_engine_step_id="12345678-1234-5678-1234-567812345678",
|
||||
)
|
||||
# No direct capsule write from the node: the lens owns the versioned
|
||||
# summary write when its background job completes.
|
||||
mock_context.data_engine.update_step_summary.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_with_native_thinking(mock_context):
|
||||
"""Test that SummarizerNode successfully generates summary from both explicit and native thoughts."""
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}])
|
||||
raw_thinking = "Explicit monologue."
|
||||
native_thinking = "Silent core reasoning."
|
||||
result = {"status": "success"}
|
||||
async def test_summarizer_degrades_gracefully_without_images(mock_context):
|
||||
"""Missing DataEngine screenshots dispatch with None bytes, no exception."""
|
||||
mock_context.data_engine.get_step_image_path.side_effect = None
|
||||
mock_context.data_engine.get_step_image_path.return_value = None
|
||||
|
||||
state = DummyState(
|
||||
structured_decisions=decisions,
|
||||
operator_raw_thinking=raw_thinking,
|
||||
operator_native_thinking=native_thinking,
|
||||
last_execution_result=result,
|
||||
last_execution_result={"status": "failed", "error": "tap missed"},
|
||||
current_step_id="12345678-1234-5678-1234-567812345678",
|
||||
)
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.content = "Summary result."
|
||||
mock_llm = MagicMock()
|
||||
node = SummarizerNode(mock_context)
|
||||
await node(state)
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm
|
||||
) as mock_get_llm,
|
||||
patch("asyncio.create_task") as mock_create_task,
|
||||
):
|
||||
node = SummarizerNode(mock_context)
|
||||
await node(state)
|
||||
|
||||
# Verify task was registered in context
|
||||
assert len(mock_context.background_tasks) == 1
|
||||
assert mock_context.background_tasks[0] == mock_create_task.return_value
|
||||
|
||||
background_coroutine = mock_create_task.call_args[0][0]
|
||||
await background_coroutine
|
||||
|
||||
assert mock_llm.astream.called
|
||||
invoked_messages = mock_llm.astream.call_args[0][0]
|
||||
human_msg = invoked_messages[1].content
|
||||
|
||||
assert "Operator explicit thoughts:" in human_msg
|
||||
assert "Explicit monologue." in human_msg
|
||||
assert "Operator native thoughts:" in human_msg
|
||||
assert "Silent core reasoning." in human_msg
|
||||
kwargs = mock_context.step_memory.dispatch.call_args.kwargs
|
||||
assert kwargs["pre_img_bytes"] is None
|
||||
assert kwargs["post_img_bytes"] is None
|
||||
assert kwargs["exec_outcome"] == "Error: tap missed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_with_chronological_trace(mock_context):
|
||||
"""Test that SummarizerNode generates a prompt containing the chronological step trace when available."""
|
||||
step_id = "12345678-1234-5678-1234-567812345678"
|
||||
async def test_summarizer_falls_back_to_structured_decisions(mock_context):
|
||||
"""Without a step record the action comes from structured_decisions, and
|
||||
extra actions ride along as additional_actions."""
|
||||
mock_context.data_engine.get_step_record.return_value = None
|
||||
|
||||
state = DummyState(
|
||||
current_step_id=step_id,
|
||||
structured_decisions=(
|
||||
'[{"action": "swipe", "coordinates": [1, 2, 3, 4]},'
|
||||
' {"action": "tap", "coordinates": [9, 9]}]'
|
||||
),
|
||||
last_execution_result={"status": "success"},
|
||||
current_step_id="12345678-1234-5678-1234-567812345678",
|
||||
)
|
||||
|
||||
mock_step_data = {
|
||||
"step_id": step_id,
|
||||
"step_number": 1,
|
||||
"relative_time": "5.0s",
|
||||
"action_taken": [{"action": "click", "target_text": "Search"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": "Let's click search."},
|
||||
],
|
||||
}
|
||||
node = SummarizerNode(mock_context)
|
||||
await node(state)
|
||||
|
||||
mock_context.data_engine.get_agent_friendly_steps.return_value = [mock_step_data]
|
||||
kwargs = mock_context.step_memory.dispatch.call_args.kwargs
|
||||
assert kwargs["action_name"] == "swipe"
|
||||
assert kwargs["action_args"]["coordinates"] == [1, 2, 3, 4]
|
||||
assert kwargs["action_args"]["additional_actions"] == [
|
||||
{"action": "tap", "coordinates": [9, 9]}
|
||||
]
|
||||
|
||||
# Mock LLM response
|
||||
mock_response = Mock()
|
||||
mock_response.content = "Summary using chronological trace."
|
||||
mock_llm = MagicMock()
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
@pytest.mark.asyncio
|
||||
async def test_summarizer_noop_without_step_id(mock_context):
|
||||
state = DummyState(current_step_id=None)
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
node = SummarizerNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm),
|
||||
patch("asyncio.create_task") as mock_create_task,
|
||||
):
|
||||
node = SummarizerNode(mock_context)
|
||||
await node(state)
|
||||
|
||||
# Verify task was registered in context
|
||||
assert len(mock_context.background_tasks) == 1
|
||||
assert mock_context.background_tasks[0] == mock_create_task.return_value
|
||||
|
||||
background_coroutine = mock_create_task.call_args[0][0]
|
||||
await background_coroutine
|
||||
|
||||
assert mock_llm.astream.called
|
||||
invoked_messages = mock_llm.astream.call_args[0][0]
|
||||
human_msg = invoked_messages[1].content
|
||||
|
||||
assert "--- Execution History ---" in human_msg
|
||||
assert "Let's click search." in human_msg
|
||||
assert result == {}
|
||||
mock_context.step_memory.dispatch.assert_not_called()
|
||||
@@ -380,3 +380,62 @@ def test_update_step_summary_includes_step_number_in_sse(tmp_path):
|
||||
assert last_update["step_number"] == 1
|
||||
|
||||
|
||||
|
||||
def _foreground_engine(tmp_path):
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_execution_setup = MagicMock()
|
||||
mock_execution_setup.traces_path = str(tmp_path)
|
||||
mock_ctx.execution_setup = mock_execution_setup
|
||||
mock_ctx.device = None
|
||||
engine = DataEngine(mock_ctx)
|
||||
engine.start_session("foreground_app persistence")
|
||||
return engine
|
||||
|
||||
|
||||
def test_record_step_persists_explicit_foreground_app(tmp_path):
|
||||
"""M5: the historically dropped foreground_app parameter is persisted."""
|
||||
engine = _foreground_engine(tmp_path)
|
||||
engine.record_step(
|
||||
foreground_app="com.android.settings",
|
||||
ui_tree=[{"package": "com.other.app", "bounds": "[0,0][10,10]"}],
|
||||
summary="Explicit parameter wins",
|
||||
)
|
||||
for t in list(engine._pending_threads):
|
||||
t.join()
|
||||
|
||||
step = engine.storage.get_steps(engine.current_session_id)[0]
|
||||
assert step.extra_metadata["foreground_app"] == "com.android.settings"
|
||||
|
||||
|
||||
def test_record_step_derives_foreground_app_from_ui_tree(tmp_path):
|
||||
"""Without the parameter, the dominant non-overlay package is stamped."""
|
||||
engine = _foreground_engine(tmp_path)
|
||||
engine.record_step(
|
||||
ui_tree=[
|
||||
{"package": "com.android.systemui", "bounds": "[0,0][10,10]"},
|
||||
{"package": "com.google.android.deskclock"},
|
||||
{
|
||||
"packageName": "com.google.android.deskclock",
|
||||
"children": [{"package": "com.google.android.deskclock"}],
|
||||
},
|
||||
],
|
||||
summary="Derived from UI tree",
|
||||
)
|
||||
for t in list(engine._pending_threads):
|
||||
t.join()
|
||||
|
||||
step = engine.storage.get_steps(engine.current_session_id)[0]
|
||||
assert step.extra_metadata["foreground_app"] == "com.google.android.deskclock"
|
||||
|
||||
|
||||
def test_record_step_without_app_data_stamps_nothing(tmp_path):
|
||||
engine = _foreground_engine(tmp_path)
|
||||
engine.record_step(
|
||||
ui_tree=[{"package": "com.android.systemui"}],
|
||||
summary="Only overlay packages",
|
||||
)
|
||||
for t in list(engine._pending_threads):
|
||||
t.join()
|
||||
|
||||
step = engine.storage.get_steps(engine.current_session_id)[0]
|
||||
assert "foreground_app" not in (step.extra_metadata or {})
|
||||
@@ -0,0 +1,105 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""History-chunk storage (M3 §6.1): append-only versioned rows keyed by step range."""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.data_engine.models import HistoryChunkRecord
|
||||
from artemis.data_engine.storage import StorageManager
|
||||
|
||||
SESSION = "11111111-1111-1111-1111-111111111111"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path):
|
||||
return StorageManager(tmp_path / "test.db", tmp_path / "traces")
|
||||
|
||||
|
||||
def _chunk(start: int, end: int, version: int = 1, **overrides) -> HistoryChunkRecord:
|
||||
fields = {
|
||||
"chunk_id": uuid4(),
|
||||
"session_id": SESSION,
|
||||
"start_step_id": f"sid-{start}",
|
||||
"end_step_id": f"sid-{end}",
|
||||
"start_step_number": start,
|
||||
"end_step_number": end,
|
||||
"source_step_ids": [f"sid-{n}" for n in range(start, end + 1)],
|
||||
"subgoal_hash": "hash-a",
|
||||
"version": version,
|
||||
"status": "pending",
|
||||
"band1": {},
|
||||
"band2": None,
|
||||
"band3": f"- Step {start} (T+00:10): Tapped 'x' at [1, 2] -> executed",
|
||||
"rendered_text": "[Chunk 1 | ...]",
|
||||
}
|
||||
fields.update(overrides)
|
||||
return HistoryChunkRecord(**fields)
|
||||
|
||||
|
||||
def test_history_chunk_roundtrip(storage):
|
||||
storage.create_history_chunk(_chunk(1, 4))
|
||||
chunks = storage.get_history_chunks(SESSION)
|
||||
assert len(chunks) == 1
|
||||
rec = chunks[0]
|
||||
assert (rec.start_step_number, rec.end_step_number) == (1, 4)
|
||||
assert rec.source_step_ids == [f"sid-{n}" for n in range(1, 5)]
|
||||
assert rec.subgoal_hash == "hash-a"
|
||||
assert rec.status == "pending"
|
||||
assert "T+00:10" in rec.band3
|
||||
|
||||
|
||||
def test_history_chunk_versions_are_append_only_latest_wins(storage):
|
||||
storage.create_history_chunk(_chunk(1, 4, version=1, status="pending"))
|
||||
storage.create_history_chunk(
|
||||
_chunk(
|
||||
1,
|
||||
4,
|
||||
version=2,
|
||||
status="ready",
|
||||
band1={"doing": "d", "verified_facts": ["f1"]},
|
||||
band2=" - Steps 1–4: did things",
|
||||
)
|
||||
)
|
||||
|
||||
latest = storage.get_history_chunks(SESSION)
|
||||
assert len(latest) == 1
|
||||
assert latest[0].version == 2
|
||||
assert latest[0].status == "ready"
|
||||
assert latest[0].band1["verified_facts"] == ["f1"]
|
||||
|
||||
trail = storage.get_history_chunks(SESSION, all_versions=True)
|
||||
assert [r.version for r in trail] == [1, 2]
|
||||
|
||||
|
||||
def test_history_chunks_ordered_by_step_range(storage):
|
||||
storage.create_history_chunk(_chunk(9, 12))
|
||||
storage.create_history_chunk(_chunk(1, 4))
|
||||
storage.create_history_chunk(_chunk(5, 8))
|
||||
latest = storage.get_history_chunks(SESSION)
|
||||
assert [(r.start_step_number, r.end_step_number) for r in latest] == [
|
||||
(1, 4),
|
||||
(5, 8),
|
||||
(9, 12),
|
||||
]
|
||||
|
||||
|
||||
def test_history_chunks_are_session_scoped(storage):
|
||||
storage.create_history_chunk(_chunk(1, 4))
|
||||
other = "22222222-2222-2222-2222-222222222222"
|
||||
storage.create_history_chunk(_chunk(1, 4, session_id=other))
|
||||
assert len(storage.get_history_chunks(SESSION)) == 1
|
||||
assert len(storage.get_history_chunks(other)) == 1
|
||||
@@ -0,0 +1,171 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Tests for the versioned, status-carrying step summary write (M0 §6.1)."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
|
||||
|
||||
def _make_engine(tmp_path):
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_execution_setup = MagicMock()
|
||||
mock_execution_setup.traces_path = str(tmp_path)
|
||||
mock_ctx.execution_setup = mock_execution_setup
|
||||
mock_ctx.device = None
|
||||
engine = DataEngine(mock_ctx)
|
||||
engine.start_session("summary versioning test")
|
||||
return engine
|
||||
|
||||
|
||||
def _flush(engine):
|
||||
for t in list(engine._pending_threads):
|
||||
t.join()
|
||||
|
||||
|
||||
def _get_step(engine, step_id):
|
||||
steps = engine.storage.get_steps(engine.current_session_id)
|
||||
for step in steps:
|
||||
if str(step.step_id) == str(step_id):
|
||||
return step
|
||||
return None
|
||||
|
||||
|
||||
def test_ready_write_persists_summary_and_metadata(tmp_path):
|
||||
engine = _make_engine(tmp_path)
|
||||
step_id = engine.record_step(summary="initial")
|
||||
_flush(engine)
|
||||
|
||||
applied = engine.storage.update_step_summary(
|
||||
step_id,
|
||||
"Tapped the login button.",
|
||||
source="visual_transition",
|
||||
model="gemini-2.5-flash-lite",
|
||||
status="ready",
|
||||
)
|
||||
assert applied is True
|
||||
|
||||
step = _get_step(engine, step_id)
|
||||
assert step.summary == "Tapped the login button."
|
||||
assert step.extra_metadata["summary_status"] == "ready"
|
||||
assert step.extra_metadata["summary_source"] == "visual_transition"
|
||||
assert step.extra_metadata["summary_version"] == 1
|
||||
assert step.extra_metadata["summary_model"] == "gemini-2.5-flash-lite"
|
||||
|
||||
|
||||
def test_auto_version_increments_per_write(tmp_path):
|
||||
engine = _make_engine(tmp_path)
|
||||
step_id = engine.record_step(summary="initial")
|
||||
_flush(engine)
|
||||
|
||||
engine.storage.update_step_summary(step_id, "v1", status="ready")
|
||||
engine.storage.update_step_summary(step_id, "v2", status="ready")
|
||||
|
||||
step = _get_step(engine, step_id)
|
||||
assert step.summary == "v2"
|
||||
assert step.extra_metadata["summary_version"] == 2
|
||||
|
||||
|
||||
def test_stale_explicit_version_is_dropped(tmp_path):
|
||||
engine = _make_engine(tmp_path)
|
||||
step_id = engine.record_step(summary="initial")
|
||||
_flush(engine)
|
||||
|
||||
assert engine.storage.update_step_summary(step_id, "newer", version=5, status="ready")
|
||||
applied = engine.storage.update_step_summary(step_id, "older", version=3, status="ready")
|
||||
assert applied is False
|
||||
|
||||
step = _get_step(engine, step_id)
|
||||
assert step.summary == "newer"
|
||||
assert step.extra_metadata["summary_version"] == 5
|
||||
|
||||
|
||||
def test_pending_does_not_downgrade_ready(tmp_path):
|
||||
engine = _make_engine(tmp_path)
|
||||
step_id = engine.record_step(summary="initial")
|
||||
_flush(engine)
|
||||
|
||||
engine.storage.update_step_summary(step_id, "done", status="ready")
|
||||
applied = engine.storage.update_step_summary(step_id, None, status="pending")
|
||||
assert applied is False
|
||||
|
||||
step = _get_step(engine, step_id)
|
||||
assert step.summary == "done"
|
||||
assert step.extra_metadata["summary_status"] == "ready"
|
||||
|
||||
|
||||
def test_failed_marks_status_without_touching_summary(tmp_path):
|
||||
engine = _make_engine(tmp_path)
|
||||
step_id = engine.record_step(summary="original text")
|
||||
_flush(engine)
|
||||
|
||||
engine.storage.update_step_summary(step_id, None, status="pending", source="visual_transition")
|
||||
applied = engine.storage.update_step_summary(
|
||||
step_id, None, status="failed", source="visual_transition"
|
||||
)
|
||||
assert applied is True
|
||||
|
||||
step = _get_step(engine, step_id)
|
||||
assert step.summary == "original text"
|
||||
assert step.extra_metadata["summary_status"] == "failed"
|
||||
|
||||
# A failed write must never clobber an already-ready summary.
|
||||
engine.storage.update_step_summary(step_id, "recovered", status="ready")
|
||||
applied = engine.storage.update_step_summary(step_id, None, status="failed")
|
||||
assert applied is False
|
||||
step = _get_step(engine, step_id)
|
||||
assert step.extra_metadata["summary_status"] == "ready"
|
||||
|
||||
|
||||
def test_missing_step_returns_false(tmp_path):
|
||||
from uuid import uuid4
|
||||
|
||||
engine = _make_engine(tmp_path)
|
||||
assert engine.storage.update_step_summary(uuid4(), "text", status="ready") is False
|
||||
|
||||
|
||||
def test_engine_level_write_carries_metadata_and_publishes_ready_only(tmp_path):
|
||||
engine = _make_engine(tmp_path)
|
||||
published = []
|
||||
engine.subscribe(lambda event_type, data: published.append((event_type, data)))
|
||||
|
||||
step_id = engine.record_step(summary="initial")
|
||||
_flush(engine)
|
||||
published.clear()
|
||||
|
||||
# Non-ready statuses persist metadata but publish no SSE.
|
||||
engine.update_step_summary(step_id, None, status="pending", source="visual_transition")
|
||||
_flush(engine)
|
||||
assert not [e for e in published if e[0] == "step_updated"]
|
||||
step = _get_step(engine, step_id)
|
||||
assert step.extra_metadata["summary_status"] == "pending"
|
||||
|
||||
engine.update_step_summary(
|
||||
step_id,
|
||||
"Ready text.",
|
||||
status="ready",
|
||||
source="visual_transition",
|
||||
model="test-model",
|
||||
)
|
||||
_flush(engine)
|
||||
ready_events = [e for e in published if e[0] == "step_updated"]
|
||||
assert len(ready_events) == 1
|
||||
assert ready_events[0][1]["summary"] == "Ready text."
|
||||
|
||||
step = _get_step(engine, step_id)
|
||||
assert step.summary == "Ready text."
|
||||
assert step.extra_metadata["summary_status"] == "ready"
|
||||
assert step.extra_metadata["summary_model"] == "test-model"
|
||||
@@ -0,0 +1,158 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Tests for record-time target semantics enrichment in McpActionExecutor."""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
from artemis.mcp.action_executor import McpActionExecutor
|
||||
|
||||
|
||||
def _make_executor(width=1080, height=2400):
|
||||
ctx = Mock()
|
||||
ctx.device.device_width = width
|
||||
ctx.device.device_height = height
|
||||
actuator = Mock()
|
||||
actuator.controller = Mock()
|
||||
return McpActionExecutor(ctx, actuator=actuator)
|
||||
|
||||
|
||||
def _state_with_elements(elements):
|
||||
state = Mock()
|
||||
state.indexed_elements = elements
|
||||
return state
|
||||
|
||||
|
||||
def test_click_target_semantics_from_pre_action_frame():
|
||||
executor = _make_executor()
|
||||
# Normalized (500, 600) on a 1080x2400 device -> pixel (540, 1440).
|
||||
state = _state_with_elements(
|
||||
[
|
||||
{
|
||||
"text": "Confirm",
|
||||
"bounds": [500, 1400, 600, 1480],
|
||||
"class": "android.widget.Button",
|
||||
"resource_id": "btn_confirm",
|
||||
"is_ocr": False,
|
||||
}
|
||||
]
|
||||
)
|
||||
semantics = executor._target_semantics("click", {"target": [500, 600]}, state)
|
||||
assert semantics == {
|
||||
"target_label_source": "hit_test",
|
||||
"target_text": "Confirm",
|
||||
"target_class": "android.widget.Button",
|
||||
"target_resource_id": "btn_confirm",
|
||||
}
|
||||
|
||||
|
||||
def test_semantics_degrade_to_none_without_element_data():
|
||||
executor = _make_executor()
|
||||
|
||||
# Empty perception data
|
||||
semantics = executor._target_semantics(
|
||||
"click", {"target": [500, 600]}, _state_with_elements([])
|
||||
)
|
||||
assert semantics == {"target_label_source": "none"}
|
||||
|
||||
# No state at all
|
||||
semantics = executor._target_semantics("click", {"target": [500, 600]}, None)
|
||||
assert semantics == {"target_label_source": "none"}
|
||||
|
||||
|
||||
def test_non_targeted_actions_and_missing_targets_skip_enrichment():
|
||||
executor = _make_executor()
|
||||
state = _state_with_elements([{"text": "X", "bounds": [0, 0, 10, 10]}])
|
||||
|
||||
assert executor._target_semantics("press_key", {"key": "BACK"}, state) is None
|
||||
# input_text without a coordinate target (focused field typing)
|
||||
assert executor._target_semantics("input_text", {"text": "hi", "target": None}, state) is None
|
||||
# Multi-point actions without resolvable points skip enrichment too.
|
||||
assert executor._target_semantics("click_sequence", {"sequence": []}, state) is None
|
||||
assert executor._target_semantics("swipe", {"start": None, "end": None}, state) is None
|
||||
|
||||
|
||||
def test_click_sequence_hit_tests_every_point_first_point_is_main_label():
|
||||
"""M5: multi-point actions get per-point best-effort semantics; the first
|
||||
point's fields are hoisted as the action's main label."""
|
||||
executor = _make_executor()
|
||||
state = _state_with_elements(
|
||||
[
|
||||
{
|
||||
"text": "Digit 1",
|
||||
"bounds": [500, 1400, 600, 1480],
|
||||
"class": "android.widget.Button",
|
||||
"is_ocr": False,
|
||||
},
|
||||
{
|
||||
"text": "Digit 2",
|
||||
"bounds": [700, 1400, 800, 1480],
|
||||
"class": "android.widget.Button",
|
||||
"is_ocr": False,
|
||||
},
|
||||
]
|
||||
)
|
||||
# Normalized points -> pixels: (500,600)->(540,1440) hits Digit 1;
|
||||
# (690,600)->(745,1440) hits Digit 2; (10,10)->(10,24) hits nothing.
|
||||
semantics = executor._target_semantics(
|
||||
"click_sequence",
|
||||
{"sequence": [[500, 600], [690, 600], [10, 10]], "delay_ms": 50},
|
||||
state,
|
||||
)
|
||||
assert semantics["target_text"] == "Digit 1"
|
||||
assert semantics["target_label_source"] == "hit_test"
|
||||
per_point = semantics["points_semantics"]
|
||||
assert len(per_point) == 3
|
||||
assert per_point[0]["target_text"] == "Digit 1"
|
||||
assert per_point[1]["target_text"] == "Digit 2"
|
||||
assert per_point[2] == {"target_label_source": "none"}
|
||||
|
||||
|
||||
def test_swipe_hit_tests_start_and_end_start_is_main_label():
|
||||
executor = _make_executor()
|
||||
state = _state_with_elements(
|
||||
[
|
||||
{
|
||||
"text": "Brightness slider",
|
||||
"bounds": [100, 1400, 900, 1480],
|
||||
"class": "android.widget.SeekBar",
|
||||
"is_ocr": False,
|
||||
}
|
||||
]
|
||||
)
|
||||
semantics = executor._target_semantics(
|
||||
"swipe", {"start": [200, 600], "end": [800, 600], "duration_ms": 400}, state
|
||||
)
|
||||
assert semantics["target_text"] == "Brightness slider"
|
||||
assert semantics["start_semantics"]["target_text"] == "Brightness slider"
|
||||
assert semantics["end_semantics"]["target_text"] == "Brightness slider"
|
||||
assert semantics["target_label_source"] == "hit_test"
|
||||
|
||||
|
||||
def test_long_press_uses_ocr_fallback():
|
||||
executor = _make_executor()
|
||||
state = _state_with_elements(
|
||||
[
|
||||
{
|
||||
"text": "OCR Word",
|
||||
"bounds": [520, 1420, 560, 1460],
|
||||
"class": None,
|
||||
"resource_id": None,
|
||||
"is_ocr": True,
|
||||
}
|
||||
]
|
||||
)
|
||||
semantics = executor._target_semantics("long_press", {"target": [500, 600]}, state)
|
||||
assert semantics["target_label_source"] == "ocr"
|
||||
assert semantics["target_text"] == "OCR Word"
|
||||
@@ -0,0 +1,304 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""ContextPolicy table (M4): byte-exact parity with the eight historical call
|
||||
sites, config overrides, and the chunk hand-off to the compiled view."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.memory.context_policy import (
|
||||
CONTEXT_POLICIES,
|
||||
DIGEST_LEDGER_MARKER_TEMPLATE,
|
||||
build_history_for,
|
||||
load_chunk_blocks,
|
||||
resolve_policy,
|
||||
)
|
||||
from artemis.utils.task_tree import build_plan_and_history
|
||||
|
||||
HASH_A = "hash-a"
|
||||
HASH_B = "hash-b"
|
||||
|
||||
TASK_PLAN = f"- [x] First milestone <!-- {HASH_A} -->\n- [/] Second milestone <!-- {HASH_B} -->"
|
||||
|
||||
|
||||
def _step(number: int, subgoal_hash: str, summary: str | None = None) -> dict:
|
||||
return {
|
||||
"step_id": f"id-{number}",
|
||||
"step_number": number,
|
||||
"relative_time": f"{number * 10}.0s",
|
||||
"summary": summary or f"Step {number} summary text.",
|
||||
"action_taken": {"action": "click", "target_text": f"button {number}"},
|
||||
"operator_raw_thinking": f"thinking {number}",
|
||||
"last_execution_result": {"status": "success"},
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash},
|
||||
}
|
||||
|
||||
|
||||
STEPS = [
|
||||
_step(1, HASH_A),
|
||||
_step(2, HASH_A),
|
||||
_step(3, HASH_A),
|
||||
_step(4, HASH_B),
|
||||
_step(5, HASH_B),
|
||||
_step(6, HASH_B),
|
||||
_step(7, HASH_B),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Golden parity: the policy table must reproduce every historical hard-coded
|
||||
# call byte for byte (kwargs recorded verbatim from the 2026-09-01 call sites).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
GOLDEN_CALLS = {
|
||||
"operator": lambda: build_plan_and_history(
|
||||
TASK_PLAN,
|
||||
STEPS,
|
||||
HASH_B,
|
||||
last_n_detailed=1,
|
||||
strict_milestone_pruning=True,
|
||||
recent_window_size=3,
|
||||
),
|
||||
"operator_cold_start": lambda: build_plan_and_history(
|
||||
TASK_PLAN,
|
||||
STEPS,
|
||||
HASH_B,
|
||||
min_summaries=len(STEPS),
|
||||
last_n_detailed=1,
|
||||
),
|
||||
"failure_analyzer": lambda: build_plan_and_history(
|
||||
TASK_PLAN,
|
||||
STEPS,
|
||||
HASH_B,
|
||||
last_n_detailed=1,
|
||||
strict_milestone_pruning=True,
|
||||
recent_window_size=3,
|
||||
for_failure_analyzer=True,
|
||||
),
|
||||
"planner": lambda: build_plan_and_history(
|
||||
"",
|
||||
STEPS,
|
||||
"default",
|
||||
last_n_detailed=2,
|
||||
strict_milestone_pruning=True,
|
||||
recent_window_size=5,
|
||||
),
|
||||
"outputter": lambda: build_plan_and_history(
|
||||
TASK_PLAN,
|
||||
STEPS,
|
||||
HASH_B,
|
||||
last_n_detailed=0,
|
||||
min_summaries=len(STEPS),
|
||||
strict_milestone_pruning=False,
|
||||
),
|
||||
"history_analyzer": lambda: build_plan_and_history(
|
||||
TASK_PLAN,
|
||||
STEPS,
|
||||
HASH_B,
|
||||
last_n_detailed=1,
|
||||
min_summaries=len(STEPS),
|
||||
),
|
||||
"diagnoser": lambda: build_plan_and_history(
|
||||
TASK_PLAN,
|
||||
STEPS,
|
||||
HASH_B,
|
||||
keep_subgoal_hashes={HASH_B},
|
||||
),
|
||||
"committee": lambda: build_plan_and_history(
|
||||
TASK_PLAN,
|
||||
STEPS,
|
||||
HASH_B,
|
||||
keep_subgoal_hashes={HASH_B},
|
||||
last_n_detailed=3,
|
||||
),
|
||||
}
|
||||
|
||||
POLICY_CALLS = {
|
||||
"operator": lambda: build_history_for(
|
||||
"operator", TASK_PLAN, STEPS, HASH_B, last_n_detailed=1
|
||||
),
|
||||
"operator_cold_start": lambda: build_history_for(
|
||||
"operator_cold_start", TASK_PLAN, STEPS, HASH_B
|
||||
),
|
||||
"failure_analyzer": lambda: build_history_for(
|
||||
"failure_analyzer", TASK_PLAN, STEPS, HASH_B
|
||||
),
|
||||
"planner": lambda: build_history_for("planner", "", STEPS, "default"),
|
||||
"outputter": lambda: build_history_for("outputter", TASK_PLAN, STEPS, HASH_B),
|
||||
"history_analyzer": lambda: build_history_for(
|
||||
"history_analyzer", TASK_PLAN, STEPS, HASH_B
|
||||
),
|
||||
"diagnoser": lambda: build_history_for(
|
||||
"diagnoser", TASK_PLAN, STEPS, HASH_B, keep_subgoal_hashes={HASH_B}
|
||||
),
|
||||
"committee": lambda: build_history_for(
|
||||
"committee", TASK_PLAN, STEPS, HASH_B, keep_subgoal_hashes={HASH_B}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("agent", sorted(GOLDEN_CALLS))
|
||||
def test_policy_table_reproduces_call_site_bytes(agent):
|
||||
assert POLICY_CALLS[agent]() == GOLDEN_CALLS[agent]()
|
||||
|
||||
|
||||
def test_every_declared_policy_is_golden_tested():
|
||||
assert set(GOLDEN_CALLS) == set(CONTEXT_POLICIES)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Config overrides
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _config_with_policies(policies):
|
||||
return SimpleNamespace(memory=SimpleNamespace(policies=policies))
|
||||
|
||||
|
||||
def test_resolve_policy_applies_config_override():
|
||||
cfg = _config_with_policies({"planner": {"last_n_detailed": 3, "bogus_field": 1}})
|
||||
with patch("artemis.config.load_agent_config", return_value=cfg):
|
||||
policy = resolve_policy("planner")
|
||||
assert policy.last_n_detailed == 3
|
||||
assert policy.strict_milestone_pruning is True # untouched fields survive
|
||||
|
||||
|
||||
def test_resolve_policy_unknown_agent_raises():
|
||||
with pytest.raises(KeyError):
|
||||
resolve_policy("no_such_agent")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Chunk hand-off (§10 decision 4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
RENDERED_CHUNK = (
|
||||
"[Chunk 1 | Steps 1–3 | T+00:10 → T+00:30]\n"
|
||||
"\n"
|
||||
"① Synopsis & effects\n"
|
||||
" What this segment was doing: warming up.\n"
|
||||
"\n"
|
||||
"② Compressed step summary\n"
|
||||
" - Steps 1–3: opened the app and logged in\n"
|
||||
"\n"
|
||||
"③ Step action ledger\n"
|
||||
"- Step 1 (T+00:10): click['button 1'] -> executed\n"
|
||||
"- Step 2 (T+00:20): click['button 2'] -> executed\n"
|
||||
"- Step 3 (T+00:30): click['button 3'] -> executed"
|
||||
)
|
||||
|
||||
|
||||
class _FakeEngine:
|
||||
def __init__(self, rows):
|
||||
self._rows = rows
|
||||
|
||||
def get_history_chunks(self, **kwargs):
|
||||
return self._rows
|
||||
|
||||
|
||||
def _chunk_row():
|
||||
return SimpleNamespace(
|
||||
start_step_number=1,
|
||||
end_step_number=3,
|
||||
rendered_text=RENDERED_CHUNK,
|
||||
status="ready",
|
||||
)
|
||||
|
||||
|
||||
def _transcript_cfg(enabled: bool):
|
||||
return SimpleNamespace(
|
||||
memory=SimpleNamespace(
|
||||
transcript=SimpleNamespace(enabled=enabled), policies={}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_flag_off_output_is_byte_identical_even_with_engine():
|
||||
engine = _FakeEngine([_chunk_row()])
|
||||
with patch("artemis.config.load_agent_config", return_value=_transcript_cfg(False)):
|
||||
with_engine = build_history_for(
|
||||
"outputter", TASK_PLAN, STEPS, HASH_B, engine=engine
|
||||
)
|
||||
without_engine = build_history_for("outputter", TASK_PLAN, STEPS, HASH_B)
|
||||
assert with_engine == without_engine
|
||||
assert "[Chunk 1" not in with_engine
|
||||
|
||||
|
||||
def test_flag_on_full_view_renders_chunk_block_and_drops_covered_steps():
|
||||
engine = _FakeEngine([_chunk_row()])
|
||||
with patch("artemis.config.load_agent_config", return_value=_transcript_cfg(True)):
|
||||
out = build_history_for("outputter", TASK_PLAN, STEPS, HASH_B, engine=engine)
|
||||
# Full view: the whole three-band block including the ③ ledger.
|
||||
assert "[Chunk 1 | Steps 1–3" in out
|
||||
assert "③ Step action ledger" in out
|
||||
assert "- Step 2 (T+00:20)" in out
|
||||
# Covered per-step summary lines are replaced by the chunk block.
|
||||
assert "*Step 2 (Start: 20.0s)" not in out
|
||||
# Steps outside the chunk range still render.
|
||||
assert "Step 5" in out
|
||||
|
||||
|
||||
def test_flag_on_digest_view_replaces_ledger_with_recall_marker():
|
||||
engine = _FakeEngine([_chunk_row()])
|
||||
cfg = _transcript_cfg(True)
|
||||
with patch("artemis.config.load_agent_config", return_value=cfg):
|
||||
out = build_history_for(
|
||||
"diagnoser",
|
||||
TASK_PLAN,
|
||||
STEPS,
|
||||
HASH_B,
|
||||
keep_subgoal_hashes={HASH_A, HASH_B},
|
||||
engine=engine,
|
||||
)
|
||||
assert "[Chunk 1 | Steps 1–3" in out
|
||||
assert "② Compressed step summary" in out
|
||||
marker = DIGEST_LEDGER_MARKER_TEMPLATE.format(start=1, end=3)
|
||||
assert marker in out
|
||||
assert "- Step 2 (T+00:20)" not in out # digest carries no ③ rows
|
||||
|
||||
|
||||
def test_chunk_view_none_never_loads_chunks():
|
||||
engine = _FakeEngine([_chunk_row()])
|
||||
with patch("artemis.config.load_agent_config", return_value=_transcript_cfg(True)):
|
||||
assert load_chunk_blocks(engine, None) is None
|
||||
# The cold-start policy declares chunk_view=None: even with an engine
|
||||
# and the flag on, its output carries no chunk blocks (the restored
|
||||
# F region must stay a pure step rendering).
|
||||
out = build_history_for(
|
||||
"operator_cold_start", TASK_PLAN, STEPS, HASH_B, engine=engine
|
||||
)
|
||||
assert CONTEXT_POLICIES["operator_cold_start"].chunk_view is None
|
||||
assert "[Chunk 1" not in out
|
||||
|
||||
|
||||
def test_most_recent_step_is_never_suppressed_by_a_chunk():
|
||||
engine = _FakeEngine(
|
||||
[
|
||||
SimpleNamespace(
|
||||
start_step_number=1,
|
||||
end_step_number=7, # covers every step incl. the newest
|
||||
rendered_text=RENDERED_CHUNK,
|
||||
status="ready",
|
||||
)
|
||||
]
|
||||
)
|
||||
with patch("artemis.config.load_agent_config", return_value=_transcript_cfg(True)):
|
||||
out = build_history_for("outputter", TASK_PLAN, STEPS, HASH_B, engine=engine)
|
||||
# The most-recent step must keep its own (detailed) rendering.
|
||||
assert "Step 7" in out
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,187 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for the shared StepMemoryService scheduling runtime (M1)."""
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.memory.step_memory import StepMemoryService
|
||||
|
||||
|
||||
class _ScriptedService(StepMemoryService):
|
||||
"""StepMemoryService with a scriptable attempt function and status log."""
|
||||
|
||||
def __init__(self, attempt_fn, **kwargs):
|
||||
super().__init__(Mock(), **kwargs)
|
||||
self._attempt_fn = attempt_fn
|
||||
self.status_events: list[tuple] = []
|
||||
|
||||
async def _attempt(self, key):
|
||||
return await self._attempt_fn(self, key)
|
||||
|
||||
def _on_status(self, key, status):
|
||||
self.status_events.append((key, status))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zero_blocking_submit_and_flush():
|
||||
"""submit() returns immediately; flush() drains the background job."""
|
||||
release = asyncio.Event()
|
||||
|
||||
async def attempt(svc, key):
|
||||
await release.wait()
|
||||
svc._summaries[key] = f"summary-for-{key}"
|
||||
return True
|
||||
|
||||
service = _ScriptedService(attempt)
|
||||
service.submit("step-1", {"step_number": 1})
|
||||
|
||||
# Zero-blocking: the job cannot have completed yet (its gate is closed).
|
||||
assert service.is_pending("step-1")
|
||||
assert not service.has_summary("step-1")
|
||||
|
||||
release.set()
|
||||
await service.flush()
|
||||
|
||||
assert service.get_summary("step-1") == "summary-for-step-1"
|
||||
assert not service.is_pending("step-1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bounded_retry_enters_failed_state():
|
||||
"""A permanently failing job stops after 1 + retry_limit attempts."""
|
||||
attempts = 0
|
||||
|
||||
async def attempt(svc, key):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
return False
|
||||
|
||||
service = _ScriptedService(attempt, retry_limit=2)
|
||||
service._retry_delays = (0.0,)
|
||||
service.submit("step-x", {"step_number": 7})
|
||||
await service.flush()
|
||||
|
||||
assert attempts == 3
|
||||
assert service.has_failed("step-x")
|
||||
assert not service.has_summary("step-x")
|
||||
# The failed job still reads as pending (lossless semantics downstream).
|
||||
assert service.is_pending("step-x")
|
||||
assert service.status_events == [("step-x", "pending"), ("step-x", "failed")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_step_id_keying_with_tool_call_alias():
|
||||
"""Jobs keyed by step id remain queryable through tool_call_id aliases."""
|
||||
|
||||
async def attempt(svc, key):
|
||||
svc._summaries[key] = "canonical summary"
|
||||
return True
|
||||
|
||||
service = _ScriptedService(attempt)
|
||||
service.submit("uuid-123", {"step_number": 4}, aliases=("tc-abc",))
|
||||
await service.flush()
|
||||
|
||||
assert service.resolve_key("tc-abc") == "uuid-123"
|
||||
assert service.has_job("tc-abc")
|
||||
assert service.has_summary("tc-abc")
|
||||
assert service.get_summary("tc-abc") == "canonical summary"
|
||||
assert service.get_summary("uuid-123") == "canonical summary"
|
||||
assert service.get_job_payload("tc-abc") == {"step_number": 4}
|
||||
assert not service.has_failed("tc-abc")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_max_concurrency_bounds_parallel_attempts():
|
||||
"""With max_concurrency=1, two jobs never run their attempts concurrently."""
|
||||
active = 0
|
||||
max_active = 0
|
||||
|
||||
async def attempt(svc, key):
|
||||
nonlocal active, max_active
|
||||
active += 1
|
||||
max_active = max(max_active, active)
|
||||
await asyncio.sleep(0.01)
|
||||
active -= 1
|
||||
svc._summaries[key] = "done"
|
||||
return True
|
||||
|
||||
service = _ScriptedService(attempt, max_concurrency=1)
|
||||
service.submit("a", {"step_number": 1})
|
||||
service.submit("b", {"step_number": 2})
|
||||
await service.flush()
|
||||
|
||||
assert service.has_summary("a") and service.has_summary("b")
|
||||
assert max_active == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flush_timeout_cancels_stragglers():
|
||||
"""flush() cancels jobs that outlive the timeout without failing them."""
|
||||
|
||||
async def attempt(svc, key):
|
||||
await asyncio.sleep(30)
|
||||
return True
|
||||
|
||||
service = _ScriptedService(attempt)
|
||||
service.submit("slow", {"step_number": 1})
|
||||
await service.flush(timeout_seconds=0.05)
|
||||
|
||||
assert service.is_pending("slow")
|
||||
assert not service.has_failed("slow")
|
||||
assert service._pending_tasks["slow"].done()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flush_uses_configured_default_timeout():
|
||||
"""flush() with no argument honors the constructor's flush_timeout_s."""
|
||||
|
||||
async def attempt(svc, key):
|
||||
await asyncio.sleep(30)
|
||||
return True
|
||||
|
||||
service = _ScriptedService(attempt, flush_timeout_s=0.05)
|
||||
service.submit("slow", {"step_number": 1})
|
||||
await asyncio.wait_for(service.flush(), timeout=5)
|
||||
|
||||
assert service._pending_tasks["slow"].done()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resubmit_clears_failed_state():
|
||||
"""Re-submitting a failed job gives it a fresh retry budget."""
|
||||
behaviors = {"fail": True}
|
||||
|
||||
async def attempt(svc, key):
|
||||
if behaviors["fail"]:
|
||||
return False
|
||||
svc._summaries[key] = "second time lucky"
|
||||
return True
|
||||
|
||||
service = _ScriptedService(attempt, retry_limit=0)
|
||||
service._retry_delays = (0.0,)
|
||||
service.submit("job", {"step_number": 1})
|
||||
await service.flush()
|
||||
assert service.has_failed("job")
|
||||
|
||||
behaviors["fail"] = False
|
||||
service._retry_counts.clear()
|
||||
service.submit("job", {"step_number": 1})
|
||||
await service.flush()
|
||||
|
||||
assert not service.has_failed("job")
|
||||
assert service.get_summary("job") == "second time lucky"
|
||||
@@ -0,0 +1,298 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for the Pro session transcript ledger (history redesign §3.2, M2).
|
||||
|
||||
Covers the four-region discipline: S-region byte stability across turns, the
|
||||
append-only active region, depth-1 stripping of old UI lists and plan
|
||||
recitations, the depth-K screenshot scrub in the Pro message shape (grace /
|
||||
placeholder / freeze — the M1 race regressions re-run against HumanMessage
|
||||
observations keyed by step id), ``T+mm:ss`` session-offset timestamps, the
|
||||
tool-call/response pairing invariant, and the cold-start restored-history
|
||||
block.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
|
||||
|
||||
from artemis.memory.step_memory import StepMemoryService
|
||||
from artemis.memory.transcript import (
|
||||
EXECUTION_RESULT_MARKER,
|
||||
PLAN_RECITATION_MARKER,
|
||||
PRO_UI_LIST_MARKER,
|
||||
RESTORED_HISTORY_HEADER,
|
||||
TranscriptLedger,
|
||||
format_session_offset,
|
||||
)
|
||||
|
||||
OFFSET_RE = re.compile(r"T\+\d{2,}:\d{2}")
|
||||
|
||||
|
||||
def _service() -> StepMemoryService:
|
||||
return StepMemoryService(ctx=None)
|
||||
|
||||
|
||||
def _observation(i: int) -> HumanMessage:
|
||||
return HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": f"# CURRENT OBSERVATION [T+00:0{i % 10}]"},
|
||||
{"type": "text", "text": f"{PLAN_RECITATION_MARKER}\n- [/] milestone {i}"},
|
||||
{"type": "text", "text": "--- Current Screenshot ---"},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG_{i}"}},
|
||||
{"type": "text", "text": f"{PRO_UI_LIST_MARKER}\n[1] button {i}"},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _turn(i: int, tool_call_count: int = 1) -> list:
|
||||
messages: list = [_observation(i)]
|
||||
if tool_call_count:
|
||||
tool_calls = [
|
||||
{"name": "click", "args": {"target": n + 1}, "id": f"tc{i}-{n}", "type": "tool_call"}
|
||||
for n in range(tool_call_count)
|
||||
]
|
||||
messages.append(AIMessage(content=f"thinking {i}", tool_calls=tool_calls))
|
||||
for tc in tool_calls:
|
||||
messages.append(ToolMessage(tool_call_id=tc["id"], content="Action Recorded"))
|
||||
return messages
|
||||
|
||||
|
||||
def _play_turn(ledger: TranscriptLedger, i: int, *, prev_key=None, prev_result=None, **turn_kwargs):
|
||||
"""Mimic the operator's real ordering: commit previous, render, stage."""
|
||||
ledger.commit_staged(step_key=prev_key, validator_result=prev_result)
|
||||
turn = _turn(i, **turn_kwargs)
|
||||
rendered = ledger.render([turn[0]])
|
||||
ledger.stage_turn(turn)
|
||||
return rendered
|
||||
|
||||
|
||||
def _fingerprint(msg) -> str:
|
||||
return json.dumps(msg.content, sort_keys=True, default=str)
|
||||
|
||||
|
||||
def _has_image(msg) -> bool:
|
||||
return any(
|
||||
isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content
|
||||
)
|
||||
|
||||
|
||||
def test_format_session_offset():
|
||||
assert format_session_offset(0) == "T+00:00"
|
||||
assert format_session_offset(83) == "T+01:23"
|
||||
# Minutes never wrap into hours: byte-stable monotonic labels.
|
||||
assert format_session_offset(3725) == "T+62:05"
|
||||
assert format_session_offset(-5) == "T+00:00"
|
||||
|
||||
|
||||
def test_static_prefix_set_once_and_byte_stable():
|
||||
ledger = TranscriptLedger(step_memory=_service())
|
||||
system = SystemMessage(content="STATIC SYSTEM PROMPT")
|
||||
ledger.set_static_prefix([system])
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
ledger.set_static_prefix([SystemMessage(content="other")])
|
||||
|
||||
fingerprints = set()
|
||||
for i in range(1, 6):
|
||||
rendered = _play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
|
||||
assert rendered[0] is system
|
||||
fingerprints.add(rendered[0].content)
|
||||
assert fingerprints == {"STATIC SYSTEM PROMPT"}
|
||||
|
||||
|
||||
def test_active_region_is_append_only():
|
||||
ledger = TranscriptLedger(step_memory=_service())
|
||||
seen_ids: list[int] = []
|
||||
for i in range(1, 5):
|
||||
_play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
|
||||
active = ledger.active_messages
|
||||
# The previously observed prefix is unchanged in identity and order.
|
||||
assert [id(m) for m in active[: len(seen_ids)]] == seen_ids
|
||||
seen_ids = [id(m) for m in active]
|
||||
assert ledger.turn_count == 3 # 3 committed, 1 still staged
|
||||
|
||||
|
||||
def test_validator_result_message_carries_session_offset():
|
||||
ledger = TranscriptLedger(step_memory=_service())
|
||||
_play_turn(ledger, 1)
|
||||
_play_turn(
|
||||
ledger,
|
||||
2,
|
||||
prev_key="step-1",
|
||||
prev_result={"status": "success", "execution": [{"action": "tap", "attempts": ["ok"]}]},
|
||||
)
|
||||
|
||||
result_messages = [
|
||||
m
|
||||
for m in ledger.active_messages
|
||||
if isinstance(m, HumanMessage)
|
||||
and str(m.content).find(EXECUTION_RESULT_MARKER) >= 0
|
||||
]
|
||||
assert len(result_messages) == 1
|
||||
text = result_messages[0].content[0]["text"]
|
||||
assert OFFSET_RE.search(text), text
|
||||
assert "Status: success" in text
|
||||
assert "ago" not in text
|
||||
|
||||
|
||||
def test_validator_result_skipped_for_actionless_turn():
|
||||
ledger = TranscriptLedger(step_memory=_service())
|
||||
_play_turn(ledger, 1)
|
||||
_play_turn(ledger, 2, prev_key="step-1", prev_result=None)
|
||||
assert not any(
|
||||
EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages
|
||||
)
|
||||
|
||||
|
||||
def test_old_turn_recitation_and_ui_list_stripped_at_depth_1():
|
||||
ledger = TranscriptLedger(step_memory=_service())
|
||||
_play_turn(ledger, 1)
|
||||
rendered = _play_turn(ledger, 2, prev_key="step-1")
|
||||
|
||||
committed_obs = ledger.active_messages[0]
|
||||
committed_text = str(committed_obs.content)
|
||||
# Depth-1 scrub: the previous turn's plan recitation and UI list are gone.
|
||||
assert PLAN_RECITATION_MARKER not in committed_text
|
||||
assert PRO_UI_LIST_MARKER not in committed_text
|
||||
# Its screenshot is still inside the K-depth active window.
|
||||
assert _has_image(committed_obs)
|
||||
# The live tail keeps both.
|
||||
tail_text = str(rendered[-1].content)
|
||||
assert PLAN_RECITATION_MARKER in tail_text
|
||||
assert PRO_UI_LIST_MARKER in tail_text
|
||||
|
||||
|
||||
def test_depth_k_image_resolved_to_ready_visual_summary():
|
||||
service = _service()
|
||||
service._step_inputs["step-1"] = {"step_number": 1}
|
||||
service._summaries["step-1"] = "Objective visual transition 1."
|
||||
|
||||
ledger = TranscriptLedger(step_memory=service, image_scrub_depth=3)
|
||||
for i in range(1, 5):
|
||||
_play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
|
||||
|
||||
first_obs = ledger.active_messages[0]
|
||||
assert not _has_image(first_obs)
|
||||
assert {
|
||||
"type": "text",
|
||||
"text": "--- Historical Visual Transition ---\nObjective visual transition 1.",
|
||||
} in first_obs.content
|
||||
|
||||
|
||||
def test_pending_grace_then_placeholder_never_backfilled():
|
||||
service = _service()
|
||||
service._step_inputs["step-1"] = {"step_number": 1} # pending job, no summary
|
||||
|
||||
ledger = TranscriptLedger(
|
||||
step_memory=service, image_scrub_depth=2, pending_grace_steps=1
|
||||
)
|
||||
# Enough turns to push turn 1 past K + grace.
|
||||
for i in range(1, 7):
|
||||
_play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
|
||||
|
||||
first_obs = ledger.active_messages[0]
|
||||
assert not _has_image(first_obs)
|
||||
assert "[visual summary pending; evidence at DataEngine step 1]" in str(
|
||||
first_obs.content
|
||||
)
|
||||
frozen = _fingerprint(first_obs)
|
||||
|
||||
# A late summary must never mutate the frozen message.
|
||||
service._summaries["step-1"] = "Too late."
|
||||
_play_turn(ledger, 7, prev_key="step-6")
|
||||
assert _fingerprint(ledger.active_messages[0]) == frozen
|
||||
|
||||
|
||||
def test_failed_summary_becomes_unavailable_placeholder():
|
||||
service = _service()
|
||||
service._step_inputs["step-1"] = {"step_number": 1}
|
||||
service._failed.add("step-1")
|
||||
|
||||
ledger = TranscriptLedger(
|
||||
step_memory=service, image_scrub_depth=2, pending_grace_steps=5
|
||||
)
|
||||
for i in range(1, 4):
|
||||
_play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
|
||||
|
||||
first_obs = ledger.active_messages[0]
|
||||
assert not _has_image(first_obs)
|
||||
assert "[visual summary unavailable; evidence at DataEngine step 1]" in str(
|
||||
first_obs.content
|
||||
)
|
||||
|
||||
|
||||
def test_tool_call_response_pairs_are_never_split():
|
||||
ledger = TranscriptLedger(step_memory=_service(), image_scrub_depth=2)
|
||||
for i in range(1, 7):
|
||||
_play_turn(
|
||||
ledger,
|
||||
i,
|
||||
prev_key=f"step-{i - 1}" if i > 1 else None,
|
||||
prev_result={"status": "success"} if i > 1 else None,
|
||||
tool_call_count=2,
|
||||
)
|
||||
|
||||
active = list(ledger.active_messages)
|
||||
for idx, msg in enumerate(active):
|
||||
if isinstance(msg, AIMessage) and msg.tool_calls:
|
||||
following = active[idx + 1 : idx + 1 + len(msg.tool_calls)]
|
||||
assert [getattr(m, "tool_call_id", None) for m in following] == [
|
||||
tc["id"] for tc in msg.tool_calls
|
||||
], f"tool-call pairing split at active index {idx}"
|
||||
|
||||
|
||||
def test_restored_history_only_seeds_an_empty_ledger():
|
||||
ledger = TranscriptLedger(step_memory=_service())
|
||||
ledger.set_restored_history(f"{RESTORED_HISTORY_HEADER} steps 1-9 ...")
|
||||
assert ledger.has_restored_history
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
ledger.set_restored_history("again")
|
||||
|
||||
ledger.set_static_prefix([SystemMessage(content="S")])
|
||||
rendered = _play_turn(ledger, 1)
|
||||
# Order: S region, then the frozen restored block, then the live tail.
|
||||
assert rendered[0].content == "S"
|
||||
assert RESTORED_HISTORY_HEADER in str(rendered[1].content)
|
||||
assert "# CURRENT OBSERVATION" in str(rendered[-1].content)
|
||||
|
||||
ledger2 = TranscriptLedger(step_memory=_service())
|
||||
_play_turn(ledger2, 1)
|
||||
ledger2.commit_staged(step_key="step-1")
|
||||
with pytest.raises(RuntimeError):
|
||||
ledger2.set_restored_history("late")
|
||||
|
||||
|
||||
def test_stage_twice_commits_the_forgotten_turn():
|
||||
ledger = TranscriptLedger(step_memory=_service())
|
||||
ledger.stage_turn(_turn(1))
|
||||
ledger.stage_turn(_turn(2))
|
||||
assert ledger.turn_count == 1
|
||||
assert ledger.has_staged_turn
|
||||
|
||||
|
||||
def test_no_ago_wording_in_ledger_output():
|
||||
ledger = TranscriptLedger(step_memory=_service())
|
||||
for i in range(1, 4):
|
||||
_play_turn(
|
||||
ledger,
|
||||
i,
|
||||
prev_key=f"step-{i - 1}" if i > 1 else None,
|
||||
prev_result={"status": "success"} if i > 1 else None,
|
||||
)
|
||||
blob = " ".join(str(m.content) for m in ledger.active_messages)
|
||||
assert " ago" not in blob
|
||||
@@ -0,0 +1,134 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for the agent.memory configuration block (M1/M2)."""
|
||||
|
||||
from artemis.config import AgentGlobalConfig
|
||||
|
||||
|
||||
def test_memory_defaults():
|
||||
cfg = AgentGlobalConfig.model_validate({})
|
||||
# 2 per the 2026-09-01 on-device baseline (serial queue at 1 pushed
|
||||
# summary-ready P90 past the scrub-edge grace window).
|
||||
assert cfg.memory.runtime.max_concurrency == 2
|
||||
assert cfg.memory.runtime.retry_limit == 3
|
||||
assert cfg.memory.runtime.flush_timeout_s == 30.0
|
||||
# M5 (2026-09-01): the transcript path ships ON by default; explicit
|
||||
# false is the byte-for-byte rollback switch (see the flip test below).
|
||||
assert cfg.memory.transcript.enabled is True
|
||||
assert cfg.memory.transcript.image_scrub_depth == 3
|
||||
assert cfg.memory.transcript.pending_grace_steps == 3
|
||||
assert cfg.memory.transcript.xml_scrub_depth == 1
|
||||
|
||||
|
||||
def test_memory_transcript_flag_explicit_disable_is_rollback_switch():
|
||||
cfg = AgentGlobalConfig.model_validate(
|
||||
{"memory": {"transcript": {"enabled": False}}}
|
||||
)
|
||||
assert cfg.memory.transcript.enabled is False
|
||||
# Sibling scrub defaults are unaffected by flipping the flag.
|
||||
assert cfg.memory.transcript.image_scrub_depth == 3
|
||||
|
||||
|
||||
def test_memory_explicit_values():
|
||||
cfg = AgentGlobalConfig.model_validate(
|
||||
{
|
||||
"memory": {
|
||||
"runtime": {"max_concurrency": 4, "retry_limit": 5, "flush_timeout_s": 10},
|
||||
"transcript": {
|
||||
"image_scrub_depth": 5,
|
||||
"pending_grace_steps": 1,
|
||||
"xml_scrub_depth": 2,
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
assert cfg.memory.runtime.max_concurrency == 4
|
||||
assert cfg.memory.runtime.retry_limit == 5
|
||||
assert cfg.memory.runtime.flush_timeout_s == 10.0
|
||||
assert cfg.memory.transcript.image_scrub_depth == 5
|
||||
assert cfg.memory.transcript.pending_grace_steps == 1
|
||||
assert cfg.memory.transcript.xml_scrub_depth == 2
|
||||
|
||||
|
||||
def test_legacy_step_summarizer_retry_limit_seeds_memory_runtime():
|
||||
"""An explicit legacy flash.step_summarizer.retry_limit keeps working."""
|
||||
cfg = AgentGlobalConfig.model_validate(
|
||||
{"flash": {"step_summarizer": {"retry_limit": 7}}}
|
||||
)
|
||||
assert cfg.flash.step_summarizer.retry_limit == 7
|
||||
assert cfg.memory.runtime.retry_limit == 7
|
||||
|
||||
|
||||
def test_explicit_memory_runtime_wins_over_legacy_alias():
|
||||
cfg = AgentGlobalConfig.model_validate(
|
||||
{
|
||||
"flash": {"step_summarizer": {"retry_limit": 7}},
|
||||
"memory": {"runtime": {"retry_limit": 2}},
|
||||
}
|
||||
)
|
||||
assert cfg.memory.runtime.retry_limit == 2
|
||||
# The legacy key itself is untouched for back-compat readers.
|
||||
assert cfg.flash.step_summarizer.retry_limit == 7
|
||||
|
||||
|
||||
def test_legacy_default_does_not_override_memory_default():
|
||||
"""A step_summarizer block without retry_limit must not seed anything."""
|
||||
cfg = AgentGlobalConfig.model_validate(
|
||||
{"flash": {"step_summarizer": {"enabled": True}}}
|
||||
)
|
||||
assert cfg.memory.runtime.retry_limit == 3
|
||||
|
||||
|
||||
def test_recall_and_similarity_defaults():
|
||||
"""M4 defaults: recall block on, bounded; similarity hint on. The distance
|
||||
threshold was calibrated to 5 in M5 (2026-09-01 on-device dHash data:
|
||||
same screen <=4, different screens >=7, valley at 5-6)."""
|
||||
cfg = AgentGlobalConfig()
|
||||
assert cfg.memory.recall.enabled is True
|
||||
assert cfg.memory.recall.max_results == 5
|
||||
assert cfg.memory.recall.max_text_tokens == 2000
|
||||
assert cfg.memory.recall.max_image_steps == 1
|
||||
assert cfg.memory.transcript.similarity_hint is True
|
||||
assert cfg.memory.transcript.similarity_max_distance == 5
|
||||
assert cfg.memory.policies == {}
|
||||
|
||||
|
||||
def test_chunking_era_cap_follows_max_chunks_by_default():
|
||||
"""M5: max_eras is an independent key; None (default) follows max_chunks."""
|
||||
cfg = AgentGlobalConfig()
|
||||
assert cfg.memory.chunking.max_chunks == 8
|
||||
assert cfg.memory.chunking.max_eras is None
|
||||
|
||||
overridden = AgentGlobalConfig.model_validate(
|
||||
{"memory": {"chunking": {"max_chunks": 12, "max_eras": 4}}}
|
||||
)
|
||||
assert overridden.memory.chunking.max_chunks == 12
|
||||
assert overridden.memory.chunking.max_eras == 4
|
||||
|
||||
|
||||
def test_recall_and_policies_overridable():
|
||||
cfg = AgentGlobalConfig.model_validate(
|
||||
{
|
||||
"memory": {
|
||||
"recall": {"enabled": False, "max_results": 3},
|
||||
"transcript": {"similarity_hint": False},
|
||||
"policies": {"planner": {"last_n_detailed": 3}},
|
||||
}
|
||||
}
|
||||
)
|
||||
assert cfg.memory.recall.enabled is False
|
||||
assert cfg.memory.recall.max_results == 3
|
||||
assert cfg.memory.transcript.similarity_hint is False
|
||||
assert cfg.memory.policies == {"planner": {"last_n_detailed": 3}}
|
||||
@@ -0,0 +1,173 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Tests for the M0 session token metering pipeline (pure recording)."""
|
||||
|
||||
from unittest.mock import Mock
|
||||
from uuid import uuid4
|
||||
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
from artemis.services.token_meter import (
|
||||
SessionTokenMeter,
|
||||
extract_usage,
|
||||
get_meter,
|
||||
record_llm_usage,
|
||||
)
|
||||
|
||||
|
||||
def _msg_with_usage(prompt=1000, completion=50, cached=0):
|
||||
usage = {
|
||||
"input_tokens": prompt,
|
||||
"output_tokens": completion,
|
||||
"total_tokens": prompt + completion,
|
||||
}
|
||||
if cached:
|
||||
usage["input_token_details"] = {"cache_read": cached}
|
||||
return AIMessage(content="ok", usage_metadata=usage)
|
||||
|
||||
|
||||
def test_extract_usage_from_usage_metadata():
|
||||
usage = extract_usage(_msg_with_usage(prompt=1200, completion=80, cached=900))
|
||||
assert usage == {
|
||||
"prompt_tokens": 1200,
|
||||
"completion_tokens": 80,
|
||||
"total_tokens": 1280,
|
||||
"cached_tokens": 900,
|
||||
}
|
||||
|
||||
|
||||
def test_extract_usage_from_response_metadata_fallback():
|
||||
msg = AIMessage(content="ok")
|
||||
msg.response_metadata = {
|
||||
"usage_metadata": {
|
||||
"prompt_token_count": 500,
|
||||
"candidates_token_count": 25,
|
||||
"total_token_count": 525,
|
||||
"cached_content_token_count": 400,
|
||||
}
|
||||
}
|
||||
usage = extract_usage(msg)
|
||||
assert usage == {
|
||||
"prompt_tokens": 500,
|
||||
"completion_tokens": 25,
|
||||
"total_tokens": 525,
|
||||
"cached_tokens": 400,
|
||||
}
|
||||
|
||||
|
||||
def test_extract_usage_absent_returns_none():
|
||||
assert extract_usage(AIMessage(content="no usage")) is None
|
||||
assert extract_usage(object()) is None
|
||||
|
||||
|
||||
def test_meter_accumulates_session_totals():
|
||||
meter = SessionTokenMeter("s1")
|
||||
meter.record({"prompt_tokens": 100, "completion_tokens": 10, "cached_tokens": 0})
|
||||
snapshot = meter.record(
|
||||
{"prompt_tokens": 300, "completion_tokens": 20, "cached_tokens": 250}
|
||||
)
|
||||
assert snapshot == {
|
||||
"session_llm_calls": 2,
|
||||
"session_prompt_tokens": 400,
|
||||
"session_completion_tokens": 30,
|
||||
"session_cached_tokens": 250,
|
||||
"session_cache_hit_calls": 1,
|
||||
}
|
||||
assert meter.last_prompt_tokens == 300
|
||||
|
||||
|
||||
def test_record_llm_usage_records_trace_with_context_base():
|
||||
engine = Mock()
|
||||
engine.current_session_id = uuid4()
|
||||
engine.current_step_id = uuid4()
|
||||
|
||||
payload = record_llm_usage(
|
||||
engine, _msg_with_usage(prompt=2000, completion=100, cached=1500), source="google:test"
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
assert payload["context_base_tokens"] == 2000
|
||||
assert payload["cached_tokens"] == 1500
|
||||
assert payload["source"] == "google:test"
|
||||
|
||||
engine.record_trace.assert_called_once()
|
||||
kwargs = engine.record_trace.call_args.kwargs
|
||||
assert kwargs["type"] == "llm_call"
|
||||
assert kwargs["name"] == "llm_usage"
|
||||
assert kwargs["step_id"] == engine.current_step_id
|
||||
assert kwargs["payload"]["prompt_tokens"] == 2000
|
||||
|
||||
|
||||
def test_record_llm_usage_accumulates_across_calls_same_session():
|
||||
engine = Mock()
|
||||
engine.current_session_id = uuid4()
|
||||
engine.current_step_id = None
|
||||
|
||||
record_llm_usage(engine, _msg_with_usage(prompt=100, completion=5))
|
||||
payload = record_llm_usage(engine, _msg_with_usage(prompt=250, completion=10, cached=90))
|
||||
|
||||
assert payload["session_llm_calls"] == 2
|
||||
assert payload["session_prompt_tokens"] == 350
|
||||
assert payload["session_cached_tokens"] == 90
|
||||
assert payload["session_cache_hit_calls"] == 1
|
||||
|
||||
meter = get_meter(engine.current_session_id)
|
||||
assert meter.last_prompt_tokens == 250
|
||||
|
||||
|
||||
def test_record_llm_usage_never_raises_and_skips_gracefully():
|
||||
# No engine
|
||||
assert record_llm_usage(None, _msg_with_usage()) is None
|
||||
|
||||
# Engine without a session
|
||||
engine = Mock()
|
||||
engine.current_session_id = None
|
||||
assert record_llm_usage(engine, _msg_with_usage()) is None
|
||||
engine.record_trace.assert_not_called()
|
||||
|
||||
# Response without usage
|
||||
engine2 = Mock()
|
||||
engine2.current_session_id = uuid4()
|
||||
assert record_llm_usage(engine2, AIMessage(content="nope")) is None
|
||||
engine2.record_trace.assert_not_called()
|
||||
|
||||
# record_trace blowing up must not propagate
|
||||
engine3 = Mock()
|
||||
engine3.current_session_id = uuid4()
|
||||
engine3.record_trace.side_effect = RuntimeError("boom")
|
||||
assert record_llm_usage(engine3, _msg_with_usage()) is None
|
||||
|
||||
|
||||
def test_record_llm_usage_lens_calls_do_not_touch_last_prompt_tokens():
|
||||
"""Background lens calls accumulate totals but never overwrite the live
|
||||
context base (last_prompt_tokens) consumed by the compaction thresholds."""
|
||||
engine = Mock()
|
||||
engine.current_session_id = uuid4()
|
||||
engine.current_step_id = None
|
||||
|
||||
record_llm_usage(engine, _msg_with_usage(prompt=15000))
|
||||
meter = get_meter(engine.current_session_id)
|
||||
assert meter.last_prompt_tokens == 15000
|
||||
|
||||
payload = record_llm_usage(
|
||||
engine,
|
||||
_msg_with_usage(prompt=300, completion=40),
|
||||
source="lens:visual_transition:test-model",
|
||||
update_last_prompt=False,
|
||||
)
|
||||
assert payload["source"] == "lens:visual_transition:test-model"
|
||||
assert meter.last_prompt_tokens == 15000 # untouched by the lens call
|
||||
assert meter.llm_calls == 2
|
||||
assert meter.prompt_tokens == 15300
|
||||
@@ -0,0 +1,265 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""recall_history boundaries and search correctness (M4, design §10).
|
||||
|
||||
Hard boundaries under test: bounded result count, bounded response tokens,
|
||||
image recall limited to the real stored screenshots of a single step, and
|
||||
every result carrying a step number. Plus: keyword/step-range/notes filter
|
||||
correctness and the era-marker re-entry (a step_range returns the full-width
|
||||
``build_action_ledger`` rows for that range).
|
||||
"""
|
||||
|
||||
import io
|
||||
import re
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
from artemis.tools.history_recall import (
|
||||
RecallHistoryTool,
|
||||
recall_history,
|
||||
search_history,
|
||||
)
|
||||
from artemis.utils.notes import save_note_content
|
||||
|
||||
|
||||
def _jpeg(color: str) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (64, 128), color).save(buf, format="JPEG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _make_engine(tmp_path):
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_execution_setup = MagicMock()
|
||||
mock_execution_setup.traces_path = str(tmp_path)
|
||||
mock_ctx.execution_setup = mock_execution_setup
|
||||
mock_ctx.device = None
|
||||
engine = DataEngine(mock_ctx)
|
||||
engine.start_session("recall test session")
|
||||
return engine
|
||||
|
||||
|
||||
def _flush(engine):
|
||||
for t in list(engine._pending_threads):
|
||||
t.join()
|
||||
|
||||
|
||||
def _cfg(**overrides):
|
||||
values = {"enabled": True, "max_results": 5, "max_text_tokens": 2000, "max_image_steps": 1}
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def engine(tmp_path):
|
||||
engine = _make_engine(tmp_path)
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=_jpeg("red"),
|
||||
summary="Opened the login page.",
|
||||
action_taken={"action": "click", "target_text": "Login entry"},
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=_jpeg("green"),
|
||||
post_screenshot_bytes=_jpeg("blue"),
|
||||
summary="Typed the verification code 4711 into the input.",
|
||||
action_taken={"action": "input_text", "text": "4711"},
|
||||
last_execution_result={"status": "success"},
|
||||
operator_raw_thinking="The code arrived via SMS; entering 4711 now.",
|
||||
)
|
||||
engine.record_step(
|
||||
summary="A promo popup appeared and was dismissed.",
|
||||
action_taken={"action": "click", "target_text": "Close popup"},
|
||||
last_execution_result={"status": "failed", "error": "element vanished"},
|
||||
)
|
||||
_flush(engine)
|
||||
return engine
|
||||
|
||||
|
||||
def _result_blocks(text: str) -> list[str]:
|
||||
return [b for b in text.split("\n") if b.startswith("[")]
|
||||
|
||||
|
||||
def test_keyword_search_finds_step_and_carries_step_number(engine):
|
||||
out = search_history(engine, query="verification code", recall_config=_cfg())
|
||||
assert isinstance(out, str)
|
||||
assert "Step 2" in out
|
||||
assert "4711" in out
|
||||
# Every result header names a step / step range / step anchor.
|
||||
for header in _result_blocks(out):
|
||||
assert re.search(r"Step[s]? \d", header), header
|
||||
|
||||
|
||||
def test_step_range_filters_matches(engine):
|
||||
out = search_history(
|
||||
engine, query="popup", step_range=[1, 2], recall_config=_cfg()
|
||||
)
|
||||
# Step 3 (the popup step) is outside the range: only the range ledger and
|
||||
# a no-match line may appear.
|
||||
assert "A promo popup appeared" not in out
|
||||
assert "No matches" in out
|
||||
|
||||
|
||||
def test_step_range_returns_full_width_ledger_rows(engine):
|
||||
"""The recall-only era period paragraph's '(Step-level ledger via
|
||||
recall_history for steps a–b)' re-entry point."""
|
||||
out = search_history(engine, query="", step_range=[1, 3], recall_config=_cfg())
|
||||
assert "Action ledger for Steps 1–3" in out
|
||||
# Full-width build_action_ledger rows: step number + T+mm:ss offset +
|
||||
# semantic action + result phrase.
|
||||
assert re.search(r"- Step 1 \(T\+\d{2}:\d{2}\): .*Login entry", out)
|
||||
assert re.search(r"- Step 2 \(T\+\d{2}:\d{2}\): ", out)
|
||||
assert re.search(r"- Step 3 \(T\+\d{2}:\d{2}\): ", out)
|
||||
|
||||
|
||||
def test_result_count_is_clamped_by_config(engine):
|
||||
for i in range(8):
|
||||
engine.record_step(
|
||||
summary=f"Scrolled the anchorterm feed page {i}.",
|
||||
action_taken={"action": "swipe"},
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
_flush(engine)
|
||||
out = search_history(
|
||||
engine, query="anchorterm", max_results=50, recall_config=_cfg(max_results=3)
|
||||
)
|
||||
assert out.count("[Step ") == 3
|
||||
assert "more not shown" in out
|
||||
|
||||
|
||||
def test_response_is_truncated_at_token_budget(engine):
|
||||
engine.record_step(
|
||||
summary="needleword " + "filler content " * 400,
|
||||
action_taken={"action": "click"},
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
_flush(engine)
|
||||
out = search_history(
|
||||
engine,
|
||||
query="needleword",
|
||||
include_details=True,
|
||||
recall_config=_cfg(max_text_tokens=150),
|
||||
)
|
||||
assert len(out) <= 150 * 4 + 200 # budget + truncation notice
|
||||
assert "truncated at the recall token budget" in out
|
||||
|
||||
|
||||
def test_notes_are_searchable_and_anchored_to_a_step(engine):
|
||||
save_note_content(engine.base_dir, "login_flow", "The OTP entry lives behind the SMS tab.")
|
||||
out = search_history(engine, query="OTP entry", recall_config=_cfg())
|
||||
assert "[Note 'login_flow'" in out
|
||||
assert re.search(r"as of Step \d|last written at Step \d", out)
|
||||
|
||||
|
||||
def test_chunk_rows_are_searchable(engine):
|
||||
engine.record_history_chunk(
|
||||
start_step_number=1,
|
||||
end_step_number=2,
|
||||
version=1,
|
||||
status="ready",
|
||||
band2=" - Steps 1–2: walked the chunkneedle flow",
|
||||
band3="- Step 1 (T+00:01): click -> executed",
|
||||
rendered_text=(
|
||||
"[Chunk 1 | Steps 1–2]\n② Compressed step summary\n"
|
||||
" - Steps 1–2: walked the chunkneedle flow\n"
|
||||
"③ Step action ledger\n- Step 1 (T+00:01): click -> executed"
|
||||
),
|
||||
)
|
||||
_flush(engine)
|
||||
out = search_history(engine, query="chunkneedle", recall_config=_cfg())
|
||||
assert "[History chunk | Steps 1–2" in out
|
||||
|
||||
|
||||
def test_images_only_for_a_single_step_and_only_real_files(engine):
|
||||
out = search_history(
|
||||
engine,
|
||||
query="verification code",
|
||||
include_images=True,
|
||||
recall_config=_cfg(),
|
||||
)
|
||||
assert isinstance(out, list)
|
||||
images = [b for b in out if b.get("type") == "image_url"]
|
||||
# Step 2 has a real pre and post screenshot on disk — but never more than
|
||||
# one step's worth of images.
|
||||
assert 1 <= len(images) <= 2
|
||||
labels = [b["text"] for b in out if b.get("type") == "text" and "screenshot of Step" in b.get("text", "")]
|
||||
assert all("Step 2" in label for label in labels)
|
||||
for block in images:
|
||||
assert block["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
||||
|
||||
|
||||
def test_images_disabled_by_config_cap(engine):
|
||||
out = search_history(
|
||||
engine,
|
||||
query="verification code",
|
||||
include_images=True,
|
||||
recall_config=_cfg(max_image_steps=0),
|
||||
)
|
||||
assert isinstance(out, str)
|
||||
|
||||
|
||||
def test_missing_image_files_yield_note_not_blocks(engine):
|
||||
# Step 3 recorded no screenshots at all.
|
||||
out = search_history(
|
||||
engine,
|
||||
query="promo popup",
|
||||
include_images=True,
|
||||
recall_config=_cfg(),
|
||||
)
|
||||
assert isinstance(out, list)
|
||||
assert not [b for b in out if b.get("type") == "image_url"]
|
||||
assert any(
|
||||
"no stored screenshots exist" in b.get("text", "") for b in out if b.get("type") == "text"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_requires_query_or_range(engine):
|
||||
tool = RecallHistoryTool()
|
||||
ctx = SimpleNamespace(data_engine=engine)
|
||||
out = await tool.execute(ctx=ctx, query="", step_range=None)
|
||||
assert "needs a query and/or a step_range" in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_without_engine_degrades():
|
||||
tool = RecallHistoryTool()
|
||||
out = await tool.execute(ctx=SimpleNamespace(data_engine=None), query="x")
|
||||
assert "no active DataEngine session" in out
|
||||
|
||||
|
||||
def test_availability_requires_engine_and_config():
|
||||
assert recall_history.is_available(None) is False
|
||||
assert recall_history.is_available(SimpleNamespace(data_engine=None)) is False
|
||||
assert recall_history.is_available(SimpleNamespace(data_engine=object())) is True
|
||||
|
||||
def test_foreground_app_stamp_joins_search_surface(tmp_path):
|
||||
"""M5: a package-name query hits the record_step foreground_app stamp."""
|
||||
engine = _make_engine(tmp_path)
|
||||
engine.record_step(
|
||||
summary="Opened the alarms list.",
|
||||
action_taken={"action": "click", "target_text": "Alarm"},
|
||||
ui_tree=[{"package": "com.google.android.deskclock", "bounds": "[0,0][9,9]"}],
|
||||
)
|
||||
_flush(engine)
|
||||
|
||||
out = search_history(engine, query="deskclock", recall_config=_cfg())
|
||||
assert "Step 1" in out
|
||||
assert "No matches" not in out
|
||||
@@ -0,0 +1,116 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Tests for record-time action semantics enrichment (element hit testing)."""
|
||||
|
||||
from artemis.utils.element_hit_test import find_element_at_point, hit_test_semantics
|
||||
|
||||
|
||||
def _xml_el(text, bounds, resource_id=None, cls="android.widget.TextView"):
|
||||
return {
|
||||
"text": text,
|
||||
"bounds": bounds,
|
||||
"class": cls,
|
||||
"resource_id": resource_id,
|
||||
"is_ocr": False,
|
||||
}
|
||||
|
||||
|
||||
def test_smallest_covering_element_wins():
|
||||
elements = [
|
||||
_xml_el("Whole screen", [0, 0, 1080, 2400]),
|
||||
_xml_el("Card", [100, 100, 500, 500]),
|
||||
_xml_el("Button", [200, 200, 300, 260], resource_id="btn_ok"),
|
||||
]
|
||||
el, source = find_element_at_point(elements, 250, 230)
|
||||
assert source == "hit_test"
|
||||
assert el["text"] == "Button"
|
||||
|
||||
semantics = hit_test_semantics(elements, 250, 230)
|
||||
assert semantics == {
|
||||
"target_label_source": "hit_test",
|
||||
"target_text": "Button",
|
||||
"target_class": "android.widget.TextView",
|
||||
"target_resource_id": "btn_ok",
|
||||
}
|
||||
|
||||
|
||||
def test_xml_element_preferred_over_smaller_ocr_element():
|
||||
elements = [
|
||||
_xml_el("Login", [100, 100, 500, 300]),
|
||||
{
|
||||
"text": "Login OCR",
|
||||
"bounds": [200, 150, 260, 180],
|
||||
"class": None,
|
||||
"resource_id": None,
|
||||
"is_ocr": True,
|
||||
},
|
||||
]
|
||||
el, source = find_element_at_point(elements, 220, 160)
|
||||
assert source == "hit_test"
|
||||
assert el["text"] == "Login"
|
||||
|
||||
|
||||
def test_ocr_fallback_when_no_xml_element_covers_point():
|
||||
elements = [
|
||||
_xml_el("Elsewhere", [800, 800, 900, 900]),
|
||||
{
|
||||
"text": "OCR Label",
|
||||
"bounds": [100, 100, 300, 200],
|
||||
"class": None,
|
||||
"resource_id": None,
|
||||
"is_ocr": True,
|
||||
},
|
||||
]
|
||||
semantics = hit_test_semantics(elements, 150, 150)
|
||||
assert semantics["target_label_source"] == "ocr"
|
||||
assert semantics["target_text"] == "OCR Label"
|
||||
assert "target_class" not in semantics
|
||||
assert "target_resource_id" not in semantics
|
||||
|
||||
|
||||
def test_string_bounds_are_parsed():
|
||||
elements = [_xml_el("Item", "[100,100][300,500]")]
|
||||
el, source = find_element_at_point(elements, 200, 300)
|
||||
assert source == "hit_test"
|
||||
assert el["text"] == "Item"
|
||||
|
||||
|
||||
def test_graceful_degradation_to_none():
|
||||
# No elements at all
|
||||
assert hit_test_semantics(None, 100, 100) == {"target_label_source": "none"}
|
||||
assert hit_test_semantics([], 100, 100) == {"target_label_source": "none"}
|
||||
|
||||
# Point outside every element
|
||||
elements = [_xml_el("Item", [0, 0, 50, 50])]
|
||||
assert hit_test_semantics(elements, 500, 500) == {"target_label_source": "none"}
|
||||
|
||||
# Malformed element data must not raise
|
||||
malformed = [
|
||||
{"text": "no bounds"},
|
||||
{"text": "bad bounds", "bounds": "garbage"},
|
||||
{"text": "short bounds", "bounds": [1, 2]},
|
||||
{"text": "non-numeric", "bounds": ["a", "b", "c", "d"]},
|
||||
"not-a-dict",
|
||||
None,
|
||||
]
|
||||
assert hit_test_semantics(malformed, 10, 10) == {"target_label_source": "none"}
|
||||
|
||||
|
||||
def test_element_without_optional_fields_only_reports_source():
|
||||
elements = [
|
||||
{"text": "", "bounds": [0, 0, 100, 100], "class": None, "resource_id": None}
|
||||
]
|
||||
semantics = hit_test_semantics(elements, 50, 50)
|
||||
assert semantics == {"target_label_source": "hit_test"}
|
||||
@@ -0,0 +1,81 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Perceptual-hash stability and distance behavior (M4 similarity hint)."""
|
||||
|
||||
import io
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
from artemis.utils.image_hash import HASH_SIZE, dhash_hex, hamming_distance_hex
|
||||
|
||||
|
||||
def _screen(draw_fn, size=(270, 600), quality=85) -> bytes:
|
||||
img = Image.new("RGB", size, "white")
|
||||
draw_fn(ImageDraw.Draw(img))
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=quality)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _login_screen(d):
|
||||
d.rectangle([20, 40, 250, 90], fill="#3355ff")
|
||||
d.rectangle([30, 200, 240, 250], outline="#333333", width=4)
|
||||
d.rectangle([30, 300, 240, 350], outline="#333333", width=4)
|
||||
d.rectangle([60, 450, 210, 510], fill="#22aa66")
|
||||
|
||||
|
||||
def _settings_screen(d):
|
||||
for y in range(60, 560, 80):
|
||||
d.rectangle([10, y, 260, y + 50], fill="#dddddd")
|
||||
d.ellipse([200, 20, 250, 45], fill="#aa2222")
|
||||
|
||||
|
||||
def test_dhash_is_stable_for_identical_bytes():
|
||||
img = _screen(_login_screen)
|
||||
assert dhash_hex(img) == dhash_hex(img)
|
||||
assert len(dhash_hex(img)) == HASH_SIZE * HASH_SIZE // 4
|
||||
|
||||
|
||||
def test_dhash_survives_jpeg_reencoding_with_small_distance():
|
||||
original = _screen(_login_screen, quality=95)
|
||||
with Image.open(io.BytesIO(original)) as img:
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="JPEG", quality=60)
|
||||
reencoded = buf.getvalue()
|
||||
|
||||
distance = hamming_distance_hex(dhash_hex(original), dhash_hex(reencoded))
|
||||
assert distance is not None and distance <= 4
|
||||
|
||||
|
||||
def test_dhash_separates_different_screens():
|
||||
distance = hamming_distance_hex(
|
||||
dhash_hex(_screen(_login_screen)), dhash_hex(_screen(_settings_screen))
|
||||
)
|
||||
assert distance is not None and distance > 8
|
||||
|
||||
|
||||
def test_dhash_best_effort_on_garbage():
|
||||
assert dhash_hex(None) is None
|
||||
assert dhash_hex(b"") is None
|
||||
assert dhash_hex(b"not an image at all") is None
|
||||
|
||||
|
||||
def test_hamming_distance_edge_cases():
|
||||
assert hamming_distance_hex(None, "ff") is None
|
||||
assert hamming_distance_hex("ff", None) is None
|
||||
assert hamming_distance_hex("ff", "ffff") is None # length mismatch
|
||||
assert hamming_distance_hex("zz", "aa") is None # not hex
|
||||
assert hamming_distance_hex("00ff", "00ff") == 0
|
||||
assert hamming_distance_hex("0000", "000f") == 4
|
||||
@@ -365,7 +365,7 @@ def test_build_plan_and_history_concatenates_thoughts():
|
||||
assert " - Raw thought" in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_chronological_last_step():
|
||||
def test_build_plan_and_history_interleaved_decision_loop():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
@@ -390,9 +390,7 @@ def test_build_plan_and_history_chronological_last_step():
|
||||
],
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(
|
||||
plan, steps, "default", last_n_detailed=1, chronological_last_step=True
|
||||
)
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
|
||||
assert plan in output
|
||||
assert "--- Execution History ---" in output
|
||||
@@ -464,9 +462,7 @@ def test_build_plan_and_history_safety_net_and_failure_analyzer():
|
||||
],
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(
|
||||
plan, steps, "default", last_n_detailed=1, chronological_last_step=True
|
||||
)
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
|
||||
assert "- **Step 1 (Most Recent Step, Start: 5.0s)**" in output
|
||||
assert "* [Operator Decision Loop]:" in output
|
||||
|
||||
Reference in new issue
Block a user