feat: unify history tools and improve execution reliability

This commit is contained in:
somew1nd committed 2026-09-03 16:05:47 -07:00
1 parent d23a4f6472
commit 2fea9865ee
116 files changed
+8291 -2503

No files matched your search

@@ -14,6 +14,7 @@
"""Recording files of cancelled / killed workers must survive and be served whole."""
import json
import os
from pathlib import Path
import time
@@ -171,3 +172,61 @@ def test_recover_orphaned_recording_returns_none_without_files(tmp_path):
missing = tmp_path / "missing" / "recording.mkv"
assert MediaService.recover_orphaned_recording(str(missing)) is None
assert MediaService.recover_orphaned_recording(None) is None
def test_resolve_video_segments_passes_through_session_offsets(tmp_path, monkeypatch):
folder = tmp_path / "artemis-traces" / "web_9_done"
folder.mkdir(parents=True)
(folder / "recording.mp4").write_bytes(b"seg0")
(folder / "recording_001.mp4").write_bytes(b"seg1")
(folder / "recording.json").write_text(
json.dumps(
{
"version": 2,
"duration": 15.0,
"segments": [
{
"file": "recording.mp4",
"start": 0.0,
"duration": 10.0,
"offset_ms": 1200,
"duration_ms": 10000,
"width": 1080,
"height": 1920,
},
{
"file": "recording_001.mp4",
"start": 10.0,
"duration": 5.0,
"offset_ms": 12700,
"duration_ms": 5000,
"width": 1920,
"height": 1080,
},
{"file": "legacy.mp4", "start": 15.0, "duration": 1.0, "width": 1, "height": 1},
],
}
),
encoding="utf-8",
)
(folder / "legacy.mp4").write_bytes(b"legacy")
monkeypatch.setattr(media_module, "WORKSPACE_ROOT", tmp_path)
segments = MediaService.resolve_video_segments(
"/videos/artemis-traces/web_9_done/recording.mp4"
)
assert [s["url"] for s in segments] == [
"/videos/artemis-traces/web_9_done/recording.mp4",
"/videos/artemis-traces/web_9_done/recording_001.mp4",
"/videos/artemis-traces/web_9_done/legacy.mp4",
]
assert [(s["start"], s["duration"]) for s in segments] == [
(0.0, 10.0),
(10.0, 5.0),
(15.0, 1.0),
]
assert segments[0]["offset_ms"] == 1200 and segments[0]["duration_ms"] == 10000
assert segments[1]["offset_ms"] == 12700 and segments[1]["duration_ms"] == 5000
# Legacy entries without offsets stay offset-free so the UI falls back to back-to-back.
assert "offset_ms" not in segments[2]
@@ -45,20 +45,38 @@ def test_get_session_checks_reads_ledger_and_outcome(tmp_path, monkeypatch):
json.dumps({"task_status": "completed", "tests": {"passed": 2, "failed": 0}}),
encoding="utf-8",
)
stream = {
"attempt_id": "abc#1",
"checkpoint_id": "abc",
"phase": "checkpoint",
"trace_id": "trace-1",
"ts": 1.5,
"truncated": False,
"dropped_chars": 0,
"segments": [
{"execution_id": "e1", "role": "thought", "when": 1.1, "text": "Looking"},
{"execution_id": "e1", "role": "answer", "when": 1.2, "text": "Seen"},
],
}
(session_dir / "check_streams.jsonl").write_text(
json.dumps(stream) + "\nnot json\n", encoding="utf-8"
)
result = MediaService.get_session_checks("sess-1")
assert [r["attempt_id"] for r in result["records"]] == ["abc#1", "final#1"]
assert result["records"][0]["subgoal_text"] == "Create the alarm"
assert result["run_outcome"]["task_status"] == "completed"
assert result["streams"] == [stream]
def test_get_session_checks_without_material(tmp_path, monkeypatch):
monkeypatch.setattr(media_service_module, "TRACES_PATH", tmp_path)
(tmp_path / "sess-2").mkdir()
assert MediaService.get_session_checks("sess-2") == {"records": [], "run_outcome": None}
assert MediaService.get_session_checks("missing") == {"records": [], "run_outcome": None}
empty = {"records": [], "streams": [], "run_outcome": None}
assert MediaService.get_session_checks("sess-2") == empty
assert MediaService.get_session_checks("missing") == empty
def test_checks_route_is_registered():
@@ -0,0 +1,119 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""``/api/sessions/{id}/usage``: token totals, live context and run tuning."""
import json
from apps.admin_console.database.connection import db_session
from apps.admin_console.database.repositories.session_repository import SessionRepository
def _insert_session(db_path, session_id, device_info):
with db_session(db_path) as conn:
conn.execute(
"INSERT INTO sessions (session_id, initial_goal, start_time, status, device_info)"
" VALUES (?, ?, ?, ?, ?)",
(session_id, "goal", 1.0, "completed", json.dumps(device_info)),
)
conn.commit()
def _insert_usage(db_path, session_id, trace_id, ts, payload, *, name="llm_usage"):
with db_session(db_path) as conn:
conn.execute(
"INSERT INTO traces (trace_id, session_id, type, name, timestamp, status, payload)"
" VALUES (?, ?, 'llm_call', ?, ?, 'success', ?)",
(trace_id, session_id, name, ts, json.dumps(payload)),
)
conn.commit()
def _usage(prompt, completion, node=None, cached=0):
payload = {
"prompt_tokens": prompt,
"completion_tokens": completion,
"total_tokens": prompt + completion,
"cached_tokens": cached,
"context_base_tokens": prompt,
}
if node:
payload["node"] = node
return payload
def test_usage_sums_all_agents_and_tracks_latest_operator_context(tmp_path):
db_path = tmp_path / "sessions.db"
repo = SessionRepository(db_path)
_insert_session(
db_path,
"s1",
{
"profile": "pro",
"run_tuning": {"verification_level": "checkpoints", "explorer_mode": "pro"},
},
)
_insert_usage(db_path, "s1", "t1", 10.0, _usage(30_000, 500, node="planner"))
_insert_usage(db_path, "s1", "t2", 11.0, _usage(120_000, 800, node="operator", cached=90_000))
_insert_usage(db_path, "s1", "t3", 12.0, _usage(5_000, 100, node="checker"))
_insert_usage(db_path, "s1", "t4", 13.0, _usage(150_000, 700, node="operator"))
# A retry trace shares the type but is not a usage record.
_insert_usage(db_path, "s1", "t5", 14.0, {"error": "503"}, name="llm_retry")
usage = repo.get_session_usage("s1")
assert usage["llm_calls"] == 4
assert usage["prompt_tokens"] == 305_000
assert usage["completion_tokens"] == 2_100
assert usage["total_tokens"] == 307_100
assert usage["cached_tokens"] == 90_000
assert usage["operator_context_tokens"] == 150_000
assert usage["operator_context_updated_at"] == 13.0
assert usage["operator_context_window_tokens"] == 1_000_000
assert usage["profile"] == "pro"
assert usage["run_tuning"] == {"verification_level": "checkpoints", "explorer_mode": "pro"}
def test_usage_flash_runner_counts_as_operator_and_has_no_tuning(tmp_path):
db_path = tmp_path / "sessions.db"
repo = SessionRepository(db_path)
_insert_session(db_path, "s2", {"profile": "flash"})
_insert_usage(db_path, "s2", "t1", 1.0, _usage(20_000, 200, node="FlashRunner"))
usage = repo.get_session_usage("s2")
assert usage["operator_context_tokens"] == 20_000
assert usage["total_tokens"] == 20_200
assert usage["profile"] == "flash"
assert usage["run_tuning"] is None
def test_usage_for_unknown_or_empty_session_is_zeroed(tmp_path):
db_path = tmp_path / "sessions.db"
repo = SessionRepository(db_path)
usage = repo.get_session_usage("missing")
assert usage["llm_calls"] == 0
assert usage["total_tokens"] == 0
assert usage["operator_context_tokens"] is None
assert usage["profile"] is None
assert usage["run_tuning"] is None
def test_usage_route_is_registered():
from apps.admin_console.routers import sessions
paths = {getattr(r, "path", None) for r in sessions.router.routes}
assert "/api/sessions/{session_id}/usage" in paths
+308 -2
View File
@@ -173,8 +173,8 @@ def test_tool_table_probe_gate():
names = {t.name for t in build_checker_tools(ctx, "checkpoint")}
assert "probe_device" in names
assert "read_note" in names
assert "get_step_detail" in names
assert "get_step_screenshot" in names
# The shared history tools, mounted as a set.
assert {"search_history", "replay_steps", "get_step_screenshot"} <= names
# Never any device action, note write, or sub-agent
assert not names & {
"save_note",
@@ -263,3 +263,309 @@ async def test_final_entry_captures_live_screen():
)
capture_mock.assert_awaited_once()
# --- Shared history tools in the check loop ------------------------------------------
def _jpeg_file(tmp_path, color="white"):
from PIL import Image
path = tmp_path / "pre.jpg"
Image.new("RGB", (200, 400), color).save(path, format="JPEG")
return path
def _screenshot_then_verdict(provider, tool_call=None):
"""LLM double: first turn asks for a step screenshot (or ``tool_call``),
second turn answers."""
from types import SimpleNamespace
report = CheckReport(
verdicts=[CheckVerdict(item_text="x", kind="verify", status="passed", evidence="e")]
)
structured = MagicMock()
structured.ainvoke = AsyncMock(return_value=report)
llm = MagicMock()
llm.endpoint = SimpleNamespace(provider=provider)
llm.with_structured_output.return_value = structured
ask = MagicMock()
ask.tool_calls = [
tool_call
or {
"name": "get_step_screenshot",
"args": {"step_number": 2, "which": "pre"},
"id": "tc-shot",
}
]
done = MagicMock()
done.tool_calls = []
return llm, AsyncMock(side_effect=[ask, done])
async def _run_with_screenshot(tmp_path, provider):
path = _jpeg_file(tmp_path)
ctx = _mock_ctx()
ctx.data_engine.get_step_image_path.return_value = path
llm, acomplete = _screenshot_then_verdict(provider)
with (
patch("artemis.agents.checker.checker.get_llm", return_value=llm),
patch("artemis.agents.checker.checker.acomplete", new=acomplete),
):
report = await run_checkpoint_check(
ctx,
check_items=[_ci(text="x")],
anchor=EvidenceAnchor(anchor_step_id="sid", trigger_ts=0.0, plan_text="- [x] G"),
goal="the goal",
subgoal_text="G",
)
ctx.data_engine.get_step_image_path.assert_called_once_with(2, "pre")
# The message list handed to the second model call carries the tool result.
messages = acomplete.call_args_list[1][0][1]
return report, messages
@pytest.mark.asyncio
async def test_step_screenshot_rides_inside_the_tool_message_for_gemini(tmp_path):
from langchain_core.messages import HumanMessage, ToolMessage
report, messages = await _run_with_screenshot(tmp_path, "google")
assert report.verdicts[0].status == "passed"
tool_msgs = [m for m in messages if isinstance(m, ToolMessage)]
assert len(tool_msgs) == 1
assert tool_msgs[0].tool_call_id == "tc-shot"
assert tool_msgs[0].name == "get_step_screenshot"
assert tool_msgs[0].status == "success"
blocks = tool_msgs[0].content
assert blocks[0]["text"] == "Screenshot of step 2 (pre-action) is attached."
assert blocks[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
# No extra human turn was spliced in for this provider.
assert sum(isinstance(m, HumanMessage) for m in messages) == 1
@pytest.mark.asyncio
async def test_step_screenshot_follows_as_a_human_message_for_openai(tmp_path):
from langchain_core.messages import HumanMessage, ToolMessage
_, messages = await _run_with_screenshot(tmp_path, "openai")
tool_msgs = [m for m in messages if isinstance(m, ToolMessage)]
assert len(tool_msgs) == 1
assert tool_msgs[0].content == "Screenshot of step 2 (pre-action) is attached."
follower = messages[messages.index(tool_msgs[0]) + 1]
assert isinstance(follower, HumanMessage)
assert follower.content[0]["text"] == "[Screenshot returned by get_step_screenshot for step 2]"
assert follower.content[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
@pytest.mark.asyncio
async def test_replay_steps_result_is_a_plain_tool_message(tmp_path):
from langchain_core.messages import ToolMessage
ctx = _mock_ctx()
ctx.data_engine.get_agent_friendly_steps_in_range.return_value = [
{
"step_id": "s3",
"step_number": 3,
"relative_time": "9.0s",
"summary": "Saved the alarm.",
"action_taken": {"action": "click", "target": [0.5, 0.5], "target_text": "Save"},
"last_execution_result": {"status": "success"},
"interleaved_events": [],
}
]
llm, acomplete = _screenshot_then_verdict(
"google", {"name": "replay_steps", "args": {"start_step": 3}, "id": "tc-replay"}
)
with (
patch("artemis.agents.checker.checker.get_llm", return_value=llm),
patch("artemis.agents.checker.checker.acomplete", new=acomplete),
):
await run_checkpoint_check(
ctx,
check_items=[_ci(text="x")],
anchor=EvidenceAnchor(anchor_step_id="sid", trigger_ts=0.0, plan_text="- [x] G"),
goal="the goal",
subgoal_text="G",
)
ctx.data_engine.get_agent_friendly_steps_in_range.assert_called_once_with(3, 3)
messages = acomplete.call_args_list[1][0][1]
tool_msg = next(m for m in messages if isinstance(m, ToolMessage))
assert tool_msg.name == "replay_steps"
assert "- **Step 3 (Start: 9.0s)**" in tool_msg.content
assert "[Screen]: Saved the alarm." in tool_msg.content
assert "[Planned Action]: Tapped 'Save' at [0.5, 0.5]" in tool_msg.content
# --- Streamed-reasoning transcript (replay of the live Thought/Work text) -------------
class _FakeBus:
"""Minimal DataEngine bus: subscribers receive every published event."""
def __init__(self):
self.subscribers = []
def subscribe(self, cb):
self.subscribers.append(cb)
def unsubscribe(self, cb):
self.subscribers.remove(cb)
def _publish(self, event_type, data):
for cb in list(self.subscribers):
cb(event_type, data)
def _bus_ctx(tmp_path):
"""Mock engine whose bus actually delivers events (the rest stays mocked)."""
ctx = _mock_ctx()
ctx.data_engine.base_dir = tmp_path
bus = _FakeBus()
ctx.data_engine.subscribers = bus.subscribers
ctx.data_engine.subscribe = bus.subscribe
ctx.data_engine.unsubscribe = bus.unsubscribe
ctx.data_engine._publish = bus._publish
return ctx
def _stream(ctx, exec_id, chunk, *, stream_type="text", parent=None):
from artemis.data_engine.context_vars import CURRENT_TRACE_ID
ctx.data_engine._publish(
"llm_stream",
{
"execution_id": exec_id,
"parent_trace_id": parent or str(CURRENT_TRACE_ID.get()),
"chunk": chunk,
"stream_type": stream_type,
},
)
def _read_streams(tmp_path):
from artemis.graph.checkpoints import read_attempt_streams
return read_attempt_streams(tmp_path)
@pytest.mark.asyncio
async def test_checker_persists_its_streamed_turns_as_timestamped_segments(tmp_path):
"""Every llm_stream chunk emitted under the Checker's own trace is kept as
one segment per (turn, kind) in first-chunk order; other agents' streams
and a discarded (mid-stream reset) turn are left out."""
ctx = _bus_ctx(tmp_path)
async def loop(ctx, messages, tools, items):
_stream(ctx, "turn-1", "Looking at ", stream_type="thinking")
_stream(ctx, "turn-1", "step 2.", stream_type="thinking")
_stream(ctx, "turn-1", "Let me replay it.")
_stream(ctx, "operator-turn", "not mine", parent="someone-else")
_stream(ctx, "turn-2", "partial ")
ctx.data_engine._publish(
"llm_stream_reset", {"stream_exec_id": "turn-2", "action": "discard"}
)
_stream(ctx, "turn-3", "The alarm is there.")
return CheckReport(verdicts=[])
with (
patch("artemis.agents.checker.checker._run_check_loop", side_effect=loop),
patch("artemis.agents.checker.checker.build_checker_tools", return_value=[]),
):
await run_checkpoint_check(
ctx,
check_items=[_ci(text="x")],
anchor=EvidenceAnchor(anchor_step_id="sid", trigger_ts=0.0, plan_text="- [x] G"),
goal="the goal",
subgoal_text="G",
attempt_id="abc#1",
)
assert ctx.data_engine.subscribers == [] # detached after the run
records = _read_streams(tmp_path)
assert len(records) == 1
rec = records[0]
assert rec["attempt_id"] == "abc#1"
assert rec["checkpoint_id"] == "abc"
assert rec["phase"] == "checkpoint"
assert rec["trace_id"]
assert rec["truncated"] is False and rec["dropped_chars"] == 0
assert isinstance(rec["ts"], float)
segs = rec["segments"]
assert [(s["execution_id"], s["role"], s["text"]) for s in segs] == [
("turn-1", "thought", "Looking at step 2."),
("turn-1", "answer", "Let me replay it."),
("turn-3", "answer", "The alarm is there."),
]
assert all(isinstance(s["when"], float) for s in segs)
assert segs[0]["when"] <= segs[1]["when"] <= segs[2]["when"]
@pytest.mark.asyncio
async def test_final_entry_persists_transcript_even_when_the_loop_fails(tmp_path):
"""The transcript is written in a ``finally``: an attempt that times out,
raises or is superseded still leaves what it said."""
ctx = _bus_ctx(tmp_path)
async def loop(ctx, messages, tools, items):
_stream(ctx, "turn-1", "Checking the final screen…")
raise RuntimeError("provider down")
with (
patch("artemis.agents.checker.checker._run_check_loop", side_effect=loop),
patch("artemis.agents.checker.checker.build_checker_tools", return_value=[]),
patch(
"artemis.agents.checker.checker._capture_final_screen",
AsyncMock(return_value=(None, "")),
),
pytest.raises(RuntimeError),
):
await run_final_check(
ctx,
goal="g",
plan_text="- [x] G",
ledger=[],
check_items=[_ci(text="x")],
attempt_id="final#1",
)
records = _read_streams(tmp_path)
assert [r["attempt_id"] for r in records] == ["final#1"]
assert records[0]["phase"] == "final" and records[0]["checkpoint_id"] == "final"
assert records[0]["segments"][0]["text"] == "Checking the final screen…"
@pytest.mark.asyncio
async def test_no_transcript_without_attempt_id_or_streamed_text(tmp_path):
ctx = _bus_ctx(tmp_path)
async def silent(ctx, messages, tools, items):
return CheckReport(verdicts=[])
async def talkative(ctx, messages, tools, items):
_stream(ctx, "turn-1", "text")
return CheckReport(verdicts=[])
with patch("artemis.agents.checker.checker.build_checker_tools", return_value=[]):
with patch("artemis.agents.checker.checker._run_check_loop", side_effect=talkative):
await run_checkpoint_check(
ctx,
check_items=[_ci(text="x")],
anchor=EvidenceAnchor(anchor_step_id="sid", trigger_ts=0.0, plan_text=""),
goal="g",
subgoal_text="G",
)
with patch("artemis.agents.checker.checker._run_check_loop", side_effect=silent):
await run_checkpoint_check(
ctx,
check_items=[_ci(text="x")],
anchor=EvidenceAnchor(anchor_step_id="sid", trigger_ts=0.0, plan_text=""),
goal="g",
subgoal_text="G",
attempt_id="abc#2",
)
assert _read_streams(tmp_path) == []
assert not (tmp_path / "check_streams.jsonl").exists()
+67
View File
@@ -138,3 +138,70 @@ def test_flash_runner_screenshot_pruning(mock_context):
assert messages[2].content[0]["type"] == "text"
assert messages[2].content[1]["type"] == "image_url"
assert messages[2].content[1]["image_url"]["url"] == "data:image/jpeg;base64,SCREENSHOT_3"
# --- Recorded action shape (coordinate space + ledger rendering) ---------------------
def _record(runner, name, args):
from types import SimpleNamespace
runner.ctx.data_engine = Mock()
runner.ctx.data_engine.current_step_id = None
runner.ctx.data_engine.record_step.return_value = "step-1"
exec_result = SimpleNamespace(
status="success", text_summary=f"{name} executed", metadata={}, ui_elements_text="ui"
)
step_id = runner._record_action_step(
name, args, exec_result, "thought", {}, b"pre", ["xml"], b"post"
)
assert step_id == "step-1"
return runner.ctx.data_engine.record_step.call_args.kwargs["action_taken"]
def test_record_action_step_stamps_the_normalized_coordinate_space(mock_context):
"""Flash stores the model's own 0–1000 target verbatim, marked as
normalized, so the agent-friendly view never re-normalizes it."""
from artemis.utils.coordinates import (
COORDINATE_SPACE_KEY,
COORDINATE_SPACE_NORMALIZED,
normalize_any_structure,
)
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="g")
recorded = _record(runner, "click", {"target": [320, 399]})
assert recorded["coordinates"] == [320, 399]
assert recorded["normalized_coordinates"] == [320, 399]
assert recorded[COORDINATE_SPACE_KEY] == COORDINATE_SPACE_NORMALIZED
assert normalize_any_structure(recorded, 1080, 2400)["coordinates"] == [320, 399]
def test_flash_records_render_in_the_ledger_with_their_arguments(mock_context):
"""The band-③ phrase reads the tool arguments Flash keeps under ``args``:
app launches name the package, key presses the key, inputs the text,
direction swipes the direction — never ``'None'``."""
from artemis.utils.task_tree import format_actions_clean
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="g")
launch = _record(runner, "manage_app", {"action": "launch", "app_name": "com.android.settings"})
assert format_actions_clean(launch) == "Launched app 'com.android.settings'"
stop = _record(runner, "manage_app", {"action": "stop", "app_name": "com.android.settings"})
assert format_actions_clean(stop) == "Stopped app 'com.android.settings'"
direct = _record(runner, "launch_app", {"app_name": "com.android.settings"})
assert format_actions_clean(direct) == "Launched app 'com.android.settings'"
key = _record(runner, "press_key", {"key": "BACK"})
assert format_actions_clean(key) == "Pressed key 'BACK'"
typed = _record(runner, "input_text", {"target": [500, 600], "text": "hello"})
assert format_actions_clean(typed) == "Inputted 'hello' into field at [500, 600]"
swipe_dir = _record(runner, "swipe", {"direction": "up"})
assert format_actions_clean(swipe_dir) == "Swiped up"
swipe_fa = _record(runner, "swipe", {"action": "down"})
assert format_actions_clean(swipe_fa) == "Swiped down"
swipe_coords = _record(runner, "swipe", {"coordinates": [556, 289, 556, 124]})
assert format_actions_clean(swipe_coords) == "Swiped from [556, 289] to [556, 124]"
+42 -15
View File
@@ -87,28 +87,55 @@ def test_video_analyzer_bound_only_when_recording_tools_enabled(mock_context):
assert "video_analyzer" not in names
def test_recall_history_bound_only_with_a_data_engine_session(mock_context):
HISTORY_TOOLS = {"search_history", "replay_steps", "get_step_screenshot"}
def test_history_tools_bound_only_with_a_data_engine_session(mock_context):
with patch("artemis.controllers.unified_controller.get_driver"):
# No DataEngine: nothing to search, the tool stays out (as in Pro).
# No DataEngine: nothing to read, the tools stay out (as in Pro).
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
assert "recall_history" not in names
assert not (HISTORY_TOOLS & set(names))
prompt = FlashRunner(mock_context, goal="g")._render_system_prompt(
FlashRunner(mock_context, goal="g")._get_tools()
)
assert "search_history" not in prompt and "replay_steps" not in prompt
mock_context.data_engine = Mock()
with patch("artemis.tools.history_recall._recall_config", return_value=None):
with patch("artemis.tools.history._recall_config", return_value=None):
runner = FlashRunner(mock_context, goal="g")
tools = runner._get_tools()
names = [t.name for t in tools]
assert "recall_history" in names
declaration = next(t for t in tools if t.name == "recall_history")
assert set(declaration.parameters["properties"]) == {
"query",
"step_range",
"include_details",
"include_images",
"max_results",
}
assert declaration.parameters["required"] == ["query"]
assert "`recall_history`" in runner._render_system_prompt(tools)
assert HISTORY_TOOLS <= set(names)
assert names[-1] == "report_task_status"
# Same declarations as the LangChain tools: derived from one args schema.
search = next(t for t in tools if t.name == "search_history")
assert set(search.parameters["properties"]) == {"query", "step_range", "max_results"}
assert search.parameters["required"] == []
replay = next(t for t in tools if t.name == "replay_steps")
assert set(replay.parameters["properties"]) == {"start_step", "end_step"}
assert replay.parameters["required"] == ["start_step"]
shot = next(t for t in tools if t.name == "get_step_screenshot")
assert shot.parameters["properties"]["which"]["enum"] == ["pre", "post", "overlay"]
prompt = runner._render_system_prompt(tools)
assert "`search_history`" in prompt
assert "`replay_steps`" in prompt
assert "`get_step_screenshot`" in prompt
def test_search_history_alone_follows_the_recall_config_gate(mock_context):
from types import SimpleNamespace
with patch("artemis.controllers.unified_controller.get_driver"):
mock_context.data_engine = Mock()
with patch(
"artemis.tools.history._recall_config",
return_value=SimpleNamespace(enabled=False),
):
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
assert "search_history" not in names
assert {"replay_steps", "get_step_screenshot"} <= set(names)
def test_video_analyzer_prompt_segment_follows_availability(mock_context):
@@ -95,7 +95,7 @@ async def test_hint_fires_on_old_matching_step():
assert len(parts) == 1
assert "Historical state hint" in parts[0]
assert "Step 2" in parts[0]
assert "recall_history" in parts[0]
assert "search_history" in parts[0]
@pytest.mark.asyncio
+19 -18
View File
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from pathlib import Path
from unittest.mock import MagicMock, patch
@@ -108,12 +107,15 @@ async def test_history_analyzer_detailed_query_with_tool_call():
},
]
mock_ctx.data_engine.get_agent_friendly_steps.return_value = steps
mock_ctx.data_engine.get_agent_friendly_steps_in_range.side_effect = lambda s, e: [
st for st in steps if s <= st["step_number"] <= e
]
# Turn 1 LLM response: Tool call to get_step_details for step 2
# Turn 1 LLM response: Tool call to replay_steps for step 2
mock_response_turn_1 = MagicMock()
mock_response_turn_1.content = ""
mock_tool_call = {
"name": "get_step_details",
"name": "replay_steps",
"args": {"start_step": 2, "end_step": 2},
"id": "call_123456",
}
@@ -160,16 +162,14 @@ async def test_history_analyzer_detailed_query_with_tool_call():
] # ToolMessage is second to last after final response is appended
assert isinstance(tool_msg, ToolMessage)
assert tool_msg.tool_call_id == "call_123456"
assert tool_msg.name == "replay_steps"
# Parse the JSON in the ToolMessage content
tool_result = json.loads(tool_msg.content)
assert len(tool_result) == 1
assert tool_result[0]["step_number"] == 2
assert tool_result[0]["operator_raw_thinking"] == "Toggle the switch to turn wifi on"
assert tool_result[0]["action_taken"] == {
"action": "tap",
"coordinates": [100, 200],
}
# The shared replay renderer: step header, screen description, thinking.
mock_ctx.data_engine.get_agent_friendly_steps_in_range.assert_called_once_with(2, 2)
assert "- **Step 2 (Start: 4.5s)**" in tool_msg.content
assert "[Screen]: Toggled the wifi switch" in tool_msg.content
assert "Toggle the switch to turn wifi on" in tool_msg.content
assert "**Step 1" not in tool_msg.content
@pytest.mark.asyncio
@@ -282,13 +282,14 @@ def test_history_analyzer_robust_tools_behavior():
]
analyzer = HistoryAnalyzer(mock_ctx)
mock_ctx.data_engine.get_agent_friendly_steps_in_range.return_value = steps
# 1. Test get_step_details with string inputs
details_tool = analyzer._get_step_details_tool(steps)
result_details = details_tool.invoke({"start_step": "1", "end_step": "1"})
parsed_result = json.loads(result_details)
assert len(parsed_result) == 1
assert parsed_result[0]["step_number"] == 1
# 1. The shared replay tool coerces string step numbers
replay_tool = next(t for t in analyzer._build_tools() if t.name == "replay_steps")
result_details = replay_tool.invoke({"start_step": "1", "end_step": "1"})
mock_ctx.data_engine.get_agent_friendly_steps_in_range.assert_called_once_with(1, 1)
assert "- **Step 1 (Start: 1.0s)**" in result_details
assert "[Screen]: Action 1" in result_details
# 2. Test read_note with .md suffix
from artemis.tools.scratchpad import get_read_note_tool_pure
+6
View File
@@ -113,6 +113,7 @@ async def test_perform_action_validation():
{
"action": "tap",
"coordinates": [200, 300],
"coordinate_space": "pixel",
"normalized_coordinates": [185, 125],
"times": 1,
"delay_ms": 100,
@@ -143,6 +144,7 @@ async def test_perform_action_validation():
assert actions[0] == {
"action": "focus_and_input_text",
"coordinates": [400, 500],
"coordinate_space": "pixel",
"normalized_coordinates": [370, 208],
"text": "hello",
"clear_before_input": True,
@@ -171,6 +173,7 @@ async def test_perform_action_validation():
{
"action": "tap",
"coordinates": [400, 500],
"coordinate_space": "pixel",
"normalized_coordinates": [370, 208],
"times": 1,
"delay_ms": 100,
@@ -263,6 +266,7 @@ async def test_bare_coordinate_click_enriched_by_hit_test():
{
"action": "tap",
"coordinates": [540, 1440],
"coordinate_space": "pixel",
"normalized_coordinates": [500, 600],
"times": 1,
"delay_ms": 100,
@@ -784,6 +788,7 @@ async def test_long_press_action_translation():
{
"action": "long_press_on",
"coordinates": [200, 300],
"coordinate_space": "pixel",
"normalized_coordinates": [185, 125],
"duration": 1500,
"target_text": "Item 1",
@@ -806,6 +811,7 @@ async def test_long_press_action_translation():
{
"action": "long_press_on",
"coordinates": [540, 1440],
"coordinate_space": "pixel",
"normalized_coordinates": [500, 600],
"duration": 1000,
"target_text": None,
+6 -2
View File
@@ -32,8 +32,12 @@ def test_operator_prompt_contract_matches_note_runtime_semantics():
)
assert "--- Execution Incident (OPEN) ---" in prompt
assert "Failure Analyzer" not in prompt
assert "(`read_note`, `list_notes`, `recall_history`)" in prompt
assert "`save_note`, and `recall_history`" not in prompt
assert (
"(`read_note`, `list_notes`, `search_history`, `replay_steps`, `get_step_screenshot`)"
in prompt
)
assert "recall_history" not in prompt
assert "`save_note`, and `search_history`" not in prompt
assert "Do NOT submit a Turn-Ending Action at the same time" not in prompt
assert "memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`" not in prompt
+8 -23
View File
@@ -12,11 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Operator dual prompt path (M2): legacy golden parity and transcript mode.
"""Operator prompt snapshots and transcript mode.
Flag off (``agent.memory.transcript.enabled=false``): the legacy 2-message
build must stay byte-for-byte identical — pinned by SHA-256 goldens recorded
against the pre-split implementation.
for the current template.
Flag on: ``_build_prompt`` renders ``S + F + A + tail`` from the session
transcript ledger; the turn's messages are staged after the tool loop and
@@ -51,20 +51,10 @@ from artemis.memory.transcript import (
TranscriptLedger,
)
# SHA-256 of the legacy system message rendered by TemplatePromptComponent
# with the fixed inputs below. Originally recorded BEFORE the M2 template
# split; re-recorded in M5 (2026-09-01) after the Short-Term Memory block was
# removed, and again on 2026-09-02 when the Failure Analyzer was retired: the
# operator.json template gained the Time-Sensitive Tasks principle, the two
# execution tiers (vetted single action / fast-action burst), the Execution
# Incident workflow section and the Phase 3 burst-discipline rule; and again
# on 2026-09-02 after the prompt consolidation (every rule stated once, the
# transcript/legacy history-marker split, the recited tool budget, the
# recommended diagnosis triggers); and again on 2026-09-03 for the nested
# plan-ledger contract and its validation gate. Any other drift means the
# flag-off path is no longer byte-identical.
GOLDEN_EMPTY_PLAN = "2ed35be503e3872f008ae5a12616af6f90cde65f23b63adbb6da782079ae4e7c"
GOLDEN_SENTINEL_PLAN = "8bd6aa60cf3fad8d21cf5c284a01f586c57d209fcd1d2b4a517a791736f9b1de"
# SHA-256 snapshots of the legacy system message with the fixed inputs below.
# Update these when an intentional template change alters the rendered prompt.
GOLDEN_EMPTY_PLAN = "dfcb15a164fa32172cf4bd2ab7b39409229a05dd1d5995eae26992784cfaa4f6"
GOLDEN_SENTINEL_PLAN = "4b49a3c1ace6387df05a6f1f163dda007e4c2f8ff3d544dac12520317f17d21b"
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
@@ -166,13 +156,8 @@ def _no_action_llm(captured: list):
@pytest.mark.asyncio
async def test_transcript_mode_two_turns_build_four_regions(tmp_path):
ctx = _transcript_ctx()
# Regression data shape (M2 cold-start bug, on-device A/B 2026-09-01):
# turn 1 runs before any step record exists; by turn 2 the first step HAS
# been recorded, so `steps` is non-empty while the previous turn is still
# staged (turn_count == 0). The cold-start check must not read that
# live-session state as a process restart — seeding would be rejected by
# the empty-ledger invariant and silently demote every turn to the legacy
# 2-message path.
# Turn 2 has a recorded step but no committed ledger turn yet. This is
# an active session, so it must not trigger cold-start history seeding.
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
ctx.data_engine.global_base_dir = str(tmp_path)
@@ -1,102 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import MagicMock, mock_open, patch
from artemis.agents.outputter.tools import (
get_search_history_tool,
get_step_details_tool,
get_step_screenshot_tool,
)
from artemis.context import ArtemisContext
import pytest
@pytest.fixture
def mock_context():
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.global_base_dir = "/tmp/fake_traces"
return ctx
def test_get_step_details_tool():
steps = [
{
"step_number": 1,
"relative_time": "1s",
"summary": "Step 1",
"action_taken": "tap",
},
{
"step_number": 2,
"relative_time": "2s",
"summary": "Step 2",
"action_taken": "swipe",
},
]
tool = get_step_details_tool(steps)
# Test valid range
result = tool.invoke({"start_step": 1, "end_step": 1})
parsed = json.loads(result)
assert len(parsed) == 1
assert parsed[0]["step_number"] == 1
assert parsed[0]["action_taken"] == "tap"
# Test invalid range
result_invalid = tool.invoke({"start_step": 3, "end_step": 4})
assert "No steps found" in result_invalid
def test_get_step_screenshot_tool(mock_context):
steps = [
{"step_number": 1, "pre_image_name": "img1", "post_image_name": "img2"},
]
tool = get_step_screenshot_tool(mock_context, steps)
# Mock open and exists
with (
patch("pathlib.Path.exists", return_value=True),
patch("builtins.open", mock_open(read_data=b"fake_image_bytes")),
):
result = tool.invoke({"step_number": 1})
assert isinstance(result, list)
assert result[0]["type"] == "text"
assert result[1]["type"] == "image_url"
assert "data:image/jpeg;base64" in result[1]["image_url"]["url"]
def test_search_history_tool(mock_context):
tool = get_search_history_tool(mock_context)
# Mock steps returned by storage
mock_step_1 = MagicMock()
mock_step_1.step_number = 1
mock_step_1.pre_image_name = "img1"
mock_step_1.post_image_name = None
mock_context.data_engine.storage.get_steps.return_value = [mock_step_1]
# Mock search_ui_by_hash
mock_context.data_engine.storage.search_ui_by_hash.return_value = [
{"type": "ocr", "matched_text": "verification code: 1234", "score": 0.9}
]
result = tool.invoke({"query": "verification"})
parsed = json.loads(result)
assert len(parsed) == 1
assert parsed[0]["step_number"] == 1
assert parsed[0]["matches"][0]["matched_text"] == "verification code: 1234"
@@ -98,3 +98,6 @@ def test_prompt_rules_are_keyed_on_target_kind():
assert "`Kind: coordinates only`" in prompt
assert "hidden" in prompt
assert "Identify what appears under the red dot" in prompt
# Rule 1 judges the dot, not the screen: a control surviving elsewhere (or as
# an active input field) does not make the tapped spot valid.
assert "Judge the red dot, not the screen" in prompt
+9 -2
View File
@@ -85,7 +85,11 @@ def test_full_toolset_is_the_default():
)
@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer", "recall_history"))
@pytest.mark.parametrize(
"tool",
OPERATOR_DEVICE_TOOLS
+ ("video_analyzer", "search_history", "replay_steps", "get_step_screenshot"),
)
def test_removed_tool_leaves_no_reference(tool):
"""The executable definition of 'an absent tool costs the model nothing'."""
for name, template in load_operator_prompts().items():
@@ -148,7 +152,10 @@ def test_full_set_enum_slots_render_verbatim():
)
assert "Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`)" in out
assert "ADB/task tools (`run_adb_command`, `manage_task`)" in out
assert "(`read_note`, `list_notes`, `recall_history`)" in out
assert (
"(`read_note`, `list_notes`, `search_history`, `replay_steps`, `get_step_screenshot`)"
in out
)
# save_note is a write-through tool, never a result-dependent one; the on-demand
# analyze_task_output is not advertised.
assert "`save_note`, and" not in out
+20 -6
View File
@@ -60,7 +60,14 @@ def mock_context(tmp_path):
engine = Mock()
engine.get_step_number.return_value = 7
engine.get_step_record.return_value = SimpleNamespace(
action_taken=[{"action": "tap", "coordinates": [500, 600], "target_text": "Search"}]
action_taken=[
{
"action": "tap",
"coordinates": [540, 1440],
"coordinate_space": "pixel",
"target_text": "Search",
}
]
)
engine.get_step_image_path.side_effect = lambda step_number, which="pre": (
pre_path if which == "pre" else post_path
@@ -84,7 +91,11 @@ async def test_summarizer_dispatches_visual_lens(mock_context):
mock_context.step_memory.dispatch.assert_called_once_with(
step_number=7,
action_name="tap",
action_args={"coordinates": [500, 600], "target_text": "Search"},
action_args={
"coordinates": [500, 600],
"coordinate_space": "normalized",
"target_text": "Search",
},
pre_img_bytes=b"PRE_IMAGE",
post_img_bytes=b"POST_IMAGE",
exec_outcome="success",
@@ -123,8 +134,9 @@ async def test_summarizer_falls_back_to_structured_decisions(mock_context):
state = DummyState(
structured_decisions=(
'[{"action": "swipe", "coordinates": [1, 2, 3, 4]},'
' {"action": "tap", "coordinates": [9, 9]}]'
'[{"action": "swipe", "coordinates": [540, 1800, 540, 600],'
' "coordinate_space": "pixel"},'
' {"action": "tap", "coordinates": [540, 1200], "coordinate_space": "pixel"}]'
),
last_execution_result={"status": "success"},
current_step_id="12345678-1234-5678-1234-567812345678",
@@ -135,8 +147,10 @@ async def test_summarizer_falls_back_to_structured_decisions(mock_context):
kwargs = mock_context.step_memory.dispatch.call_args.kwargs
assert kwargs["action_name"] == "swipe"
assert kwargs["action_args"]["coordinates"] == [1, 2, 3, 4]
assert kwargs["action_args"]["additional_actions"] == [{"action": "tap", "coordinates": [9, 9]}]
assert kwargs["action_args"]["coordinates"] == [500, 750, 500, 250]
assert kwargs["action_args"]["additional_actions"] == [
{"action": "tap", "coordinates": [500, 500], "coordinate_space": "normalized"}
]
@pytest.mark.asyncio
+196 -5
View File
@@ -28,6 +28,13 @@ from langchain_core.messages import AIMessage
import pytest
from artemis.agents.validator.categories import ValidationErrorCategory
from artemis.agents.validator.execution_loop import BURST_SKIPPED_MARKER
from artemis.agents.validator.precondition_xml import (
XML_STRONG_MIN_ELEMENTS,
XML_VERDICT_STRONG,
XML_VERDICT_WEAK,
classify_xml_verdict,
)
from artemis.agents.validator.validator import ValidatorNode
from artemis.context import ArtemisContext
from artemis.mcp.action_types import ActionCode, ActionResult
@@ -107,6 +114,28 @@ def temp_screenshot(tmp_path):
return str(p)
def _unrelated_hierarchy(count: int) -> list[dict]:
"""``count`` labelled rows far below the target region (nothing matches it)."""
return [
{
"text": f"Item {i}",
"resource-id": f"row_{i}",
"bounds": f"[0,{1000 + i * 100}][1080,{1080 + i * 100}]",
"clickable": "true",
}
for i in range(count)
]
_LOGIN_TAP = {
"action": "tap",
"coordinates": [200, 250],
"target_text": "Login",
"target_bounds": [100, 200, 300, 300],
"target_resource_id": "btn_login",
}
@pytest.fixture
def mock_mcp():
fake = FakeActionSession()
@@ -175,17 +204,20 @@ async def test_validator_exec_error_opens_incident(mock_mcp, mock_context, temp_
assert incident["reason"] == "Error: Element not found"
assert incident["consecutive_failures"] == 1
assert incident["burst_size"] == 1
assert incident["action_description"] == "Tapped element at [105, 205]"
# The Operator reads the incident: the pixel target [105, 205] is rendered
# in its own 0-1000 space (1080x2400 frame).
assert incident["action_description"] == "Tapped element at [97, 85]"
# The same record rides in graph state for the Operator prompt.
assert result["open_incident"] == incident
@pytest.mark.asyncio
async def test_validator_burst_skips_safety_net_and_aborts_on_first_failure(
async def test_validator_burst_vets_first_member_only_and_aborts_on_first_failure(
mock_mcp, mock_context, temp_screenshot
):
"""Two or more actions form a fast-action burst: no precondition gate, a
single dispatch per member, and the first failure aborts the rest."""
"""Two or more actions form a fast-action burst: the precondition gate runs
for the first member only, every member is dispatched once, and the first
failure aborts the rest."""
calls = []
def handler(name, args):
@@ -213,9 +245,12 @@ async def test_validator_burst_skips_safety_net_and_aborts_on_first_failure(
) as mock_gate,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_gate.return_value = (True, ValidationErrorCategory.NONE, "")
result = await node(state)
mock_gate.assert_not_called()
# Only the first member (decided on the still-live screen) is vetted.
mock_gate.assert_called_once()
assert mock_gate.call_args.args[2]["coordinates"] == [105, 205]
report = result["last_execution_result"]
assert report["burst"] is True
@@ -237,6 +272,42 @@ async def test_validator_burst_skips_safety_net_and_aborts_on_first_failure(
assert result["open_incident"] == incident
@pytest.mark.asyncio
async def test_validator_burst_first_member_blocked_aborts_whole_burst(
mock_mcp, mock_context, temp_screenshot
):
"""A refused first member opens a safety-net incident and skips the rest
of the burst; nothing is dispatched to the device."""
mock_mcp.hierarchy = _unrelated_hierarchy(XML_STRONG_MIN_ELEMENTS + 2)
decisions = json.dumps([dict(_LOGIN_TAP), {"action": "press_key", "keycode": "ENTER"}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
patch.object(
ValidatorNode, "_validate_action_precondition_pixel", new_callable=AsyncMock
) as mock_pixel,
):
result = await node(state)
report = result["last_execution_result"]
assert report["burst"] is True
assert report["status"] == "failed"
assert report["execution"][0]["attempts"][0].startswith("Pre-execution validation failed")
assert report["execution"][1]["attempts"] == [BURST_SKIPPED_MARKER]
assert mock_mcp.calls_for("click") == []
assert mock_mcp.calls_for("press_key") == []
mock_pixel.assert_not_called()
incident = report["incident"]
assert incident["kind"] == "safety_net"
assert incident["category"] == ValidationErrorCategory.TARGET_DISAPPEARED.value
assert incident["action_index"] == 0
assert incident["burst_size"] == 2
assert result["open_incident"] == incident
@pytest.mark.asyncio
async def test_validator_burst_success_executes_every_member(
mock_mcp, mock_context, temp_screenshot
@@ -572,6 +643,126 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
)
def test_classify_xml_verdict_grades_strength():
strong, why = classify_xml_verdict(XML_STRONG_MIN_ELEMENTS, "Login", None)
assert (strong, why) == (XML_VERDICT_STRONG, "")
assert classify_xml_verdict(XML_STRONG_MIN_ELEMENTS, None, "btn_login")[0] == XML_VERDICT_STRONG
weak, why = classify_xml_verdict(XML_STRONG_MIN_ELEMENTS - 1, "Login", "btn_login")
assert weak == XML_VERDICT_WEAK and "sparse hierarchy" in why
weak, why = classify_xml_verdict(XML_STRONG_MIN_ELEMENTS, None, None)
assert weak == XML_VERDICT_WEAK and "no resource-id/text" in why
@pytest.mark.asyncio
async def test_validator_strong_xml_missing_verdict_is_final_without_pixel_judge(
mock_mcp, mock_context, temp_screenshot
):
"""A strong XML verdict blocks the action without consulting the pixel judge."""
mock_mcp.hierarchy = _unrelated_hierarchy(XML_STRONG_MIN_ELEMENTS + 2)
state = DummyState(
structured_decisions=json.dumps([dict(_LOGIN_TAP)]), latest_screenshot=temp_screenshot
)
node = ValidatorNode(mock_context)
with (
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode, "_validate_action_precondition_pixel", new_callable=AsyncMock
) as mock_pixel,
):
# A positive pixel verdict must not override strong XML evidence.
mock_pixel.return_value = (True, ValidationErrorCategory.NONE, "")
result = await node(state)
mock_pixel.assert_not_called()
assert mock_mcp.calls_for("click") == []
incident = result["last_execution_result"]["incident"]
assert incident["kind"] == "safety_net"
assert incident["category"] == ValidationErrorCategory.TARGET_DISAPPEARED.value
assert "was not found on the screen" in incident["reason"]
assert "pixel judge not consulted" in incident["reason"]
assert incident["evidence"]["xml_verdict"] == XML_VERDICT_STRONG
assert incident["evidence"]["xml_element_count"] == XML_STRONG_MIN_ELEMENTS + 2
assert incident["evidence"]["pixel_judge"] == "not consulted (strong XML verdict)"
# The clause also rides in the action record the traces keep.
assert (
"pixel judge not consulted"
in result["last_execution_result"]["execution"][0]["attempts"][0]
)
@pytest.mark.asyncio
async def test_validator_weak_xml_sparse_tree_lets_pixel_judge_allow(
mock_mcp, mock_context, temp_screenshot
):
"""A sparse hierarchy (WebView/game-like) cannot vouch for a missing target;
the pixel judge decides, and a confident "present" lets the action run."""
mock_mcp.hierarchy = _unrelated_hierarchy(2)
state = DummyState(
structured_decisions=json.dumps([dict(_LOGIN_TAP)]), latest_screenshot=temp_screenshot
)
node = ValidatorNode(mock_context)
with (
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode, "_validate_action_precondition_pixel", new_callable=AsyncMock
) as mock_pixel,
):
mock_pixel.return_value = (True, ValidationErrorCategory.NONE, "")
result = await node(state)
mock_pixel.assert_called_once()
report = result["last_execution_result"]
assert report["status"] == "success"
assert report["incident"] is None
assert len(mock_mcp.calls_for("click")) == 1
assert "safety_net_evidence" not in report["execution"][0]
@pytest.mark.asyncio
async def test_validator_weak_xml_identifierless_target_pixel_judge_blocks(
mock_mcp, mock_context, temp_screenshot
):
"""Bare bounds from a hit test carry no identifiers to search for, so the
XML "missing" is weak; the pixel judge is consulted and its refusal is
recorded next to the XML verdict."""
mock_mcp.hierarchy = _unrelated_hierarchy(XML_STRONG_MIN_ELEMENTS + 2)
item = {
"action": "tap",
"coordinates": [200, 250],
"target_class": "android.widget.ImageView",
"target_bounds": [100, 200, 300, 300],
"target_label_source": "hit_test",
}
state = DummyState(structured_decisions=json.dumps([item]), latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode, "_validate_action_precondition_pixel", new_callable=AsyncMock
) as mock_pixel,
):
mock_pixel.return_value = (
False,
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"Pixel check failed",
)
result = await node(state)
mock_pixel.assert_called_once()
incident = result["last_execution_result"]["incident"]
assert incident["category"] == ValidationErrorCategory.TARGET_DISAPPEARED.value
assert "XML evidence weak (target had no resource-id/text" in incident["reason"]
assert "pixel judge did not confirm the target" in incident["reason"]
assert incident["evidence"]["xml_verdict"] == XML_VERDICT_WEAK
assert incident["evidence"]["pixel_judge"].startswith("consulted (weak XML verdict)")
@pytest.mark.asyncio
async def test_validator_pixel_validation_success(mock_mcp, mock_context, temp_screenshot):
"""Test that the pixel safety net succeeds when Gemini reports target is present."""
+191
View File
@@ -438,3 +438,194 @@ def test_record_step_without_app_data_stamps_nothing(tmp_path):
step = engine.storage.get_steps(engine.current_session_id)[0]
assert "foreground_app" not in (step.extra_metadata or {})
# --- Agent-friendly single-step / range lookups (step replay backing) ----------------
def _replay_engine(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
engine = DataEngine(mock_ctx)
engine.start_session("replay lookup session")
return engine
def _join_pending(engine):
for t in list(engine._pending_threads):
t.join()
def test_friendly_step_range_expands_tool_traces_and_matches_full_listing(tmp_path):
engine = _replay_engine(tmp_path)
engine.record_step(
summary="Asked the explorer about the toggle.",
action_taken={"action": "click", "target": [540, 1200], "target_text": "Save"},
last_execution_result={"status": "success"},
)
engine.record_trace(
type="tool",
name="ask_explorer",
payload={"args": {"question": "toggle?"}, "result": "The toggle is ON"},
step_id=engine.last_recorded_step_id,
)
engine.record_step(summary="Second step.", action_taken={"action": "swipe"})
engine.record_step(summary="Third step.", action_taken={"action": "press_key", "key": "BACK"})
_join_pending(engine)
rows = engine.get_agent_friendly_steps_in_range(1, 2)
assert [r["step_number"] for r in rows] == [1, 2]
calls = rows[0]["tool_calls"]
assert calls and calls[0]["name"] == "ask_explorer"
assert calls[0]["args"] == {"question": "toggle?"}
assert calls[0]["result"] == "The toggle is ON"
assert any(e["type"] == "tool_call" for e in rows[0]["interleaved_events"])
# The single-step path renders exactly what the whole-session listing does
# (same trace expansion, same coordinate normalization).
full = engine.get_agent_friendly_steps()
assert rows[0]["action_taken"] == full[0]["action_taken"]
assert rows[0]["interleaved_events"] == full[0]["interleaved_events"]
# Reversed bounds are tolerated; a single step defaults end to start.
assert [r["step_number"] for r in engine.get_agent_friendly_steps_in_range(3, 2)] == [2, 3]
assert [r["step_number"] for r in engine.get_agent_friendly_steps_in_range(3)] == [3]
one = engine.get_agent_friendly_step(3)
assert one is not None and one["summary"] == "Third step."
assert engine.get_agent_friendly_step(99) is None
def test_friendly_step_lookups_without_session_are_empty(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
engine = DataEngine(mock_ctx)
assert engine.get_agent_friendly_steps_in_range(1, 5) == []
assert engine.get_agent_friendly_step(1) is None
def test_tool_result_images_are_described_as_step_screenshots(tmp_path):
"""An image a tool result embedded (stored as an <ImageRef: sha256=…>
placeholder) is rewritten into the description of the step screenshot it
is, never dropped silently; unknown images get the generic label."""
import hashlib
import io
from PIL import Image
from artemis.data_engine.engine import GENERIC_IMAGE_LABEL
buf = io.BytesIO()
Image.new("RGB", (32, 64), "red").save(buf, format="JPEG")
pre_bytes = buf.getvalue()
pre_hash = hashlib.sha256(pre_bytes).hexdigest()
engine = _replay_engine(tmp_path)
engine.record_step(
pre_screenshot_bytes=pre_bytes,
summary="Opened the alarms list.",
action_taken={"action": "click"},
ocr_result=[{"text": "Alarm 07:30 needleocr", "bounds": [0, 0, 10, 10]}],
)
engine.record_step(summary="Recalled the earlier screen.", action_taken={"action": "click"})
engine.record_trace(
type="tool",
name="get_step_screenshot",
payload={
"args": {"step_number": 1, "which": "pre"},
"result": [
{"type": "text", "text": "Screenshot of step 1 (pre-action) is attached."},
{
"type": "image_url",
"image_url": {"url": f"<ImageRef: sha256={pre_hash} length=99999>"},
},
{
"type": "image_url",
"image_url": {"url": "<ImageRef: sha256=" + "0" * 64 + " length=5>"},
},
],
},
step_id=engine.last_recorded_step_id,
)
_join_pending(engine)
step2 = engine.get_agent_friendly_step(2)
result = step2["tool_calls"][0]["result"]
assert all(block["type"] == "text" for block in result)
assert result[1]["text"] == "[screenshot: pre-action of Step 1]"
assert result[1]["image_name"] == pre_hash
assert result[2]["text"] == GENERIC_IMAGE_LABEL
# Whole-session listing describes the same way.
assert engine.get_agent_friendly_steps()[1]["tool_calls"][0]["result"] == result
# --- Coordinate space of agent-friendly steps -------------------------
def test_flash_normalized_record_survives_agent_friendly_steps_unchanged(tmp_path):
"""Flash records the model's 0–1000 target with an explicit space marker;
the agent-friendly view must not normalize it a second time."""
from artemis.utils.coordinates import COORDINATE_SPACE_KEY, COORDINATE_SPACE_NORMALIZED
engine = _replay_engine(tmp_path)
engine.record_step(
action_taken={
"action": "click",
"coordinates": [320, 399],
"normalized_coordinates": [320, 399],
COORDINATE_SPACE_KEY: COORDINATE_SPACE_NORMALIZED,
"args": {"target": [320, 399]},
},
last_execution_result={"status": "success", "result": "click executed"},
)
_join_pending(engine)
row = engine.get_agent_friendly_steps()[0]
assert row["action_taken"]["coordinates"] == [320, 399]
assert row["action_taken"][COORDINATE_SPACE_KEY] == COORDINATE_SPACE_NORMALIZED
assert engine.get_agent_friendly_step(1)["action_taken"]["coordinates"] == [320, 399]
def test_pro_pixel_record_is_normalized_exactly_once(tmp_path):
"""Pro records physical pixels (with the frame size in extra_metadata);
the friendly view converts them once, and the result is stamped so a
downstream pass (MCP inspector, replay) leaves it alone."""
from artemis.utils.coordinates import (
COORDINATE_SPACE_KEY,
COORDINATE_SPACE_NORMALIZED,
COORDINATE_SPACE_PIXEL,
normalize_any_structure,
)
engine = _replay_engine(tmp_path)
action = {
"action": "tap",
"coordinates": [540, 1200],
"normalized_coordinates": [500, 500],
COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
"target_text": "Battery",
}
engine.record_step(
action_taken=[action],
last_execution_result={
"status": "success",
"execution": [dict(action, attempts=["Success"])],
},
extra_metadata={"width": 1080, "height": 2400},
)
_join_pending(engine)
row = engine.get_agent_friendly_steps()[0]
friendly = row["action_taken"][0]
assert friendly["coordinates"] == [500, 500]
assert friendly[COORDINATE_SPACE_KEY] == COORDINATE_SPACE_NORMALIZED
assert row["last_execution_result"]["execution"][0]["coordinates"] == [500, 500]
again = normalize_any_structure(row["action_taken"], 1080, 2400)
assert again == row["action_taken"]
@@ -0,0 +1,139 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""OfflineHistoryReader: the MCP inspector's read-only twin of the live engine.
The offline reader must produce exactly the records the live ``DataEngine``
produces (same ``friendly_step``), open the database read-only, and satisfy
the ``HistoryReader`` protocol the history tools are written against.
"""
import io
import sqlite3
from unittest.mock import MagicMock
import pytest
from PIL import Image
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
from artemis.data_engine.history_reader import HistoryReader, OfflineHistoryReader
def _jpeg(color: str) -> bytes:
buf = io.BytesIO()
Image.new("RGB", (64, 128), color).save(buf, format="JPEG")
return buf.getvalue()
def _engine(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.execution_setup = MagicMock()
ctx.execution_setup.traces_path = str(tmp_path)
ctx.device = None
engine = DataEngine(ctx)
engine.start_session("offline reader test")
return engine
def _flush(engine):
for t in list(engine._pending_threads):
t.join()
@pytest.fixture
def recorded(tmp_path):
engine = _engine(tmp_path)
engine.record_step(
pre_screenshot_bytes=_jpeg("red"),
post_screenshot_bytes=_jpeg("blue"),
summary="Tapped Save after asking the explorer.",
action_taken={"action": "click", "target": [540, 1200], "target_text": "Save"},
last_execution_result={"status": "success"},
operator_raw_thinking="Confirm the toggle first.",
extra_metadata={"width": 1080, "height": 2400},
)
engine.record_trace(
type="tool",
name="ask_explorer",
payload={"args": {"question": "toggle?"}, "result": "The toggle is ON."},
step_id=engine.last_recorded_step_id,
)
engine.record_step(
summary="Went back.",
action_taken={"action": "press_key", "key": "back"},
last_execution_result={"status": "success"},
)
engine.record_history_chunk(
start_step_number=1,
end_step_number=2,
version=1,
status="ready",
band2=" - Steps 1–2: saved the toggle",
band3="- Step 1 (T+00:00): click -> executed",
rendered_text="[Chunk 1 | Steps 1–2]",
)
_flush(engine)
return engine
def _reader(engine) -> OfflineHistoryReader:
return OfflineHistoryReader(engine.db_path, engine.global_base_dir, engine.current_session_id)
def test_offline_reader_matches_live_friendly_steps(recorded):
reader = _reader(recorded)
assert isinstance(reader, HistoryReader)
assert reader.session_start_time == recorded.session_start_time
assert reader.base_dir == recorded.base_dir
live = recorded.get_agent_friendly_steps()
offline = reader.get_agent_friendly_steps()
assert offline == live
assert offline[0]["tool_calls"][0]["name"] == "ask_explorer"
# Coordinates normalized exactly like the live engine.
assert offline[0]["action_taken"]["target"] == live[0]["action_taken"]["target"]
assert offline[0]["relative_time"] == live[0]["relative_time"]
assert reader.get_agent_friendly_steps_in_range(2, 1) == live
assert reader.get_agent_friendly_step(2) == live[1]
assert reader.get_agent_friendly_step(9) is None
def test_offline_reader_step_records_images_and_chunks(recorded):
reader = _reader(recorded)
record = reader.get_step_record(1)
assert record is not None and record.step_number == 1
assert reader.get_step_record(7) is None
pre = reader.get_step_image_path(1, "pre")
post = reader.get_step_image_path(1, "post")
assert pre is not None and pre.exists()
assert post is not None and post.exists()
assert pre == recorded.get_image_path(record.pre_image_name)
assert reader.get_step_image_path(2, "pre") is None
chunks = reader.get_history_chunks()
assert len(chunks) == 1
assert chunks[0].start_step_number == 1
def test_offline_reader_is_read_only(recorded):
reader = _reader(recorded)
with reader.storage._get_connection() as conn:
with pytest.raises(sqlite3.OperationalError):
conn.execute("DELETE FROM steps")
# No directories or tables are created for a missing database.
with pytest.raises(FileNotFoundError):
OfflineHistoryReader(recorded.global_base_dir / "nope.db", recorded.global_base_dir, "x")
+92
View File
@@ -702,3 +702,95 @@ async def test_exception_fail_open_releases_but_books_inconclusive(tmp_path):
for r in read_ledger(tmp_path):
assert r["status"] == "inconclusive"
assert "checker blew up" in r["evidence"]
# --- Attempt stream transcripts: bounded, append-only, replayable ------------------------
def _seg(exec_id, text, role="answer", when=1.0):
return {"execution_id": exec_id, "role": role, "when": when, "text": text}
def test_clamp_keeps_short_transcripts_verbatim():
from artemis.graph.checkpoints import clamp_stream_segments
segs = [_seg("a", "hello", "thought"), _seg("b", "world")]
out, dropped = clamp_stream_segments(segs, limit=20)
assert out == segs and dropped == 0
assert out[0] is not segs[0] # copies, the input is never mutated
def test_clamp_cuts_the_middle_once_and_keeps_head_and_tail():
from artemis.graph.checkpoints import clamp_stream_segments
segs = [
_seg("a", "A" * 40, "thought", when=1.0),
_seg("b", "B" * 40, when=2.0),
_seg("c", "C" * 40, when=3.0),
_seg("d", "D" * 40, when=4.0),
]
out, dropped = clamp_stream_segments(segs, limit=100)
assert dropped == 60
# head: 50 chars = all of a + 10 of b; tail: 50 chars = 10 of c + all of d
assert [s["execution_id"] for s in out] == ["a", "b", "c", "d"]
assert out[0]["text"] == "A" * 40 and out[0]["role"] == "thought"
assert out[1]["text"] == "B" * 10 + "\n…[60 chars truncated]…\n"
assert out[2]["text"] == "C" * 10
assert out[3]["text"] == "D" * 40
joined = "".join(s["text"] for s in out)
assert joined.count("chars truncated") == 1
assert len(joined) == 100 + len("\n…[60 chars truncated]…\n")
# Timestamps and the other fields ride along untouched.
assert [s["when"] for s in out] == [1.0, 2.0, 3.0, 4.0]
def test_clamp_drops_segments_that_fall_entirely_into_the_cut():
from artemis.graph.checkpoints import clamp_stream_segments
segs = [_seg("a", "A" * 10), _seg("mid", "M" * 100), _seg("z", "Z" * 10)]
out, dropped = clamp_stream_segments(segs, limit=20)
assert dropped == 100
assert [s["execution_id"] for s in out] == ["a", "z"]
assert out[0]["text"] == "A" * 10
assert out[1]["text"] == "\n…[100 chars truncated]…\n" + "Z" * 10
def test_clamp_single_huge_segment_keeps_both_ends():
from artemis.graph.checkpoints import clamp_stream_segments
out, dropped = clamp_stream_segments([_seg("a", "H" * 30 + "M" * 100 + "T" * 30)], limit=60)
assert dropped == 100
assert len(out) == 1
assert out[0]["text"] == "H" * 30 + "\n…[100 chars truncated]…\n" + "T" * 30
def test_stream_records_round_trip_and_carry_truncation_flags(tmp_path):
from artemis.graph.checkpoints import (
STREAM_TEXT_LIMIT,
append_attempt_stream_record,
read_attempt_streams,
streams_path,
)
assert STREAM_TEXT_LIMIT == 20_000
append_attempt_stream_record(
tmp_path,
{"attempt_id": "abc#1", "trace_id": "t1", "segments": [_seg("a", "short")]},
)
append_attempt_stream_record(
tmp_path,
{"attempt_id": "abc#2", "trace_id": "t2", "segments": [_seg("b", "x" * 50)]},
limit=10,
)
with streams_path(tmp_path).open("a", encoding="utf-8") as f:
f.write("not json\n\n")
records = read_attempt_streams(tmp_path)
assert [r["attempt_id"] for r in records] == ["abc#1", "abc#2"]
assert records[0]["truncated"] is False and records[0]["dropped_chars"] == 0
assert records[0]["segments"] == [_seg("a", "short")]
assert isinstance(records[0]["ts"], float)
assert records[1]["truncated"] is True and records[1]["dropped_chars"] == 40
assert records[1]["segments"][0]["text"] == "x" * 5 + "\n…[40 chars truncated]…\n" + "x" * 5
assert read_attempt_streams(tmp_path / "nowhere") == []
@@ -1,7 +1,7 @@
"""Helper-tool routing in McpActionExecutor (Flash profile).
The Flash runner binds ``ask_explorer``, ``video_analyzer`` and
``recall_history`` declarations next to its device actions; the executor must
The Flash runner binds ``ask_explorer``, ``video_analyzer`` and the history
tools (``search_history`` / ``replay_steps`` / ``get_step_screenshot``) next to its device actions; the executor must
route them to the same implementations the Pro operator's LangChain tools
delegate to, as agent-side text tools (no screenshot capture, no device
action), and report their status explicitly instead of sniffing the text for
@@ -38,7 +38,13 @@ def _state():
def test_helper_tools_are_not_device_actions():
executor = _make_executor()
assert {"ask_explorer", "video_analyzer", "recall_history"} <= AGENT_TOOL_NAMES
assert {
"ask_explorer",
"video_analyzer",
"search_history",
"replay_steps",
"get_step_screenshot",
} <= AGENT_TOOL_NAMES
assert not (AGENT_TOOL_NAMES & executor.action_tool_names)
@@ -231,22 +237,32 @@ async def test_video_analyzer_exception_is_contained():
# ---------------------------------------------------------------------------
# recall_history
# history tools (search_history / replay_steps / get_step_screenshot)
# ---------------------------------------------------------------------------
def _history_tool_double(return_value=None, side_effect=None):
from artemis.tools.history import SearchHistoryArgs
tool = Mock()
tool.args_schema = SearchHistoryArgs
tool.execute = AsyncMock(return_value=return_value, side_effect=side_effect)
return tool
@pytest.mark.asyncio
async def test_recall_history_routes_to_the_shared_tool_with_text_result():
async def test_history_tool_routes_to_the_shared_tool_with_text_result():
executor = _make_executor()
with patch("artemis.tools.history_recall.recall_history") as tool:
tool.execute = AsyncMock(return_value="- Step 3 (T+00:30): tap btn3 -> executed")
tool = _history_tool_double(return_value="- Step 3 (T+00:30): tap btn3 -> executed")
with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool) as by_name:
result = await executor.execute(
"recall_history",
"search_history",
{"query": "login Error timeout", "step_range": [1, 5], "unknown_arg": 1},
"tc-recall",
"tc-search",
None,
)
by_name.assert_called_once_with("search_history")
tool.execute.assert_awaited_once_with(
ctx=executor.ctx, query="login Error timeout", step_range=[1, 5]
)
@@ -257,27 +273,40 @@ async def test_recall_history_routes_to_the_shared_tool_with_text_result():
@pytest.mark.asyncio
async def test_recall_history_forwards_multimodal_blocks():
async def test_history_tool_forwards_multimodal_blocks():
executor = _make_executor()
blocks = [
{"type": "text", "text": "Step 4 matched."},
{"type": "text", "text": "Screenshot of step 4 (pre-action) is attached."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,AAA"}},
]
with patch("artemis.tools.history_recall.recall_history") as tool:
tool.execute = AsyncMock(return_value=blocks)
result = await executor.execute(
"recall_history", {"query": "settings", "include_images": True}, "tc", None
)
tool = _history_tool_double(return_value=blocks)
with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool):
result = await executor.execute("get_step_screenshot", {"query": "unused"}, "tc", None)
assert result.status == "success"
assert result.text_summary == "Step 4 matched."
assert result.text_summary == "Screenshot of step 4 (pre-action) is attached."
assert result.raw_result == blocks
@pytest.mark.asyncio
async def test_recall_history_exception_is_contained():
async def test_history_tool_exception_is_contained():
executor = _make_executor()
with patch("artemis.tools.history_recall.recall_history") as tool:
tool.execute = AsyncMock(side_effect=RuntimeError("db gone"))
result = await executor.execute("recall_history", {"query": "x"}, "tc", None)
tool = _history_tool_double(side_effect=RuntimeError("db gone"))
with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool):
result = await executor.execute("replay_steps", {"query": "x"}, "tc", None)
assert result.status == "success" # contained: the answer explains the failure
assert result.text_summary == "recall_history failed: db gone"
assert result.text_summary == "replay_steps failed: db gone"
@pytest.mark.asyncio
async def test_history_tools_dispatch_to_the_real_shared_instances():
"""No per-tool branch: every history tool name resolves through the package."""
executor = _make_executor() # ctx.data_engine is None -> tools degrade to text
for name in ("search_history", "replay_steps", "get_step_screenshot"):
result = await executor.execute(
name, {"query": "x", "start_step": 1, "step_number": 1}, "tc", None
)
assert result.status == "success"
assert (
"no active execution history" in result.text_summary
or "no execution history" in result.text_summary
)
+153
View File
@@ -77,6 +77,10 @@ def test_tool_signatures():
assert "action" in sig_inspect.parameters
assert "trace_id" in sig_inspect.parameters
assert "step_number" in sig_inspect.parameters
assert "query" in sig_inspect.parameters
assert "step_range" in sig_inspect.parameters
assert sig_inspect.parameters["max_results"].default == 5
assert "'search'" in (mobile_inspect_trace.__doc__ or "")
def test_mobile_run_task_invalid_model():
@@ -501,3 +505,152 @@ def test_mobile_manage_task_stop_via_daemon(temp_trace_env, monkeypatch):
mock_stop.assert_called_once_with(trace_id)
assert res["status"] == "cancelled"
assert trace_store.read_status(trace_id)["status"] == "cancelled"
def _seed_trace(temp_dir: str, trace_id: str, *, with_image: bool = True) -> str:
"""Writes one Flash-style step (with a tool trace and a pre screenshot)
through the real storage layer, exactly as a live run would."""
import io
from PIL import Image
from artemis.data_engine.models import (
ImageRecord,
SessionMetadata,
StepRecord,
TraceRecord,
)
from artemis.data_engine.storage import StorageManager
storage = StorageManager(os.path.join(temp_dir, "data_engine.db"), temp_dir)
storage.create_session(
SessionMetadata(session_id=trace_id, initial_goal="Replay test", start_time=1000.0)
)
image_name = None
if with_image:
image_name = "a" * 64
images_dir = os.path.join(temp_dir, "images")
os.makedirs(images_dir, exist_ok=True)
buf = io.BytesIO()
Image.new("RGB", (1080, 2400), "white").save(buf, format="JPEG")
with open(os.path.join(images_dir, f"{image_name}.jpg"), "wb") as f:
f.write(buf.getvalue())
storage.create_image(
ImageRecord(
image_name=image_name,
ocr_result=[{"text": "Save ocrneedle", "bounds": [0, 0, 10, 10]}],
)
)
step_id = str(uuid.uuid4())
storage.create_step(
StepRecord(
step_id=step_id,
session_id=trace_id,
step_number=1,
timestamp=1012.0,
pre_image_name=image_name,
summary="Asked the explorer, then tapped Save.",
action_taken={"action": "click", "target": [540, 1200], "target_text": "Save"},
operator_raw_thinking="Confirm the toggle first.",
last_execution_result={"status": "success"},
extra_metadata={"width": 1080, "height": 2400},
)
)
storage.create_trace(
TraceRecord(
trace_id=str(uuid.uuid4()),
session_id=trace_id,
step_id=step_id,
type="tool",
name="ask_explorer",
status="success",
timestamp=1011.0,
duration=0.8,
payload={"args": {"question": "toggle?"}, "result": "The toggle is ON. " * 100},
)
)
return step_id
@pytest.mark.asyncio
async def test_mobile_inspect_trace_step_details_replays_tool_calls_for_flash(temp_trace_env):
"""view_step_details works for Flash runs too and replays the recorded
tool calls through the same friendly-step path the live DataEngine uses."""
trace_id = str(uuid.uuid4())
trace_store.init_trace(trace_id, "Replay test", "Flash", "conv-1", device_serial="pixel-7")
_seed_trace(temp_trace_env, trace_id)
res = await mobile_inspect_trace(action="view_step_details", trace_id=trace_id, step_number=1)
assert "error" not in res
assert res["device_serial"] == "pixel-7"
details = res["details"]
assert "- **Step 1 (Start: 12.0s)**" in details
assert "[Screen]: Asked the explorer, then tapped Save." in details
assert "Confirm the toggle first." in details
assert "`ask_explorer(" in details
assert "The toggle is ON. " * 100 in details
# Coordinates are replayed normalized, not as stored pixels.
assert "[540, 1200]" not in details
assert "Tapped 'Save'" in details
missing = await mobile_inspect_trace(
action="view_step_details", trace_id=trace_id, step_number=9
)
assert missing["error"] == "Step not found"
@pytest.mark.asyncio
async def test_mobile_inspect_trace_step_screenshots_draw_the_overlay(temp_trace_env):
trace_id = str(uuid.uuid4())
trace_store.init_trace(trace_id, "Shot test", "Flash", "conv-1", device_serial="pixel-7")
_seed_trace(temp_trace_env, trace_id)
res = await mobile_inspect_trace(
action="view_step_screenshots", trace_id=trace_id, step_number=1
)
assert "error" not in res
assert res["before_screenshot"].startswith("file://")
assert res["before_screenshot"].endswith(f"{'a' * 64}.jpg")
assert res["after_screenshot"] is None
overlay = res["action_overlay_screenshot"]
assert overlay is not None
overlay_path = overlay[len("file://") :]
assert os.path.exists(overlay_path)
assert overlay_path.endswith("step_1_overlay.jpg")
# The overlay differs from the plain pre screenshot: the action was drawn.
with open(overlay_path, "rb") as f, open(res["before_screenshot"][7:], "rb") as g:
assert f.read() != g.read()
# Second call reuses the file it already wrote.
again = await mobile_inspect_trace(
action="view_step_screenshots", trace_id=trace_id, step_number=1
)
assert again["action_overlay_screenshot"] == overlay
no_step = await mobile_inspect_trace(
action="view_step_screenshots", trace_id=trace_id, step_number=None
)
assert no_step["error"] == "Missing parameter"
@pytest.mark.asyncio
async def test_mobile_inspect_trace_search_uses_the_shared_history_search(temp_trace_env):
trace_id = str(uuid.uuid4())
trace_store.init_trace(trace_id, "Search test", "Flash", "conv-1", device_serial="pixel-7")
_seed_trace(temp_trace_env, trace_id)
res = await mobile_inspect_trace(action="search", trace_id=trace_id, query="toggle")
assert "error" not in res
assert res["device_serial"] == "pixel-7"
assert "[Step 1 " in res["results"]
assert "Screen: Asked the explorer, then tapped Save." in res["results"]
# On-screen OCR text of the step's screenshot is part of the search surface.
res = await mobile_inspect_trace(action="search", trace_id=trace_id, query="ocrneedle")
assert "[Step 1 " in res["results"]
ledger = await mobile_inspect_trace(action="search", trace_id=trace_id, step_range=[1, 1])
assert "Action ledger for Steps 1–1" in ledger["results"]
empty = await mobile_inspect_trace(action="search", trace_id=trace_id)
assert empty["error"] == "Missing parameter"
+301 -5
View File
@@ -358,7 +358,7 @@ def _hashes(boundary_after: int):
def test_milestone_switch_closes_previous_segment_whole_and_gates_swap():
"""Ready-gated swap (user decision 2026-09-01): a milestone switch closes
"""Ready-gated swap: a milestone switch closes
the previous segment whole and dispatches its capsule, but the original
turns stay in the transcript until the capsule header is ready; the swap
happens at the first render after readiness."""
@@ -726,7 +726,7 @@ def test_era_overflow_compresses_oldest_era_to_period_paragraph():
assert "Verified: " in match.group(1)
# Recall guidance line is retained.
assert (
f" (Step-level ledger via recall_history for steps"
f" (Step-level ledger via search_history for steps"
f" {era.start_step_number}–{era.end_step_number})"
) in frozen
# Never-evict: the user injection survives even the period paragraph.
@@ -799,7 +799,7 @@ def test_period_paragraph_degrades_to_milestone_list_when_all_band1_missing():
block = render_era_block(era)
assert block.startswith(f"[Era 3 | Steps 1–8 | T+01:00 → T+08:59] {paragraph}")
assert "(Step-level ledger via recall_history for steps 1–8)" in block
assert "(Step-level ledger via search_history for steps 1–8)" in block
assert 'User @ Step 6: "stay logged in"' in block
@@ -1034,7 +1034,7 @@ async def test_flush_harvests_late_capsule_and_persists_ready_version():
# ---------------------------------------------------------------------------
# Ready-gated swap (user decision 2026-09-01): degradation ladder
# Ready-gated swap: degradation ladder
# ---------------------------------------------------------------------------
@@ -1098,7 +1098,7 @@ def test_hard_threshold_force_swaps_pending_chunks_into_l3():
# ---------------------------------------------------------------------------
# Band ① note-linkage machine check (user decision 2026-09-01)
# Band ① note-linkage machine check
# ---------------------------------------------------------------------------
@@ -1176,3 +1176,299 @@ def test_chunk_lists_every_step_of_a_multi_action_turn():
assert chunk.source_step_ids == ["s1", "s2", "s3"]
assert chunk.start_step_number == 1 and chunk.end_step_number == 3
assert "- Step 2 (T+00:20)" in chunk.band3
# ---------------------------------------------------------------------------
# Capsule source = the operator's own transcript
# ---------------------------------------------------------------------------
def _rich_turn(i: int) -> list:
"""A turn that asks the explorer before acting, with a UI list and a
native thinking block — the parts the old per-step projection dropped."""
from artemis.memory.transcript import PRO_UI_LIST_MARKER
return [
HumanMessage(
content=[
{"type": "text", "text": f"# CURRENT OBSERVATION [T+00:{i % 60:02d}]"},
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG_{i}"}},
{"type": "text", "text": f"{PRO_UI_LIST_MARKER}\n[0] Buy button"},
]
),
AIMessage(
content=[
{"type": "thinking", "thinking": "need the price first"},
{"type": "text", "text": "Ask the explorer for the price."},
],
tool_calls=[
{
"name": "ask_explorer",
"args": {"question": "price?"},
"id": f"tcx{i}",
"type": "tool_call",
}
],
),
ToolMessage(tool_call_id=f"tcx{i}", content="Price shown: ¥39"),
AIMessage(
content=f"thinking {i}",
tool_calls=[
{"name": "click", "args": {"target": i}, "id": f"tc{i}", "type": "tool_call"}
],
),
ToolMessage(tool_call_id=f"tc{i}", content="Action Recorded"),
]
def _payload_with_step(capsule: StubCapsuleService, step_number: int) -> dict:
for payload in capsule._step_inputs.values():
if any(s.get("step_number") == step_number for s in payload.get("steps", [])):
return payload
raise AssertionError(f"no capsule payload carries step {step_number}")
def test_capsule_payload_carries_each_turn_transcript_as_the_operator_saw_it():
"""The segment capsule digests exactly the text it replaces: every turn's
post-scrub transcript — reasoning (native thinking included), every tool
call with its arguments and returned result, the validator result — not
a hand-picked projection. Scrubbed parts (UI list, screenshot) are gone
from the source too, because the operator no longer saw them either."""
steps = [_step(i) for i in range(1, 7)]
ledger, chunker, _engine, capsule = _make(steps, min_active=2, max_steps=3)
for i in range(1, 7):
ledger.commit_staged(
step_key=f"s{i - 1}" if i > 1 else None,
validator_result={"status": "success"} if i > 1 else None,
)
ledger.render([_observation(i)])
ledger.stage_turn(_rich_turn(i) if i == 2 else _turn(i))
chunker.on_step_stamped(f"s{i}", "hash-a")
payload = _payload_with_step(capsule, 2)
turn = next(t for t in payload["turns"] if t["steps"][0]["step_number"] == 2)
transcript = turn["transcript"]
# What the operator saw during that turn, verbatim and in order.
assert "(thinking) need the price first" in transcript
assert "Ask the explorer for the price." in transcript
assert '[tool call] ask_explorer({"question": "price?"})' in transcript
assert "[tool result ask_explorer]\nPrice shown: ¥39" in transcript
assert '[tool call] click({"target": 2})' in transcript
assert "--- Action Execution Result" in transcript and "Status: success" in transcript
assert transcript.index("Price shown") < transcript.index("click(")
# What the scrub edge already removed from the operator's view.
assert "Buy button" not in transcript and "IMG_2" not in transcript
# The lens request renders the turn block: step facts, then the transcript.
lens = StepCapsuleLens(model_name="test", llm=object())
text = lens.build_messages(payload)[1].content
assert "## Step 2 (T+00:20)" in text
assert "- Step 2 (T+00:20): " in text and "Visual transition of step 2." in text
assert "Transcript as seen by the operator during this turn:" in text
assert text.index("Recorded steps:") < text.index("Price shown: ¥39")
assert "Reasoning excerpt" not in text # the transcript supersedes the excerpt
def test_chunk_without_ledger_falls_back_to_per_step_projection():
"""A chunk built without a live ledger (no transcripts) keeps the
per-step fallback rendering so the capsule can still be produced."""
steps = [_step(i) for i in range(1, 3)]
_ledger, chunker, _engine, capsule = _make(steps)
turns = [{"step_key": f"s{i}", "step_keys": [f"s{i}"], "start": 0, "end": 3} for i in (1, 2)]
chunk = chunker._create_chunk(turns, None, {s["step_id"]: s for s in steps})
assert chunk is not None
payload = capsule._step_inputs[chunk.capsule_key]
assert "turns" not in payload
text = StepCapsuleLens(model_name="test", llm=object()).build_messages(payload)[1].content
assert "## Step 1 (T+00:10)" in text and "## Step 2 (T+00:20)" in text
assert "Reasoning excerpt: thinking 1" in text
assert "Transcript as seen by the operator" not in text
def test_turn_transcript_is_capped_in_place_inside_the_capsule_request():
lens = StepCapsuleLens(model_name="test", llm=object())
cap = StepCapsuleLens.MAX_TURN_TRANSCRIPT_CHARS
step = {"step_number": 4, "offset": "T+00:40", "action": "tap", "outcome": "executed"}
payload = {
"start_step": 4,
"end_step": 4,
"steps": [step],
"turns": [{"steps": [step], "transcript": "x" * (cap + 500)}],
}
text = lens.build_messages(payload)[1].content
assert "x" * (cap + 1) not in text
assert "500 characters cut here" in text
def test_capped_transcript_keeps_its_tail_so_the_validator_result_survives():
"""The cut lands in the middle: the head (observation + reasoning) and the
tail (the validator result that closes every turn) both stay verbatim."""
cap = StepCapsuleLens.MAX_TURN_TRANSCRIPT_CHARS
tail = "\n[observation]\n--- Action Execution Result (T+09:59) ---\nStatus: failed\nError: boom"
transcript = "HEAD-START " + "h" * (cap + 3000) + tail
capped = StepCapsuleLens._cap_transcript(transcript)
assert capped.startswith("HEAD-START ")
assert capped.endswith(tail)
assert "characters cut here" in capped
assert len(capped) <= cap + 200 # cap plus the announcement line
def test_visual_transition_line_is_skipped_when_the_transcript_already_has_it():
"""The resolved visual summary sits verbatim in the transcript once the
scrub edge replaced the screenshot; only an unresolved turn (placeholder
still there) gets the DataEngine summary as a separate line."""
from artemis.agents.flash.context_compressor import HISTORY_SUMMARY_PREFIX
step = {
"step_number": 4,
"offset": "T+00:40",
"action": "tap",
"outcome": "executed",
"visual_summary": "The cart badge changed from 0 to 1.",
}
resolved = f"[observation]\n{HISTORY_SUMMARY_PREFIX}The cart badge changed from 0 to 1."
unresolved = "[observation]\n[screenshot]"
for transcript, expected_count in ((resolved, 1), (unresolved, 1), ("", 1)):
payload = {"start_step": 4, "end_step": 4, "steps": [step]}
if transcript:
payload["turns"] = [{"steps": [step], "transcript": transcript}]
text = StepCapsuleLens(model_name="test", llm=object()).build_messages(payload)[1].content
assert text.count("The cart badge changed from 0 to 1.") == expected_count, transcript
resolved_text = (
StepCapsuleLens(model_name="test", llm=object())
.build_messages(
{
"start_step": 4,
"end_step": 4,
"steps": [step],
"turns": [{"steps": [step], "transcript": resolved}],
}
)[1]
.content
)
assert "Visual transition:" not in resolved_text
unresolved_text = (
StepCapsuleLens(model_name="test", llm=object())
.build_messages(
{
"start_step": 4,
"end_step": 4,
"steps": [step],
"turns": [{"steps": [step], "transcript": unresolved}],
}
)[1]
.content
)
assert "Visual transition: The cart badge" in unresolved_text
def test_size_trigger_measures_the_rendered_transcript_the_capsule_receives():
"""``turn_text_chars`` sizes the exact text the lens gets — tool-call
arguments included — so the source-token trigger and the capsule payload
never disagree about how big a segment is."""
steps = [_step(i) for i in range(1, 5)]
ledger, chunker, _engine, capsule = _make(steps, min_active=1, target_tokens=10**9)
_run_turns(ledger, chunker, 1, 4)
turns = ledger.unchunked_turns()
assert ledger.turn_text_chars(turns) == sum(len(ledger.turn_transcript(t)) for t in turns)
assert ledger.turn_text_chars(turns[:1]) == len(ledger.turn_transcript(turns[0]))
assert '[tool call] click({"target": 1})' in ledger.turn_transcript(turns[0])
@pytest.mark.asyncio
async def test_ready_capsule_releases_its_turn_transcripts():
"""Once a capsule landed the job is never re-run, so the service drops the
bulky ``turns`` from the retained payload (the flat step facts stay)."""
from artemis.memory.chunking import ChunkCapsuleService
class EchoLens(StepCapsuleLens):
async def render(self, key, payload):
return json.dumps(_capsule(1, 1))
service = ChunkCapsuleService(None, EchoLens(model_name="test", llm=object()))
payload = {
"start_step": 1,
"end_step": 1,
"step_number": 1,
"steps": [{"step_number": 1}],
"turns": [{"steps": [{"step_number": 1}], "transcript": "big text"}],
}
service.submit("chunk:1-1", payload)
await service.flush()
assert service.has_summary("chunk:1-1")
assert "turns" not in service.get_job_payload("chunk:1-1")
assert service.get_job_payload("chunk:1-1")["steps"] == [{"step_number": 1}]
def test_multi_action_turn_renders_one_block_with_every_recorded_step():
lens = StepCapsuleLens(model_name="test", llm=object())
steps = [
{"step_number": n, "offset": f"T+00:{n}0", "action": f"tap btn{n}", "outcome": "executed"}
for n in (7, 8)
]
payload = {"start_step": 7, "end_step": 8, "steps": steps}
payload["turns"] = [{"steps": steps, "transcript": "[operator]\nfired two taps"}]
text = lens.build_messages(payload)[1].content
assert "## Steps 7–8 (T+00:70 → T+00:80)" in text
assert "- Step 7 (T+00:70): tap btn7 -> executed" in text
assert "- Step 8 (T+00:80): tap btn8 -> executed" in text
assert "fired two taps" in text
# ---------------------------------------------------------------------------
# Band ② must not restate band ③
# ---------------------------------------------------------------------------
def test_band2_coordinates_are_machine_checked():
"""A coordinate literal in any band-② interval text fails the attempt
(regenerate), same rank as the coverage check: band ③ already carries
every action's target, so ② repeating it is the ledger written twice."""
from artemis.memory.chunking import band2_intervals_carry_coordinates
lens = StepCapsuleLens(model_name="test", llm=object())
payload = {"start_step": 3, "end_step": 6, "steps": []}
with_coords = dict(
_capsule(3, 6),
intervals=[
{"start_step": 3, "end_step": 5, "text": "opened three sections in turn"},
{"start_step": 6, "end_step": 6, "text": "tapped 'Battery' at [320, 399]"},
],
)
assert band2_intervals_carry_coordinates(with_coords["intervals"])
assert lens.parse_capsule(json.dumps(with_coords), payload) is None
swipe_literal = dict(
_capsule(3, 6),
intervals=[{"start_step": 3, "end_step": 6, "text": "swiped [556, 289, 556, 124]"}],
)
assert lens.parse_capsule(json.dumps(swipe_literal), payload) is None
behavior_only = dict(
_capsule(3, 6),
intervals=[
{
"start_step": 3,
"end_step": 6,
"text": "Opened 'Apps' and 'Battery' and returned to the main list each time;"
" the Battery page showed 100% and 3 rows.",
}
],
)
assert not band2_intervals_carry_coordinates(behavior_only["intervals"])
assert lens.parse_capsule(json.dumps(behavior_only), payload) is not None
def test_capsule_prompt_forbids_band2_coordinates_and_sets_a_length_target():
"""The prompt carries the ② contract (behavior + observed result, no
coordinates, homogeneous runs merged) and the ①+② soft length target,
while keeping the preserve-don't-abstract rule."""
prompt = StepCapsuleLens(model_name="test", llm=object())._prompt
assert "NEVER include coordinates" in prompt
assert "NEVER narrate step by step" in prompt
assert "Merge homogeneous consecutive actions" in prompt
assert "one third of the source text" in prompt
assert "Preserve, don't abstract" in prompt
@@ -320,3 +320,101 @@ def test_commit_records_every_step_key_of_a_multi_action_turn():
ledger.stage_turn(_turn(2))
ledger.commit_staged(step_key=None)
assert ledger.unchunked_turns()[1]["step_keys"] == []
# ---------------------------------------------------------------------------
# Turn transcript rendering (chunk-capsule source)
# ---------------------------------------------------------------------------
def test_render_turn_transcript_is_lossless_over_text_thinking_tools_and_results():
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
from artemis.memory.transcript import SCREENSHOT_PLACEHOLDER, render_turn_transcript
messages = [
HumanMessage(
content=[
{"type": "text", "text": "# CURRENT OBSERVATION [T+01:00]"},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG"}},
]
),
AIMessage(
content=[
{"type": "thinking", "thinking": "compare the two prices"},
{"type": "text", "text": "I will recall the earlier price."},
],
tool_calls=[
{
"name": "search_history",
"args": {"query": "price"},
"id": "c1",
"type": "tool_call",
}
],
),
ToolMessage(tool_call_id="c1", content="Step 3: price ¥39", status="success"),
ToolMessage(
tool_call_id="c2", name="adb_shell", content="permission denied", status="error"
),
ToolMessage(tool_call_id="c9", content="orphan result"),
SystemMessage(content="You have reached the maximum number of tool calls"),
AIMessage(
content="tap now",
tool_calls=[{"name": "click", "args": {"target": [0.5, 0.5]}, "id": "c3"}],
),
]
text = render_turn_transcript(messages)
assert text.splitlines()[0] == "[observation]"
assert "# CURRENT OBSERVATION [T+01:00]" in text
assert SCREENSHOT_PLACEHOLDER in text and "IMG" not in text
assert "[operator]\n(thinking) compare the two prices\nI will recall the earlier price." in text
assert '[tool call] search_history({"query": "price"})' in text
# A Pro ToolMessage carries no name: resolved through the turn's own call.
assert "[tool result search_history]\nStep 3: price ¥39" in text
assert "[tool result adb_shell (error)]\npermission denied" in text
assert "[tool result c9]\norphan result" in text # unresolvable: id shown
assert "[system]\nYou have reached the maximum number of tool calls" in text
assert '[tool call] click({"target": [0.5, 0.5]})' in text
# Order is message order.
assert text.index("search_history(") < text.index("price ¥39") < text.index("tap now")
def test_ledger_turn_transcript_reflects_the_scrubbed_active_region():
"""The transcript handed to chunk compression is the turn as it stands
after the scrub edge: the UI list stripped, the validator result present."""
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from artemis.memory.transcript import PRO_UI_LIST_MARKER, TranscriptLedger
ledger = TranscriptLedger(clock=lambda: 0.0)
def turn(i: int) -> list:
return [
HumanMessage(
content=[
{"type": "text", "text": f"# CURRENT OBSERVATION [T+00:0{i}]"},
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG{i}"}},
{"type": "text", "text": f"{PRO_UI_LIST_MARKER}\n[0] Button {i}"},
]
),
AIMessage(
content=f"thought {i}",
tool_calls=[{"name": "click", "args": {}, "id": f"t{i}"}],
),
ToolMessage(tool_call_id=f"t{i}", content="Action Recorded"),
]
ledger.stage_turn(turn(1))
ledger.commit_staged(step_key="s1", validator_result={"status": "success"})
ledger.stage_turn(turn(2))
ledger.commit_staged(step_key="s2", validator_result={"status": "failed", "error": "boom"})
ledger.render([HumanMessage(content=[{"type": "text", "text": "tail"}])])
first, second = ledger.unchunked_turns()
text = ledger.turn_transcript(first)
assert "thought 1" in text and "[tool call] click({})" in text
assert "Button 1" not in text # depth-1 strip already applied
assert "--- Action Execution Result (T+00:00) ---\nStatus: success" in text
assert "Status: failed" in ledger.turn_transcript(second)
assert "thought 2" not in text # a turn transcript never leaks into another turn
+41
View File
@@ -0,0 +1,41 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The Pro tuning summary the SDK persists with every session's device_info."""
from artemis.sdk.agent import run_tuning_summary
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
def test_flash_runs_persist_no_tuning():
config = AgentConfigBuilder().build(validate_profiles=False)
assert run_tuning_summary(config, "flash") is None
def test_pro_runs_persist_the_two_launcher_sliders():
config = (
AgentConfigBuilder()
.with_verification_level("checkpoints")
.with_explorer(pro_mode="ultra")
.build(validate_profiles=False)
)
assert run_tuning_summary(config, "pro") == {
"verification_level": "checkpoints",
"explorer_mode": "ultra",
}
def test_pro_summary_reflects_checker_switched_off():
config = AgentConfigBuilder().with_verification_level("off").build(validate_profiles=False)
assert run_tuning_summary(config, "pro")["verification_level"] == "off"
+39
View File
@@ -808,3 +808,42 @@ class TestVerificationLevelPresets:
)
== "off"
)
class TestRunTuningSummary:
"""``run_tuning_for_profile``: the per-run tuning persisted with a session."""
def test_flash_has_no_tuning(self):
from artemis.config import CheckerConfig, ExplorerConfig, run_tuning_for_profile
kwargs = {"checker": CheckerConfig(), "explorer": ExplorerConfig()}
assert run_tuning_for_profile("flash", **kwargs) is None
assert run_tuning_for_profile(None, **kwargs) is None
def test_pro_reports_both_sliders(self):
from artemis.config import (
CheckerConfig,
ExplorerConfig,
checker_overrides_for_level,
run_tuning_for_profile,
)
summary = run_tuning_for_profile(
" Pro ",
checker=CheckerConfig(**checker_overrides_for_level("strict")),
explorer=ExplorerConfig(pro_mode="ultra"),
)
assert summary == {"verification_level": "strict", "explorer_mode": "ultra"}
def test_pro_defaults_match_launcher_defaults(self):
from artemis.config import (
CheckerConfig,
ExplorerConfig,
run_tuning_for_profile,
verification_level_for_checker,
)
checker, explorer = CheckerConfig(), ExplorerConfig()
summary = run_tuning_for_profile("pro", checker=checker, explorer=explorer)
assert summary["verification_level"] == verification_level_for_checker(checker)
assert summary["explorer_mode"] == explorer.resolve(profile="pro")
-1
View File
@@ -239,4 +239,3 @@ async def test_mid_stream_failure_records_stream_reset_payload(monkeypatch):
assert payload["reason"] == "mid_stream_failure"
assert "stream_exec_id" in payload
assert "lower API priority" in payload["message"]
+9 -2
View File
@@ -93,7 +93,10 @@ def test_recall_and_similarity_defaults():
assert cfg.memory.recall.enabled is True
assert cfg.memory.recall.max_results == 5
assert cfg.memory.recall.max_text_tokens == 2000
assert cfg.memory.recall.max_image_steps == 1
assert cfg.memory.recall.screen_scan_steps == 150
assert not hasattr(cfg.memory.recall, "max_image_steps")
assert cfg.memory.replay.max_steps == 5
assert cfg.memory.replay.max_tokens == 12000
assert cfg.memory.transcript.similarity_hint is True
assert cfg.memory.transcript.similarity_max_distance == 5
assert cfg.memory.policies == {}
@@ -116,7 +119,8 @@ def test_recall_and_policies_overridable():
cfg = AgentGlobalConfig.model_validate(
{
"memory": {
"recall": {"enabled": False, "max_results": 3},
"recall": {"enabled": False, "max_results": 3, "screen_scan_steps": 20},
"replay": {"max_steps": 2, "max_tokens": 4000},
"transcript": {"similarity_hint": False},
"policies": {"planner": {"last_n_detailed": 3}},
}
@@ -124,5 +128,8 @@ def test_recall_and_policies_overridable():
)
assert cfg.memory.recall.enabled is False
assert cfg.memory.recall.max_results == 3
assert cfg.memory.recall.screen_scan_steps == 20
assert cfg.memory.replay.max_steps == 2
assert cfg.memory.replay.max_tokens == 4000
assert cfg.memory.transcript.similarity_hint is False
assert cfg.memory.policies == {"planner": {"last_n_detailed": 3}}
+22
View File
@@ -169,3 +169,25 @@ def test_record_llm_usage_lens_calls_do_not_touch_last_prompt_tokens():
assert meter.last_prompt_tokens == 15000 # untouched by the lens call
assert meter.llm_calls == 2
assert meter.prompt_tokens == 15300
def test_record_llm_usage_tags_the_traced_node():
"""Each usage trace names the traced node that issued the call so the
session usage endpoint can pick the executor's (Operator/FlashRunner)
prompt size out as the live context."""
from artemis.data_engine.context_vars import CURRENT_NODE_NAME
engine = Mock()
engine.current_session_id = uuid4()
engine.current_step_id = None
token = CURRENT_NODE_NAME.set("operator")
try:
payload = record_llm_usage(engine, _msg_with_usage(prompt=42000, completion=90))
finally:
CURRENT_NODE_NAME.reset(token)
assert payload["node"] == "operator"
assert payload["context_base_tokens"] == 42000
untagged = record_llm_usage(engine, _msg_with_usage(prompt=10))
assert "node" not in untagged
+117
View File
@@ -6,12 +6,21 @@
#
# http://www.apache.org/licenses/LICENSE-2.0
import io
import os
import subprocess
import sys
import pytest
from rich.console import Console
from artemis.interfaces.cli.commands import ui
from artemis.interfaces.cli.commands.ui import (
_npm_install_required,
_resolve_npm_executable,
_run_build_step,
_showcase_build_required,
ensure_showcase_built,
)
@@ -64,3 +73,111 @@ def test_showcase_build_not_required_when_build_is_current(tmp_path):
os.utime(built_index, (200, 200))
assert _showcase_build_required(showcase_dir) is False
def _showcase_with_manifests(tmp_path, *, stamp_mtime=None):
showcase_dir = tmp_path / "showcase_ui"
showcase_dir.mkdir()
for name in ("package.json", "package-lock.json"):
path = showcase_dir / name
path.write_text("{}", encoding="utf-8")
os.utime(path, (100, 100))
if stamp_mtime is not None:
stamp = showcase_dir / "node_modules" / ".package-lock.json"
stamp.parent.mkdir()
stamp.write_text("{}", encoding="utf-8")
os.utime(stamp, (stamp_mtime, stamp_mtime))
return showcase_dir
def test_npm_install_required_when_node_modules_missing(tmp_path):
assert _npm_install_required(_showcase_with_manifests(tmp_path)) is True
def test_npm_install_not_required_when_lock_stamp_is_current(tmp_path):
assert _npm_install_required(_showcase_with_manifests(tmp_path, stamp_mtime=200)) is False
def test_npm_install_required_when_package_lock_is_newer(tmp_path):
showcase_dir = _showcase_with_manifests(tmp_path, stamp_mtime=200)
os.utime(showcase_dir / "package-lock.json", (300, 300))
assert _npm_install_required(showcase_dir) is True
def _recording_console():
return Console(file=io.StringIO(), force_terminal=False, width=120)
def test_ensure_showcase_built_skips_install_when_deps_current(monkeypatch):
steps = []
monkeypatch.setattr(ui, "_showcase_build_required", lambda showcase_dir: True)
monkeypatch.setattr(ui, "_npm_install_required", lambda showcase_dir: False)
monkeypatch.setattr(ui, "_resolve_npm_executable", lambda: "npm")
monkeypatch.setattr(
ui, "_run_build_step", lambda console, label, cmd, cwd: steps.append((label, cmd))
)
console = _recording_console()
ensure_showcase_built(console)
assert [label for label, _ in steps] == ["② ng build"]
assert steps[0][1] == ["npm", "run", "build"]
assert "Showcase UI built in" in console.file.getvalue()
def test_ensure_showcase_built_runs_install_first_when_deps_changed(monkeypatch):
steps = []
monkeypatch.setattr(ui, "_showcase_build_required", lambda showcase_dir: True)
monkeypatch.setattr(ui, "_npm_install_required", lambda showcase_dir: True)
monkeypatch.setattr(ui, "_resolve_npm_executable", lambda: "npm")
monkeypatch.setattr(
ui, "_run_build_step", lambda console, label, cmd, cwd: steps.append((label, cmd))
)
ensure_showcase_built(_recording_console())
assert [label for label, _ in steps] == ["① npm install (dependencies changed)", "② ng build"]
assert steps[0][1] == ["npm", "install", "--no-audit", "--no-fund", "--loglevel=warn"]
def test_ensure_showcase_built_reports_failure_with_manual_command(monkeypatch):
monkeypatch.setattr(ui, "_showcase_build_required", lambda showcase_dir: True)
monkeypatch.setattr(ui, "_npm_install_required", lambda showcase_dir: False)
monkeypatch.setattr(ui, "_resolve_npm_executable", lambda: "npm")
def failing_step(console, label, cmd, cwd):
raise subprocess.CalledProcessError(1, cmd)
monkeypatch.setattr(ui, "_run_build_step", failing_step)
console = _recording_console()
ensure_showcase_built(console) # must not raise: the server still starts
output = console.file.getvalue()
assert "Failed to auto-build Showcase UI" in output
assert "npm install && npm run build" in output
def test_run_build_step_streams_child_output_and_raises_on_failure(tmp_path):
console = _recording_console()
cmd = [sys.executable, "-c", "print('hello from child'); import sys; sys.exit(3)"]
with pytest.raises(subprocess.CalledProcessError) as excinfo:
_run_build_step(console, "② ng build", cmd, tmp_path)
assert excinfo.value.returncode == 3
output = console.file.getvalue()
assert "② ng build" in output
assert "hello from child" in output
def test_run_build_step_strips_ansi_and_succeeds(tmp_path):
console = _recording_console()
cmd = [sys.executable, "-c", "print('\\x1b[32mdone\\x1b[0m')"]
_run_build_step(console, "step", cmd, tmp_path)
output = console.file.getvalue()
assert "done" in output
assert "\x1b[32m" not in output
@@ -621,3 +621,85 @@ async def test_next_segment_anchors_at_first_frame(mock_ctx, tmp_path):
assert session.android_segment_started_at >= before
assert session.android_rotation == 1
assert session.local_video_path == tmp_path / "recording_001.mkv"
@pytest.mark.asyncio
async def test_stop_recording_passes_session_offsets_to_manifest(mock_ctx, tmp_path):
"""The playlist manifest must carry each segment's real session-relative start."""
controller = UnifiedMobileController(mock_ctx)
remove_active_session("emulator-5554")
seg0 = tmp_path / "recording.mkv"
seg0.write_bytes(b"segment 0")
seg1 = tmp_path / "recording_001.mkv"
seg1.write_bytes(b"segment 1")
now = time.time()
session_t0 = now - 40.0
# First frame landed 1.2s after the session started; rotation restarted
# scrcpy so segment 1 begins 11.5s after that first frame (10s of video + 1.5s gap).
recording_t0 = session_t0 + 1.2
session = RecordingSession(
video_id=uuid4(),
device_id="emulator-5554",
start_time=recording_t0,
data_engine_start_time=session_t0,
local_video_path=seg1,
android_video_segments=[seg0],
android_segment_records=[
{
"path": seg0,
"output_path": tmp_path / "recording.mp4",
"start": 0.0,
"end": 10.0,
}
],
android_segment_index=1,
android_segment_started_at=recording_t0 + 11.5,
process=MagicMock(returncode=None, terminate=MagicMock(), wait=AsyncMock()),
)
set_active_session("emulator-5554", session)
manifest = AsyncMock(return_value=tmp_path / "recording.json")
with (
patch(
"artemis.controllers.unified_controller.remux_recording_to_mp4",
AsyncMock(side_effect=lambda src, dst: (dst.write_bytes(b"mp4"), True)[1]),
),
patch("artemis.controllers.unified_controller.write_recording_manifest", manifest),
):
res = await controller.stop_video_recording()
assert res.success is True
manifest.assert_awaited_once()
output_dir, mp4_paths, offsets = manifest.await_args.args
assert output_dir == tmp_path
assert mp4_paths == [tmp_path / "recording.mp4", tmp_path / "recording_001.mp4"]
assert abs(offsets[tmp_path / "recording.mp4"] - 1.2) < 0.01
assert abs(offsets[tmp_path / "recording_001.mp4"] - 12.7) < 0.01
def test_segment_session_offsets_without_data_engine_anchor(tmp_path):
session = RecordingSession(
video_id=uuid4(),
device_id="emulator-5554",
start_time=1000.0,
data_engine_start_time=None,
local_video_path=tmp_path / "recording.mkv",
android_segment_records=[
{"path": tmp_path / "a.mkv", "output_path": tmp_path / "recording.mp4", "start": 0.0},
{
"path": tmp_path / "b.mkv",
"output_path": tmp_path / "recording_001.mp4",
"start": 7.5,
},
],
)
fallback = tmp_path / "recording_converted.mp4"
offsets = UnifiedMobileController._segment_session_offsets(
session, [tmp_path / "recording.mp4", tmp_path / "recording_001.mp4", fallback]
)
assert offsets[tmp_path / "recording.mp4"] == 0.0
assert offsets[tmp_path / "recording_001.mp4"] == 7.5
# An emergency fallback remux has no record; it starts at the recording anchor.
assert offsets[fallback] == 0.0
@@ -0,0 +1,99 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""One args schema, two contracts: the Flash ``ToolDeclaration`` and the
LangChain tool of every history tool are derived from the same Pydantic model."""
from types import SimpleNamespace
import pytest
from artemis.core.tool_declaration import ToolDeclaration
from artemis.tools.history import (
GET_STEP_SCREENSHOT_TOOL,
HISTORY_TOOL_DECLARATIONS,
HISTORY_TOOL_NAMES,
HISTORY_TOOL_WRAPPERS,
HISTORY_TOOLS,
REPLAY_STEPS_TOOL,
SEARCH_HISTORY_TOOL,
get_history_tools,
history_tool_by_name,
history_tool_declarations,
)
@pytest.mark.parametrize("tool", HISTORY_TOOLS, ids=lambda t: t.name)
def test_tool_declaration_mirrors_the_args_schema(tool):
declaration = tool.to_tool_declaration()
assert isinstance(declaration, ToolDeclaration)
assert declaration.name == tool.name
assert declaration.description == tool.description
properties = declaration.parameters["properties"]
assert list(properties) == list(tool.args_schema.model_fields)
for field_name, info in tool.args_schema.model_fields.items():
assert properties[field_name]["description"] == info.description
assert declaration.parameters["required"] == [
name for name, info in tool.args_schema.model_fields.items() if info.is_required()
]
# The LangChain export binds the identical schema.
lc_tool = tool.to_langchain_tool(SimpleNamespace(data_engine=None))
assert lc_tool.name == tool.name
assert lc_tool.args_schema is tool.args_schema
# And the GenAI export agrees on names and required fields.
genai = tool.to_genai_declaration()
assert set(genai.parameters.properties) == set(properties)
assert list(genai.parameters.required or []) == declaration.parameters["required"]
def test_declared_types_and_enums():
search = SEARCH_HISTORY_TOOL.parameters["properties"]
assert search["query"]["type"] == "string"
assert search["step_range"] == {
"type": "array",
"items": {"type": "integer"},
"description": search["step_range"]["description"],
}
assert search["max_results"]["type"] == "integer"
assert search["max_results"]["default"] == 5
assert SEARCH_HISTORY_TOOL.parameters["required"] == []
replay = REPLAY_STEPS_TOOL.parameters["properties"]
assert replay["start_step"]["type"] == "integer"
assert replay["end_step"]["type"] == "integer"
assert REPLAY_STEPS_TOOL.parameters["required"] == ["start_step"]
shot = GET_STEP_SCREENSHOT_TOOL.parameters["properties"]
assert shot["step_number"]["type"] == "integer"
assert shot["which"]["type"] == "string"
assert shot["which"]["enum"] == ["pre", "post", "overlay"]
assert shot["which"]["default"] == "pre"
assert GET_STEP_SCREENSHOT_TOOL.parameters["required"] == ["step_number"]
def test_exports_are_consistent():
assert HISTORY_TOOL_NAMES == {"search_history", "replay_steps", "get_step_screenshot"}
assert [d.name for d in HISTORY_TOOL_DECLARATIONS] == [t.name for t in HISTORY_TOOLS]
assert len(HISTORY_TOOL_WRAPPERS) == 3
assert history_tool_by_name("default_api:replay_steps").name == "replay_steps"
assert history_tool_by_name("nope") is None
ctx = SimpleNamespace(data_engine=object())
expected = [t.name for t in HISTORY_TOOLS]
assert [t.name for t in get_history_tools(ctx)] == expected
assert [d.name for d in history_tool_declarations(ctx)] == expected
assert history_tool_declarations(SimpleNamespace(data_engine=None)) == []
+150
View File
@@ -0,0 +1,150 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""replay_steps: range replay, per-call step cap, whole-step token budget,
``[Screen]`` status lines, and the no-history degradation."""
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from artemis.tools.history import ReplayStepsTool, replay_steps, replay_steps_text
def _friendly_step(number, tool_result="The toggle is ON", **overrides):
step = {
"step_id": f"step_{number}",
"step_number": number,
"relative_time": f"{number * 3}.0s",
"summary": f"Step {number} summary.",
"action_taken": {"action": "click", "target": [0.5, 0.5], "target_text": "Save"},
"last_execution_result": {"status": "success"},
"interleaved_events": [
{"type": "thought", "content": f"Thinking at step {number}."},
{
"type": "tool_call",
"name": "ask_explorer",
"args": {"question": "toggle?"},
"result": tool_result,
},
],
}
step.update(overrides)
return step
def _reader(steps):
reader = MagicMock()
reader.get_agent_friendly_steps_in_range.side_effect = lambda s, e: [
st for st in steps if s <= st["step_number"] <= e
]
return reader
def test_replay_range_renders_every_step_in_full():
reader = _reader([_friendly_step(3), _friendly_step(4, tool_result="The toggle is OFF " * 100)])
out = replay_steps_text(reader, 3, 4)
reader.get_agent_friendly_steps_in_range.assert_called_once_with(3, 4)
assert "- **Step 3 (Start: 9.0s)**" in out
assert "- **Step 4 (Start: 12.0s)**" in out
assert "[Screen]: Step 3 summary." in out
assert "Thinking at step 4." in out
assert "`ask_explorer(" in out
# Loose clamp: a ~1.8k-char tool result is replayed in full.
assert "The toggle is OFF " * 100 in out
assert "[Planned Action]: Tapped 'Save' at [0.5, 0.5]" in out
def test_replay_single_step_reversed_bounds_and_missing_step():
reader = _reader([_friendly_step(2)])
assert "- **Step 2 (Start: 6.0s)**" in replay_steps_text(reader, 2)
reader.get_agent_friendly_steps_in_range.assert_called_once_with(2, 2)
reader = _reader([_friendly_step(2), _friendly_step(3)])
out = replay_steps_text(reader, 3, 2)
reader.get_agent_friendly_steps_in_range.assert_called_once_with(2, 3)
assert out.index("**Step 2") < out.index("**Step 3")
assert replay_steps_text(_reader([]), 9) == "Error: step 9 not found."
assert replay_steps_text(_reader([]), 9, 12) == "Error: no recorded steps in range 9–12."
assert "must be integers" in replay_steps_text(_reader([]), "x")
def test_replay_caps_the_range_per_call():
reader = _reader([_friendly_step(n) for n in range(1, 30)])
out = replay_steps_text(reader, 1, 20, max_steps=5)
reader.get_agent_friendly_steps_in_range.assert_called_once_with(1, 5)
assert "**Step 5" in out
assert "**Step 6" not in out
assert "Call again from Step 6" in out
def test_replay_token_budget_drops_whole_trailing_steps():
big = "x" * 4000 # ~1k tokens per step
reader = _reader([_friendly_step(n, tool_result=big) for n in range(1, 6)])
out = replay_steps_text(reader, 1, 5, max_steps=5, max_tokens=2500)
# Two steps fit; the rest are dropped as whole steps, never cut mid-step.
assert "**Step 1" in out and "**Step 2" in out
assert "**Step 3" not in out
assert out.count(big) == 2
assert "Token budget reached: Steps 1–2 replayed; 3 more step(s)" in out
assert "Call again from Step 3" in out
def test_replay_always_shows_at_least_the_first_step():
reader = _reader([_friendly_step(1, tool_result="y" * 4000), _friendly_step(2)])
out = replay_steps_text(reader, 1, 2, max_tokens=10)
assert "**Step 1" in out
assert "y" * 4000 in out
assert "**Step 2" not in out
def test_replay_screen_line_states():
with_text = replay_steps_text(_reader([_friendly_step(1)]), 1)
assert "[Screen]: Step 1 summary." in with_text
assert "[Summary]" not in with_text
pending = replay_steps_text(
_reader([_friendly_step(1, summary=None, extra_metadata={"summary_status": "pending"})]),
1,
)
assert "[Screen]: (screen description pending)" in pending
failed = replay_steps_text(
_reader([_friendly_step(1, summary="", extra_metadata={"summary_status": "failed"})]),
1,
)
assert "[Screen]: (screen description unavailable)" in failed
no_status = replay_steps_text(_reader([_friendly_step(1, summary=None)]), 1)
assert "[Screen]" not in no_status
def test_replay_without_history():
assert "no execution history" in replay_steps_text(None, 1)
@pytest.mark.asyncio
async def test_tool_execute_reads_the_context_engine_and_degrades_without_one():
reader = _reader([_friendly_step(2)])
out = await ReplayStepsTool().execute(ctx=SimpleNamespace(data_engine=reader), start_step=2)
assert "- **Step 2 (Start: 6.0s)**" in out
out = await replay_steps.execute(ctx=SimpleNamespace(data_engine=None), start_step=1)
assert "no execution history" in out
assert replay_steps.is_available(SimpleNamespace(data_engine=None)) is False
assert replay_steps.is_available(SimpleNamespace(data_engine=reader)) is True
@@ -0,0 +1,120 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""tool_result_messages: one helper decides how a tool result — with or
without images — enters the conversation, per LLM provider."""
from enum import StrEnum
from types import SimpleNamespace
import pytest
from langchain_core.messages import HumanMessage, ToolMessage
from artemis.tools.tool_wrapper import (
resolve_image_carrier,
split_multimodal_result,
tool_result_messages,
)
IMAGE = {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,AAAA"}}
SHOT = [{"type": "text", "text": "Screenshot of step 7 (pre-action) is attached."}, IMAGE]
class _Provider(StrEnum):
GOOGLE = "google"
VERTEX_AI = "vertexai"
def _llm(provider):
return SimpleNamespace(endpoint=SimpleNamespace(provider=provider))
@pytest.mark.parametrize(
("provider", "carrier"),
[
("google", "tool"),
("anthropic", "tool"),
("vertexai", "human"),
("openai", "human"),
("openrouter", "human"),
("xai", "human"),
("ollama", "human"),
(None, "human"),
],
)
def test_resolve_image_carrier_by_provider_name(provider, carrier):
assert resolve_image_carrier(provider) == carrier
assert resolve_image_carrier(_llm(provider)) == carrier
def test_resolve_image_carrier_accepts_enums_and_endpoint_less_llms():
assert resolve_image_carrier(_Provider.GOOGLE) == "tool"
assert resolve_image_carrier(_llm(_Provider.VERTEX_AI)) == "human"
assert resolve_image_carrier(SimpleNamespace()) == "human"
def test_split_multimodal_result():
assert split_multimodal_result("plain") == ("plain", [])
assert split_multimodal_result(None) == ("", [])
assert split_multimodal_result(ToolMessage(content="x", tool_call_id="t")) == ("x", [])
text, images = split_multimodal_result(SHOT)
assert text == "Screenshot of step 7 (pre-action) is attached."
assert images == [IMAGE]
def test_text_only_results_stay_a_single_tool_message():
msgs = tool_result_messages("tc-1", "Step 3 replay", name="replay_steps", status="error")
assert len(msgs) == 1
assert isinstance(msgs[0], ToolMessage)
assert msgs[0].content == "Step 3 replay"
assert msgs[0].tool_call_id == "tc-1"
assert msgs[0].name == "replay_steps"
assert msgs[0].status == "error"
# Text-block lists pass through unchanged; other objects are stringified.
blocks = [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]
assert tool_result_messages("tc", blocks)[0].content == blocks
assert tool_result_messages("tc", {"k": 1})[0].content == "{'k': 1}"
def test_tool_carrier_puts_the_image_inside_the_tool_message():
msgs = tool_result_messages("tc-2", SHOT, name="get_step_screenshot", llm=_llm("google"))
assert len(msgs) == 1
assert isinstance(msgs[0], ToolMessage)
assert msgs[0].content == SHOT
assert msgs[0].name == "get_step_screenshot"
def test_human_carrier_splits_text_and_image_with_a_caption():
msgs = tool_result_messages(
"tc-3", SHOT, name="get_step_screenshot", status="success", llm=_llm("openai")
)
assert len(msgs) == 2
tool_msg, human_msg = msgs
assert isinstance(tool_msg, ToolMessage)
assert tool_msg.content == "Screenshot of step 7 (pre-action) is attached."
assert tool_msg.tool_call_id == "tc-3"
assert isinstance(human_msg, HumanMessage)
assert human_msg.content[0] == {
"type": "text",
"text": "[Screenshot returned by get_step_screenshot for step 7]",
}
assert human_msg.content[1] == IMAGE
def test_explicit_carrier_wins_over_the_llm():
msgs = tool_result_messages("tc", SHOT, image_carrier="human", llm=_llm("google"))
assert len(msgs) == 2
msgs = tool_result_messages("tc", SHOT, image_carrier="tool", llm=_llm("openai"))
assert len(msgs) == 1
+109
View File
@@ -0,0 +1,109 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""get_step_screenshot: pre / post / overlay, missing files, invalid variants."""
import base64
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from PIL import Image
from artemis.tools.history import get_step_screenshot, load_step_screenshot
def _jpeg_file(tmp_path, name="pre.jpg", color="white"):
path = tmp_path / name
Image.new("RGB", (200, 400), color).save(path, format="JPEG")
return path
def _reader(pre=None, post=None, action=None, record=True):
reader = MagicMock()
reader.get_step_image_path.side_effect = lambda n, which="pre": pre if which == "pre" else post
reader.get_step_record.return_value = SimpleNamespace(action_taken=action) if record else None
return reader
def test_pre_and_post_screenshots_are_attached_as_content_blocks(tmp_path):
pre = _jpeg_file(tmp_path, "pre.jpg", "white")
post = _jpeg_file(tmp_path, "post.jpg", "black")
reader = _reader(pre=pre, post=post)
shot = load_step_screenshot(reader, 2, "pre")
reader.get_step_image_path.assert_called_with(2, "pre")
assert shot.image_bytes == pre.read_bytes()
assert shot.description == "Screenshot of step 2 (pre-action) is attached."
blocks = shot.to_content_blocks()
assert blocks[0] == {"type": "text", "text": shot.description}
assert blocks[1]["type"] == "image_url"
url = blocks[1]["image_url"]["url"]
assert url.startswith("data:image/jpeg;base64,")
assert base64.b64decode(url.split(",", 1)[1]) == pre.read_bytes()
shot = load_step_screenshot(reader, 2, "post")
reader.get_step_image_path.assert_called_with(2, "post")
assert shot.image_bytes == post.read_bytes()
assert "post-action" in shot.description
def test_overlay_draws_the_recorded_raw_action(tmp_path):
pre = _jpeg_file(tmp_path)
reader = _reader(pre=pre, action={"action": "click", "target": [100, 200]})
shot = load_step_screenshot(reader, 2, "overlay")
reader.get_step_image_path.assert_called_once_with(2, "pre")
assert shot.overlay_drawn is True
assert "action drawn on it" in shot.description
assert shot.image_bytes != pre.read_bytes()
def test_overlay_falls_back_to_plain_pre_without_action(tmp_path):
pre = _jpeg_file(tmp_path)
shot = load_step_screenshot(_reader(pre=pre, action=None), 2, "overlay")
assert shot.overlay_drawn is False
assert shot.image_bytes == pre.read_bytes()
assert "plain screenshot" in shot.description
def test_missing_file_and_missing_step_return_plain_text():
shot = load_step_screenshot(_reader(pre=None, post=None), 3, "post")
assert shot.image_bytes is None
assert shot.to_content_blocks() == "No post-action screenshot recorded for step 3."
shot = load_step_screenshot(_reader(pre=None, record=False), 9, "pre")
assert shot.to_content_blocks() == "Error: step 9 not found."
def test_invalid_variant_and_step_number_are_rejected():
shot = load_step_screenshot(_reader(), 1, "bogus")
assert shot.image_bytes is None
assert "'pre', 'post' or 'overlay'" in shot.description
assert "must be an integer" in load_step_screenshot(_reader(), "one").description
assert "no execution history" in load_step_screenshot(None, 1).description
@pytest.mark.asyncio
async def test_tool_execute_returns_blocks_or_text(tmp_path):
pre = _jpeg_file(tmp_path)
ctx = SimpleNamespace(data_engine=_reader(pre=pre))
out = await get_step_screenshot.execute(ctx=ctx, step_number=1, which="pre")
assert isinstance(out, list) and out[1]["type"] == "image_url"
out = await get_step_screenshot.execute(ctx=ctx, step_number=1, which="post")
assert out == "No post-action screenshot recorded for step 1."
out = await get_step_screenshot.execute(ctx=SimpleNamespace(data_engine=None), step_number=1)
assert "no execution history" in out
+334
View File
@@ -0,0 +1,334 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""search_history boundaries and search correctness.
Hard boundaries under test: bounded result count, bounded response tokens,
every result carrying a step number. Plus: keyword/step-range/notes/chunk
filter correctness, the step-range ledger re-entry (full-width
``build_action_ledger`` rows), and the screen-text surface: the step's own
pre AND post screenshots plus screenshots its tool results embedded.
"""
import hashlib
import io
import re
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from PIL import Image
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
from artemis.tools.history import (
SearchHistoryTool,
search_history,
search_history_available,
search_history_text,
)
from artemis.tools.history.search import result_search_text
from artemis.utils.notes import save_note_content
def _jpeg(color: str) -> bytes:
buf = io.BytesIO()
Image.new("RGB", (64, 128), color).save(buf, format="JPEG")
return buf.getvalue()
def _make_engine(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
engine = DataEngine(mock_ctx)
engine.start_session("search test session")
return engine
def _flush(engine):
for t in list(engine._pending_threads):
t.join()
def _cfg(**overrides):
values = {"enabled": True, "max_results": 5, "max_text_tokens": 2000, "screen_scan_steps": 150}
values.update(overrides)
return SimpleNamespace(**values)
@pytest.fixture
def engine(tmp_path):
engine = _make_engine(tmp_path)
engine.record_step(
pre_screenshot_bytes=_jpeg("red"),
summary="Opened the login page.",
action_taken={"action": "click", "target_text": "Login entry"},
last_execution_result={"status": "success"},
)
engine.record_step(
pre_screenshot_bytes=_jpeg("green"),
post_screenshot_bytes=_jpeg("blue"),
summary="Typed the verification code 4711 into the input.",
action_taken={"action": "input_text", "text": "4711"},
last_execution_result={"status": "success"},
operator_raw_thinking="The code arrived via SMS; entering 4711 now.",
)
engine.record_step(
summary="A promo popup appeared and was dismissed.",
action_taken={"action": "click", "target_text": "Close popup"},
last_execution_result={"status": "failed", "error": "element vanished"},
)
_flush(engine)
return engine
def _result_blocks(text: str) -> list[str]:
return [b for b in text.split("\n") if b.startswith("[")]
def test_keyword_search_finds_step_and_carries_step_number(engine):
out = search_history_text(engine, query="verification code", recall_config=_cfg())
assert isinstance(out, str)
assert "Step 2" in out
assert "4711" in out
# Every result header names a step / step range / step anchor.
for header in _result_blocks(out):
assert re.search(r"Step[s]? \d", header), header
def test_step_range_filters_matches(engine):
out = search_history_text(engine, query="popup", step_range=[1, 2], recall_config=_cfg())
# Step 3 (the popup step) is outside the range: only the range ledger and
# a no-match line may appear.
assert "A promo popup appeared" not in out
assert "No matches" in out
def test_step_range_returns_full_width_ledger_rows(engine):
"""The compressed-history marker line's re-entry point: a step range
returns that range's full per-step action ledger."""
out = search_history_text(engine, query="", step_range=[3, 1], recall_config=_cfg())
assert "Action ledger for Steps 1–3" in out
# Full-width build_action_ledger rows: step number + T+mm:ss offset +
# semantic action + result phrase.
assert re.search(r"- Step 1 \(T\+\d{2}:\d{2}\): .*Login entry", out)
assert re.search(r"- Step 2 \(T\+\d{2}:\d{2}\): ", out)
assert re.search(r"- Step 3 \(T\+\d{2}:\d{2}\): ", out)
def test_result_count_is_clamped_by_config(engine):
for i in range(8):
engine.record_step(
summary=f"Scrolled the anchorterm feed page {i}.",
action_taken={"action": "swipe"},
last_execution_result={"status": "success"},
)
_flush(engine)
out = search_history_text(
engine, query="anchorterm", max_results=50, recall_config=_cfg(max_results=3)
)
assert out.count("[Step ") == 3
assert "more not shown" in out
def test_response_is_truncated_at_token_budget(engine):
for i in range(6):
engine.record_step(
summary=f"needleword page {i} " + "filler content " * 60,
action_taken={"action": "click"},
last_execution_result={"status": "success"},
)
_flush(engine)
out = search_history_text(
engine, query="needleword", max_results=6, recall_config=_cfg(max_text_tokens=150)
)
assert len(out) <= 150 * 4 + 200 # budget + truncation notice
assert "truncated at the search token budget" in out
def test_notes_are_searchable_and_anchored_to_a_step(engine):
save_note_content(engine.base_dir, "login_flow", "The OTP entry lives behind the SMS tab.")
out = search_history_text(engine, query="OTP entry", recall_config=_cfg())
assert "[Note 'login_flow'" in out
assert re.search(r"as of Step \d|last written at Step \d", out)
def test_chunk_rows_are_searchable(engine):
engine.record_history_chunk(
start_step_number=1,
end_step_number=2,
version=1,
status="ready",
band2=" - Steps 1–2: walked the chunkneedle flow",
band3="- Step 1 (T+00:01): click -> executed",
rendered_text=(
"[Chunk 1 | Steps 1–2]\n② Compressed step summary\n"
" - Steps 1–2: walked the chunkneedle flow\n"
"③ Step action ledger\n- Step 1 (T+00:01): click -> executed"
),
)
_flush(engine)
out = search_history_text(engine, query="chunkneedle", recall_config=_cfg())
assert "[History chunk | Steps 1–2" in out
def test_foreground_app_stamp_joins_search_surface(tmp_path):
"""A package-name query hits the record_step foreground_app stamp."""
engine = _make_engine(tmp_path)
engine.record_step(
summary="Opened the alarms list.",
action_taken={"action": "click", "target_text": "Alarm"},
ui_tree=[{"package": "com.google.android.deskclock", "bounds": "[0,0][9,9]"}],
)
_flush(engine)
out = search_history_text(engine, query="deskclock", recall_config=_cfg())
assert "Step 1" in out
assert "No matches" not in out
def test_tool_calls_of_a_step_are_searchable(engine):
engine.record_step(
summary="Asked the explorer.",
action_taken={"action": "click", "target_text": "Save"},
last_execution_result={"status": "success"},
)
engine.record_trace(
type="tool",
name="ask_explorer",
payload={"args": {"question": "toolneedle?"}, "result": "The resultneedle is ON."},
step_id=engine.last_recorded_step_id,
)
_flush(engine)
assert "[Step 4 " in search_history_text(engine, query="toolneedle", recall_config=_cfg())
assert "[Step 4 " in search_history_text(engine, query="resultneedle", recall_config=_cfg())
# --- Screen text: pre + post screenshots and referenced screenshots ----------------------
def test_post_screenshot_text_joins_the_search_surface(engine):
"""OCR text that only exists on the step's post-action screenshot is found."""
record = engine.get_step_record(2)
engine.storage.update_image_data(
record.post_image_name,
ocr_result=[{"text": "Welcome back postneedle", "bounds": [0, 0, 10, 10]}],
ui_tree=None,
)
out = search_history_text(engine, query="postneedle", recall_config=_cfg())
assert "[Step 2 " in out
def test_screen_scan_cap_limits_the_screenshot_sweep(engine):
record = engine.get_step_record(1)
engine.storage.update_image_data(
record.pre_image_name,
ocr_result=[{"text": "capneedle", "bounds": [0, 0, 10, 10]}],
ui_tree=None,
)
# Only the most recent step is scanned: Step 1's OCR is out of reach ...
out = search_history_text(engine, query="capneedle", recall_config=_cfg(screen_scan_steps=1))
assert "No matches" in out
# ... unless the range narrows the sweep onto it.
out = search_history_text(
engine, query="capneedle", step_range=[1, 1], recall_config=_cfg(screen_scan_steps=1)
)
assert "[Step 1 " in out
def test_embedded_screenshot_description_joins_the_search_surface(tmp_path):
"""A step whose tool result embedded an earlier screenshot is found by
that screenshot's OCR text: the image is described, not stripped."""
engine = _make_engine(tmp_path)
pre_bytes = _jpeg("red")
engine.record_step(
pre_screenshot_bytes=pre_bytes,
summary="Opened the alarms list.",
action_taken={"action": "click", "target_text": "Alarm"},
ocr_result=[{"text": "Alarm 07:30 ocrneedle", "bounds": [0, 0, 10, 10]}],
)
engine.record_step(
summary="Looked back at the alarm screen.",
action_taken={"action": "click", "target_text": "Save"},
)
engine.record_trace(
type="tool",
name="get_step_screenshot",
payload={
"args": {"step_number": 1, "which": "pre"},
"result": [
{"type": "text", "text": "Screenshot of step 1 (pre-action) is attached."},
{
"type": "image_url",
"image_url": {
"url": f"<ImageRef: sha256={hashlib.sha256(pre_bytes).hexdigest()} length=1>"
},
},
],
},
step_id=engine.last_recorded_step_id,
)
_flush(engine)
out = search_history_text(engine, query="ocrneedle", recall_config=_cfg())
assert "[Step 1 " in out
assert "[Step 2 " in out
assert "base64" not in out
def test_result_search_text_strips_image_blocks():
blocks = [
{"type": "text", "text": "hello needle"},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,QUJDRA=="}},
]
text = result_search_text(blocks)
assert "hello needle" in text
assert "base64" not in text and "QUJDRA" not in text
assert result_search_text({"answer": "yes"}) == '{"answer": "yes"}'
assert result_search_text(None) == ""
# --- Tool surface ---------------------------------------------------------------------
@pytest.mark.asyncio
async def test_execute_requires_query_or_range(engine):
tool = SearchHistoryTool()
out = await tool.execute(ctx=SimpleNamespace(data_engine=engine), query="", step_range=None)
assert "needs a query and/or a step_range" in out
@pytest.mark.asyncio
async def test_execute_searches_and_never_returns_images(engine):
out = await search_history.execute(
ctx=SimpleNamespace(data_engine=engine), query="verification code"
)
assert isinstance(out, str)
assert "Step 2" in out
@pytest.mark.asyncio
async def test_execute_without_engine_degrades():
out = await search_history.execute(ctx=SimpleNamespace(data_engine=None), query="x")
assert "no active execution history" in out
def test_availability_requires_engine_and_config():
assert search_history.is_available(None) is False
assert search_history.is_available(SimpleNamespace(data_engine=None)) is False
assert search_history.is_available(SimpleNamespace(data_engine=object())) is True
assert search_history_available(SimpleNamespace(data_engine=object())) is True
+1 -1
View File
@@ -89,7 +89,7 @@ class TestCommitteeTool(unittest.IsolatedAsyncioTestCase):
mock_response_tool_call = MagicMock()
mock_response_tool_call.tool_calls = [
{
"name": "get_step_details",
"name": "replay_steps",
"args": {"start_step": 1, "end_step": 1},
"id": "call_1",
}
-268
View File
@@ -1,268 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""recall_history boundaries and search correctness (M4, design §10).
Hard boundaries under test: bounded result count, bounded response tokens,
image recall limited to the real stored screenshots of a single step, and
every result carrying a step number. Plus: keyword/step-range/notes filter
correctness and the era-marker re-entry (a step_range returns the full-width
``build_action_ledger`` rows for that range).
"""
import io
import re
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from PIL import Image
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
from artemis.tools.history_recall import (
RecallHistoryTool,
recall_history,
search_history,
)
from artemis.utils.notes import save_note_content
def _jpeg(color: str) -> bytes:
buf = io.BytesIO()
Image.new("RGB", (64, 128), color).save(buf, format="JPEG")
return buf.getvalue()
def _make_engine(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
engine = DataEngine(mock_ctx)
engine.start_session("recall test session")
return engine
def _flush(engine):
for t in list(engine._pending_threads):
t.join()
def _cfg(**overrides):
values = {"enabled": True, "max_results": 5, "max_text_tokens": 2000, "max_image_steps": 1}
values.update(overrides)
return SimpleNamespace(**values)
@pytest.fixture
def engine(tmp_path):
engine = _make_engine(tmp_path)
engine.record_step(
pre_screenshot_bytes=_jpeg("red"),
summary="Opened the login page.",
action_taken={"action": "click", "target_text": "Login entry"},
last_execution_result={"status": "success"},
)
engine.record_step(
pre_screenshot_bytes=_jpeg("green"),
post_screenshot_bytes=_jpeg("blue"),
summary="Typed the verification code 4711 into the input.",
action_taken={"action": "input_text", "text": "4711"},
last_execution_result={"status": "success"},
operator_raw_thinking="The code arrived via SMS; entering 4711 now.",
)
engine.record_step(
summary="A promo popup appeared and was dismissed.",
action_taken={"action": "click", "target_text": "Close popup"},
last_execution_result={"status": "failed", "error": "element vanished"},
)
_flush(engine)
return engine
def _result_blocks(text: str) -> list[str]:
return [b for b in text.split("\n") if b.startswith("[")]
def test_keyword_search_finds_step_and_carries_step_number(engine):
out = search_history(engine, query="verification code", recall_config=_cfg())
assert isinstance(out, str)
assert "Step 2" in out
assert "4711" in out
# Every result header names a step / step range / step anchor.
for header in _result_blocks(out):
assert re.search(r"Step[s]? \d", header), header
def test_step_range_filters_matches(engine):
out = search_history(engine, query="popup", step_range=[1, 2], recall_config=_cfg())
# Step 3 (the popup step) is outside the range: only the range ledger and
# a no-match line may appear.
assert "A promo popup appeared" not in out
assert "No matches" in out
def test_step_range_returns_full_width_ledger_rows(engine):
"""The recall-only era period paragraph's '(Step-level ledger via
recall_history for steps a–b)' re-entry point."""
out = search_history(engine, query="", step_range=[1, 3], recall_config=_cfg())
assert "Action ledger for Steps 1–3" in out
# Full-width build_action_ledger rows: step number + T+mm:ss offset +
# semantic action + result phrase.
assert re.search(r"- Step 1 \(T\+\d{2}:\d{2}\): .*Login entry", out)
assert re.search(r"- Step 2 \(T\+\d{2}:\d{2}\): ", out)
assert re.search(r"- Step 3 \(T\+\d{2}:\d{2}\): ", out)
def test_result_count_is_clamped_by_config(engine):
for i in range(8):
engine.record_step(
summary=f"Scrolled the anchorterm feed page {i}.",
action_taken={"action": "swipe"},
last_execution_result={"status": "success"},
)
_flush(engine)
out = search_history(
engine, query="anchorterm", max_results=50, recall_config=_cfg(max_results=3)
)
assert out.count("[Step ") == 3
assert "more not shown" in out
def test_response_is_truncated_at_token_budget(engine):
engine.record_step(
summary="needleword " + "filler content " * 400,
action_taken={"action": "click"},
last_execution_result={"status": "success"},
)
_flush(engine)
out = search_history(
engine,
query="needleword",
include_details=True,
recall_config=_cfg(max_text_tokens=150),
)
assert len(out) <= 150 * 4 + 200 # budget + truncation notice
assert "truncated at the recall token budget" in out
def test_notes_are_searchable_and_anchored_to_a_step(engine):
save_note_content(engine.base_dir, "login_flow", "The OTP entry lives behind the SMS tab.")
out = search_history(engine, query="OTP entry", recall_config=_cfg())
assert "[Note 'login_flow'" in out
assert re.search(r"as of Step \d|last written at Step \d", out)
def test_chunk_rows_are_searchable(engine):
engine.record_history_chunk(
start_step_number=1,
end_step_number=2,
version=1,
status="ready",
band2=" - Steps 1–2: walked the chunkneedle flow",
band3="- Step 1 (T+00:01): click -> executed",
rendered_text=(
"[Chunk 1 | Steps 1–2]\n② Compressed step summary\n"
" - Steps 1–2: walked the chunkneedle flow\n"
"③ Step action ledger\n- Step 1 (T+00:01): click -> executed"
),
)
_flush(engine)
out = search_history(engine, query="chunkneedle", recall_config=_cfg())
assert "[History chunk | Steps 1–2" in out
def test_images_only_for_a_single_step_and_only_real_files(engine):
out = search_history(
engine,
query="verification code",
include_images=True,
recall_config=_cfg(),
)
assert isinstance(out, list)
images = [b for b in out if b.get("type") == "image_url"]
# Step 2 has a real pre and post screenshot on disk — but never more than
# one step's worth of images.
assert 1 <= len(images) <= 2
labels = [
b["text"]
for b in out
if b.get("type") == "text" and "screenshot of Step" in b.get("text", "")
]
assert all("Step 2" in label for label in labels)
for block in images:
assert block["image_url"]["url"].startswith("data:image/jpeg;base64,")
def test_images_disabled_by_config_cap(engine):
out = search_history(
engine,
query="verification code",
include_images=True,
recall_config=_cfg(max_image_steps=0),
)
assert isinstance(out, str)
def test_missing_image_files_yield_note_not_blocks(engine):
# Step 3 recorded no screenshots at all.
out = search_history(
engine,
query="promo popup",
include_images=True,
recall_config=_cfg(),
)
assert isinstance(out, list)
assert not [b for b in out if b.get("type") == "image_url"]
assert any(
"no stored screenshots exist" in b.get("text", "") for b in out if b.get("type") == "text"
)
@pytest.mark.asyncio
async def test_execute_requires_query_or_range(engine):
tool = RecallHistoryTool()
ctx = SimpleNamespace(data_engine=engine)
out = await tool.execute(ctx=ctx, query="", step_range=None)
assert "needs a query and/or a step_range" in out
@pytest.mark.asyncio
async def test_execute_without_engine_degrades():
tool = RecallHistoryTool()
out = await tool.execute(ctx=SimpleNamespace(data_engine=None), query="x")
assert "no active DataEngine session" in out
def test_availability_requires_engine_and_config():
assert recall_history.is_available(None) is False
assert recall_history.is_available(SimpleNamespace(data_engine=None)) is False
assert recall_history.is_available(SimpleNamespace(data_engine=object())) is True
def test_foreground_app_stamp_joins_search_surface(tmp_path):
"""M5: a package-name query hits the record_step foreground_app stamp."""
engine = _make_engine(tmp_path)
engine.record_step(
summary="Opened the alarms list.",
action_taken={"action": "click", "target_text": "Alarm"},
ui_tree=[{"package": "com.google.android.deskclock", "bounds": "[0,0][9,9]"}],
)
_flush(engine)
out = search_history(engine, query="deskclock", recall_config=_cfg())
assert "Step 1" in out
assert "No matches" not in out
+104
View File
@@ -0,0 +1,104 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Coordinate-space contract of recorded actions.
Pro records pixels (``coordinate_space="pixel"``), Flash records the model's
own 0–1000 values (``coordinate_space="normalized"``); every model-facing
render normalizes, and the explicit marker — never a magnitude heuristic —
is what makes that pass idempotent.
"""
from artemis.utils.coordinates import (
COORDINATE_SPACE_KEY,
COORDINATE_SPACE_NORMALIZED,
COORDINATE_SPACE_PIXEL,
normalize_action_dict,
normalize_any_structure,
)
W, H = 1080, 2400
def test_pixel_record_is_normalized_once_and_stamped():
pixel = {
"action": "tap",
"coordinates": [540, 1200],
"normalized_coordinates": [500, 500],
COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
}
once = normalize_action_dict(pixel, W, H)
assert once["coordinates"] == [500, 500]
assert once[COORDINATE_SPACE_KEY] == COORDINATE_SPACE_NORMALIZED
assert pixel["coordinates"] == [540, 1200] # input untouched
twice = normalize_action_dict(once, W, H)
assert twice == once
def test_normalized_record_is_returned_verbatim_even_when_values_look_like_pixels():
"""No magnitude guessing: a Flash record of [320, 399] stays [320, 399]
(the buggy double pass used to turn it into [296, 166])."""
flash = {
"action": "click",
"coordinates": [320, 399],
COORDINATE_SPACE_KEY: COORDINATE_SPACE_NORMALIZED,
"args": {"target": [320, 399]},
}
out = normalize_action_dict(flash, W, H)
assert out == flash
assert out is not flash
big = dict(flash, coordinates=[1080, 2400])
assert normalize_action_dict(big, W, H)["coordinates"] == [1080, 2400]
def test_legacy_unstamped_record_is_converted_once_then_idempotent():
legacy = {"action": "swipe", "coordinates": [540, 1800, 540, 600]}
once = normalize_action_dict(legacy, W, H)
assert once["coordinates"] == [500, 750, 500, 250]
assert once[COORDINATE_SPACE_KEY] == COORDINATE_SPACE_NORMALIZED
assert normalize_action_dict(once, W, H) == once
# A dict without coordinate fields gets no marker (nothing was converted).
assert COORDINATE_SPACE_KEY not in normalize_action_dict({"action": "press_key"}, W, H)
def test_normalize_any_structure_recurses_into_incident_containers():
"""An execution incident holds an ``action`` *dict* — it is a container,
not an action item, so its nested action is normalized too."""
report = {
"status": "failed",
"execution": [
{
"action": "tap",
"coordinates": [540, 1200],
COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
"attempts": ["Error"],
}
],
"incident": {
"kind": "exec_error",
"action": {
"action": "tap",
"coordinates": [540, 1200],
COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
},
},
}
out = normalize_any_structure(report, W, H)
assert out["execution"][0]["coordinates"] == [500, 500]
assert out["incident"]["action"]["coordinates"] == [500, 500]
assert out["incident"]["kind"] == "exec_error"
assert normalize_any_structure(out, W, H) == out
+130
View File
@@ -621,3 +621,133 @@ def test_get_recent_subgoal_hashes_robust_milestone_exclusion(tmp_path):
# 5. Older failed subgoals from previously completed milestones (hash_failed_old)
assert hash_failed_old not in keep_hashes
# --- Step replay (shared renderer for replay_steps / MCP view_step_details) ----------
def _replay_step(tool_result, **overrides):
step = {
"step_id": "step_4",
"step_number": 4,
"relative_time": "12.0s",
"summary": "Confirmed the Wi-Fi toggle state via the explorer.",
"action_taken": {"action": "click", "target": [0.5, 0.5], "target_text": "Save"},
"last_execution_result": {"status": "success"},
"interleaved_events": [
{"type": "thought", "content": "Need to confirm the toggle before saving."},
{
"type": "tool_call",
"name": "ask_explorer",
"args": {"question": "Is the Wi-Fi toggle on?"},
"result": tool_result,
},
{"type": "tool_call", "name": "click", "args": {"target": [0.5, 0.5]}, "result": "ok"},
],
}
step.update(overrides)
return step
def test_render_step_replay_shows_summary_thoughts_and_full_tool_results():
from artemis.utils.task_tree import render_step_replay
long_result = "The Wi-Fi toggle is ON and the SSID row reads 'HomeNet'. " * 60 # ~3.5k chars
out = render_step_replay(_replay_step(long_result))
assert "- **Step 4 (Start: 12.0s)**" in out
assert "[Screen]: Confirmed the Wi-Fi toggle state via the explorer." in out
assert "Need to confirm the toggle before saving." in out
# The tool call is replayed verbatim: name, args and the whole result.
assert "`ask_explorer(" in out
assert long_result in out
# Action tools are the planned action, not a tool-call line.
assert "[Planned Action]: Tapped 'Save' at [0.5, 0.5]" in out
assert "`click(" not in out
def test_render_step_replay_clamps_only_pathological_results():
from artemis.utils.task_tree import REPLAY_RESULT_CHARS, render_step_replay
huge = "y" * (REPLAY_RESULT_CHARS + 500)
out = render_step_replay(_replay_step(huge))
assert huge not in out
assert ("y" * REPLAY_RESULT_CHARS) + "..." in out
def test_live_window_keeps_the_tight_tool_result_clamp():
"""The Operator's live context window still clamps tool results tightly;
only the replay renderer is loose."""
from artemis.utils.task_tree import LIVE_RESULT_CHARS
long_result = "z" * 3000
out = build_plan_and_history(
"- [ ] Check toggle", [_replay_step(long_result)], "default", last_n_detailed=1
)
assert long_result not in out
assert ("z" * LIVE_RESULT_CHARS) + "..." in out
# No screen-description line in the live detailed view.
assert "[Screen]:" not in out
def test_render_step_replay_reports_missing_screen_description_status():
from artemis.utils.task_tree import render_step_replay
pending = render_step_replay(
_replay_step("ok", summary=None, extra_metadata={"summary_status": "pending"})
)
assert "[Screen]: (screen description pending)" in pending
failed = render_step_replay(
_replay_step("ok", summary="", extra_metadata={"summary_status": "failed"})
)
assert "[Screen]: (screen description unavailable)" in failed
assert "[Screen]" not in render_step_replay(_replay_step("ok", summary=None))
def test_render_step_replay_shows_described_images_in_tool_results():
from artemis.utils.task_tree import render_step_replay
result = [
{"type": "text", "text": "--- pre-action screenshot of Step 1 ---"},
{"type": "text", "text": "[screenshot: pre-action of Step 1]", "image_name": "ab" * 32},
]
out = render_step_replay(_replay_step(result))
assert "[screenshot: pre-action of Step 1]" in out
def test_format_action_clean_reads_flash_arguments_and_maps_manage_app():
"""Read Flash action arguments from ``args`` when formatting app launches."""
from artemis.utils.task_tree import format_action_clean
assert (
format_action_clean(
{
"action": "manage_app",
"coordinates": None,
"args": {"action": "launch", "app_name": "com.android.settings"},
}
)
== "Launched app 'com.android.settings'"
)
assert (
format_action_clean({"action": "manage_app", "args": {"action": "stop", "app_name": "x"}})
== "Stopped app 'x'"
)
# Pro's own shape keeps rendering as before.
assert format_action_clean({"action": "launch_app", "app_name": "x"}) == "Launched app 'x'"
assert (
format_action_clean({"action": "manage_app", "app_name": "x", "intent": "clear"})
== "Managed app 'x' (action: clear)"
)
# Top-level fields win over args; args only fill the gaps.
assert (
format_action_clean(
{
"action": "click",
"coordinates": [1, 2],
"target_text": "Save",
"args": {"target": [9, 9]},
}
)
== "Tapped 'Save' at [1, 2]"
)
+132
View File
@@ -0,0 +1,132 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Recording playlist manifest: session-relative segment offsets."""
import json
from unittest.mock import AsyncMock, patch
import pytest
from artemis.utils.video import RECORDING_MANIFEST_VERSION, write_recording_manifest
def _probe(durations: dict[str, float]):
async def probe(path):
return {"duration": durations[path.name], "width": 1080, "height": 1920}
return AsyncMock(side_effect=probe)
@pytest.mark.asyncio
async def test_manifest_carries_session_offsets_and_restart_gaps(tmp_path):
seg0 = tmp_path / "recording.mp4"
seg1 = tmp_path / "recording_001.mp4"
seg2 = tmp_path / "recording_002.mp4"
for path in (seg0, seg1, seg2):
path.write_bytes(b"mp4")
# Recording started 1.2s after the session; scrcpy restart gaps of 1.5s and 0.8s.
offsets = {seg0: 1.2, seg1: 12.7, seg2: 18.5}
with patch(
"artemis.utils.video.probe_video_segment",
_probe({seg0.name: 10.0, seg1.name: 5.0, seg2.name: 4.0}),
):
manifest_path = await write_recording_manifest(tmp_path, [seg0, seg1, seg2], offsets)
assert manifest_path == tmp_path / "recording.json"
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
assert payload["version"] == RECORDING_MANIFEST_VERSION
# Legacy back-to-back timeline is preserved for the existing player.
assert payload["duration"] == 19.0
assert [s["start"] for s in payload["segments"]] == [0.0, 10.0, 15.0]
assert [s["duration"] for s in payload["segments"]] == [10.0, 5.0, 4.0]
# Real session-relative timeline, including the gaps between segments.
assert [s["offset_ms"] for s in payload["segments"]] == [1200, 12700, 18500]
assert [s["duration_ms"] for s in payload["segments"]] == [10000, 5000, 4000]
assert payload["session_offset_ms"] == 1200
assert payload["session_end_ms"] == 22500
assert [s["file"] for s in payload["segments"]] == [seg0.name, seg1.name, seg2.name]
assert all(s["width"] == 1080 and s["height"] == 1920 for s in payload["segments"])
assert not (tmp_path / "recording.part.json").exists()
@pytest.mark.asyncio
async def test_manifest_without_offsets_falls_back_to_back_to_back(tmp_path):
seg0 = tmp_path / "recording.mp4"
seg1 = tmp_path / "recording_001.mp4"
seg0.write_bytes(b"mp4")
seg1.write_bytes(b"mp4")
with patch(
"artemis.utils.video.probe_video_segment",
_probe({seg0.name: 3.25, seg1.name: 2.0}),
):
manifest_path = await write_recording_manifest(tmp_path, [seg0, seg1])
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
assert [s["offset_ms"] for s in payload["segments"]] == [0, 3250]
assert [s["duration_ms"] for s in payload["segments"]] == [3250, 2000]
assert payload["session_offset_ms"] == 0
assert payload["session_end_ms"] == 5250
@pytest.mark.asyncio
async def test_manifest_fills_missing_offset_after_known_segment(tmp_path):
seg0 = tmp_path / "recording.mp4"
seg1 = tmp_path / "recording_001.mp4"
seg0.write_bytes(b"mp4")
seg1.write_bytes(b"mp4")
with patch(
"artemis.utils.video.probe_video_segment",
_probe({seg0.name: 10.0, seg1.name: 5.0}),
):
manifest_path = await write_recording_manifest(tmp_path, [seg0, seg1], {seg0: 2.0})
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
# Unknown offsets continue right after the previous segment (no gap assumed).
assert [s["offset_ms"] for s in payload["segments"]] == [2000, 12000]
@pytest.mark.asyncio
async def test_manifest_skips_invalid_segments_and_clamps_negative_offsets(tmp_path):
good = tmp_path / "recording.mp4"
broken = tmp_path / "recording_001.mp4"
missing = tmp_path / "recording_002.mp4"
good.write_bytes(b"mp4")
broken.write_bytes(b"mp4")
with patch(
"artemis.utils.video.probe_video_segment",
_probe({good.name: 4.0, broken.name: 0.0}),
):
manifest_path = await write_recording_manifest(
tmp_path, [good, broken, missing], {good: -0.4}
)
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
assert [s["file"] for s in payload["segments"]] == [good.name]
assert payload["segments"][0]["offset_ms"] == 0
@pytest.mark.asyncio
async def test_manifest_returns_none_without_valid_segments(tmp_path):
with patch(
"artemis.utils.video.probe_video_segment",
AsyncMock(return_value={"duration": 0, "width": 0, "height": 0}),
):
assert await write_recording_manifest(tmp_path, [tmp_path / "nope.mp4"]) is None
assert not (tmp_path / "recording.json").exists()