feat: unify history tools and improve execution reliability
This commit is contained in:
1 parent
d23a4f6472
commit
2fea9865ee
116 files changed
+8291
-2503
No files matched your search
@@ -14,6 +14,7 @@
|
||||
|
||||
"""Recording files of cancelled / killed workers must survive and be served whole."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
@@ -171,3 +172,61 @@ def test_recover_orphaned_recording_returns_none_without_files(tmp_path):
|
||||
missing = tmp_path / "missing" / "recording.mkv"
|
||||
assert MediaService.recover_orphaned_recording(str(missing)) is None
|
||||
assert MediaService.recover_orphaned_recording(None) is None
|
||||
|
||||
|
||||
def test_resolve_video_segments_passes_through_session_offsets(tmp_path, monkeypatch):
|
||||
folder = tmp_path / "artemis-traces" / "web_9_done"
|
||||
folder.mkdir(parents=True)
|
||||
(folder / "recording.mp4").write_bytes(b"seg0")
|
||||
(folder / "recording_001.mp4").write_bytes(b"seg1")
|
||||
(folder / "recording.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"version": 2,
|
||||
"duration": 15.0,
|
||||
"segments": [
|
||||
{
|
||||
"file": "recording.mp4",
|
||||
"start": 0.0,
|
||||
"duration": 10.0,
|
||||
"offset_ms": 1200,
|
||||
"duration_ms": 10000,
|
||||
"width": 1080,
|
||||
"height": 1920,
|
||||
},
|
||||
{
|
||||
"file": "recording_001.mp4",
|
||||
"start": 10.0,
|
||||
"duration": 5.0,
|
||||
"offset_ms": 12700,
|
||||
"duration_ms": 5000,
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
},
|
||||
{"file": "legacy.mp4", "start": 15.0, "duration": 1.0, "width": 1, "height": 1},
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(folder / "legacy.mp4").write_bytes(b"legacy")
|
||||
monkeypatch.setattr(media_module, "WORKSPACE_ROOT", tmp_path)
|
||||
|
||||
segments = MediaService.resolve_video_segments(
|
||||
"/videos/artemis-traces/web_9_done/recording.mp4"
|
||||
)
|
||||
|
||||
assert [s["url"] for s in segments] == [
|
||||
"/videos/artemis-traces/web_9_done/recording.mp4",
|
||||
"/videos/artemis-traces/web_9_done/recording_001.mp4",
|
||||
"/videos/artemis-traces/web_9_done/legacy.mp4",
|
||||
]
|
||||
assert [(s["start"], s["duration"]) for s in segments] == [
|
||||
(0.0, 10.0),
|
||||
(10.0, 5.0),
|
||||
(15.0, 1.0),
|
||||
]
|
||||
assert segments[0]["offset_ms"] == 1200 and segments[0]["duration_ms"] == 10000
|
||||
assert segments[1]["offset_ms"] == 12700 and segments[1]["duration_ms"] == 5000
|
||||
# Legacy entries without offsets stay offset-free so the UI falls back to back-to-back.
|
||||
assert "offset_ms" not in segments[2]
|
||||
@@ -45,20 +45,38 @@ def test_get_session_checks_reads_ledger_and_outcome(tmp_path, monkeypatch):
|
||||
json.dumps({"task_status": "completed", "tests": {"passed": 2, "failed": 0}}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
stream = {
|
||||
"attempt_id": "abc#1",
|
||||
"checkpoint_id": "abc",
|
||||
"phase": "checkpoint",
|
||||
"trace_id": "trace-1",
|
||||
"ts": 1.5,
|
||||
"truncated": False,
|
||||
"dropped_chars": 0,
|
||||
"segments": [
|
||||
{"execution_id": "e1", "role": "thought", "when": 1.1, "text": "Looking"},
|
||||
{"execution_id": "e1", "role": "answer", "when": 1.2, "text": "Seen"},
|
||||
],
|
||||
}
|
||||
(session_dir / "check_streams.jsonl").write_text(
|
||||
json.dumps(stream) + "\nnot json\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
result = MediaService.get_session_checks("sess-1")
|
||||
|
||||
assert [r["attempt_id"] for r in result["records"]] == ["abc#1", "final#1"]
|
||||
assert result["records"][0]["subgoal_text"] == "Create the alarm"
|
||||
assert result["run_outcome"]["task_status"] == "completed"
|
||||
assert result["streams"] == [stream]
|
||||
|
||||
|
||||
def test_get_session_checks_without_material(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(media_service_module, "TRACES_PATH", tmp_path)
|
||||
(tmp_path / "sess-2").mkdir()
|
||||
|
||||
assert MediaService.get_session_checks("sess-2") == {"records": [], "run_outcome": None}
|
||||
assert MediaService.get_session_checks("missing") == {"records": [], "run_outcome": None}
|
||||
empty = {"records": [], "streams": [], "run_outcome": None}
|
||||
assert MediaService.get_session_checks("sess-2") == empty
|
||||
assert MediaService.get_session_checks("missing") == empty
|
||||
|
||||
|
||||
def test_checks_route_is_registered():
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""``/api/sessions/{id}/usage``: token totals, live context and run tuning."""
|
||||
|
||||
import json
|
||||
|
||||
from apps.admin_console.database.connection import db_session
|
||||
from apps.admin_console.database.repositories.session_repository import SessionRepository
|
||||
|
||||
|
||||
def _insert_session(db_path, session_id, device_info):
|
||||
with db_session(db_path) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions (session_id, initial_goal, start_time, status, device_info)"
|
||||
" VALUES (?, ?, ?, ?, ?)",
|
||||
(session_id, "goal", 1.0, "completed", json.dumps(device_info)),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _insert_usage(db_path, session_id, trace_id, ts, payload, *, name="llm_usage"):
|
||||
with db_session(db_path) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO traces (trace_id, session_id, type, name, timestamp, status, payload)"
|
||||
" VALUES (?, ?, 'llm_call', ?, ?, 'success', ?)",
|
||||
(trace_id, session_id, name, ts, json.dumps(payload)),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _usage(prompt, completion, node=None, cached=0):
|
||||
payload = {
|
||||
"prompt_tokens": prompt,
|
||||
"completion_tokens": completion,
|
||||
"total_tokens": prompt + completion,
|
||||
"cached_tokens": cached,
|
||||
"context_base_tokens": prompt,
|
||||
}
|
||||
if node:
|
||||
payload["node"] = node
|
||||
return payload
|
||||
|
||||
|
||||
def test_usage_sums_all_agents_and_tracks_latest_operator_context(tmp_path):
|
||||
db_path = tmp_path / "sessions.db"
|
||||
repo = SessionRepository(db_path)
|
||||
_insert_session(
|
||||
db_path,
|
||||
"s1",
|
||||
{
|
||||
"profile": "pro",
|
||||
"run_tuning": {"verification_level": "checkpoints", "explorer_mode": "pro"},
|
||||
},
|
||||
)
|
||||
_insert_usage(db_path, "s1", "t1", 10.0, _usage(30_000, 500, node="planner"))
|
||||
_insert_usage(db_path, "s1", "t2", 11.0, _usage(120_000, 800, node="operator", cached=90_000))
|
||||
_insert_usage(db_path, "s1", "t3", 12.0, _usage(5_000, 100, node="checker"))
|
||||
_insert_usage(db_path, "s1", "t4", 13.0, _usage(150_000, 700, node="operator"))
|
||||
# A retry trace shares the type but is not a usage record.
|
||||
_insert_usage(db_path, "s1", "t5", 14.0, {"error": "503"}, name="llm_retry")
|
||||
|
||||
usage = repo.get_session_usage("s1")
|
||||
|
||||
assert usage["llm_calls"] == 4
|
||||
assert usage["prompt_tokens"] == 305_000
|
||||
assert usage["completion_tokens"] == 2_100
|
||||
assert usage["total_tokens"] == 307_100
|
||||
assert usage["cached_tokens"] == 90_000
|
||||
assert usage["operator_context_tokens"] == 150_000
|
||||
assert usage["operator_context_updated_at"] == 13.0
|
||||
assert usage["operator_context_window_tokens"] == 1_000_000
|
||||
assert usage["profile"] == "pro"
|
||||
assert usage["run_tuning"] == {"verification_level": "checkpoints", "explorer_mode": "pro"}
|
||||
|
||||
|
||||
def test_usage_flash_runner_counts_as_operator_and_has_no_tuning(tmp_path):
|
||||
db_path = tmp_path / "sessions.db"
|
||||
repo = SessionRepository(db_path)
|
||||
_insert_session(db_path, "s2", {"profile": "flash"})
|
||||
_insert_usage(db_path, "s2", "t1", 1.0, _usage(20_000, 200, node="FlashRunner"))
|
||||
|
||||
usage = repo.get_session_usage("s2")
|
||||
|
||||
assert usage["operator_context_tokens"] == 20_000
|
||||
assert usage["total_tokens"] == 20_200
|
||||
assert usage["profile"] == "flash"
|
||||
assert usage["run_tuning"] is None
|
||||
|
||||
|
||||
def test_usage_for_unknown_or_empty_session_is_zeroed(tmp_path):
|
||||
db_path = tmp_path / "sessions.db"
|
||||
repo = SessionRepository(db_path)
|
||||
|
||||
usage = repo.get_session_usage("missing")
|
||||
|
||||
assert usage["llm_calls"] == 0
|
||||
assert usage["total_tokens"] == 0
|
||||
assert usage["operator_context_tokens"] is None
|
||||
assert usage["profile"] is None
|
||||
assert usage["run_tuning"] is None
|
||||
|
||||
|
||||
def test_usage_route_is_registered():
|
||||
from apps.admin_console.routers import sessions
|
||||
|
||||
paths = {getattr(r, "path", None) for r in sessions.router.routes}
|
||||
assert "/api/sessions/{session_id}/usage" in paths
|
||||
@@ -173,8 +173,8 @@ def test_tool_table_probe_gate():
|
||||
names = {t.name for t in build_checker_tools(ctx, "checkpoint")}
|
||||
assert "probe_device" in names
|
||||
assert "read_note" in names
|
||||
assert "get_step_detail" in names
|
||||
assert "get_step_screenshot" in names
|
||||
# The shared history tools, mounted as a set.
|
||||
assert {"search_history", "replay_steps", "get_step_screenshot"} <= names
|
||||
# Never any device action, note write, or sub-agent
|
||||
assert not names & {
|
||||
"save_note",
|
||||
@@ -263,3 +263,309 @@ async def test_final_entry_captures_live_screen():
|
||||
)
|
||||
|
||||
capture_mock.assert_awaited_once()
|
||||
|
||||
|
||||
# --- Shared history tools in the check loop ------------------------------------------
|
||||
|
||||
|
||||
def _jpeg_file(tmp_path, color="white"):
|
||||
from PIL import Image
|
||||
|
||||
path = tmp_path / "pre.jpg"
|
||||
Image.new("RGB", (200, 400), color).save(path, format="JPEG")
|
||||
return path
|
||||
|
||||
|
||||
def _screenshot_then_verdict(provider, tool_call=None):
|
||||
"""LLM double: first turn asks for a step screenshot (or ``tool_call``),
|
||||
second turn answers."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
report = CheckReport(
|
||||
verdicts=[CheckVerdict(item_text="x", kind="verify", status="passed", evidence="e")]
|
||||
)
|
||||
structured = MagicMock()
|
||||
structured.ainvoke = AsyncMock(return_value=report)
|
||||
llm = MagicMock()
|
||||
llm.endpoint = SimpleNamespace(provider=provider)
|
||||
llm.with_structured_output.return_value = structured
|
||||
|
||||
ask = MagicMock()
|
||||
ask.tool_calls = [
|
||||
tool_call
|
||||
or {
|
||||
"name": "get_step_screenshot",
|
||||
"args": {"step_number": 2, "which": "pre"},
|
||||
"id": "tc-shot",
|
||||
}
|
||||
]
|
||||
done = MagicMock()
|
||||
done.tool_calls = []
|
||||
return llm, AsyncMock(side_effect=[ask, done])
|
||||
|
||||
|
||||
async def _run_with_screenshot(tmp_path, provider):
|
||||
path = _jpeg_file(tmp_path)
|
||||
ctx = _mock_ctx()
|
||||
ctx.data_engine.get_step_image_path.return_value = path
|
||||
llm, acomplete = _screenshot_then_verdict(provider)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.checker.checker.get_llm", return_value=llm),
|
||||
patch("artemis.agents.checker.checker.acomplete", new=acomplete),
|
||||
):
|
||||
report = await run_checkpoint_check(
|
||||
ctx,
|
||||
check_items=[_ci(text="x")],
|
||||
anchor=EvidenceAnchor(anchor_step_id="sid", trigger_ts=0.0, plan_text="- [x] G"),
|
||||
goal="the goal",
|
||||
subgoal_text="G",
|
||||
)
|
||||
ctx.data_engine.get_step_image_path.assert_called_once_with(2, "pre")
|
||||
# The message list handed to the second model call carries the tool result.
|
||||
messages = acomplete.call_args_list[1][0][1]
|
||||
return report, messages
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_step_screenshot_rides_inside_the_tool_message_for_gemini(tmp_path):
|
||||
from langchain_core.messages import HumanMessage, ToolMessage
|
||||
|
||||
report, messages = await _run_with_screenshot(tmp_path, "google")
|
||||
assert report.verdicts[0].status == "passed"
|
||||
|
||||
tool_msgs = [m for m in messages if isinstance(m, ToolMessage)]
|
||||
assert len(tool_msgs) == 1
|
||||
assert tool_msgs[0].tool_call_id == "tc-shot"
|
||||
assert tool_msgs[0].name == "get_step_screenshot"
|
||||
assert tool_msgs[0].status == "success"
|
||||
blocks = tool_msgs[0].content
|
||||
assert blocks[0]["text"] == "Screenshot of step 2 (pre-action) is attached."
|
||||
assert blocks[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
||||
# No extra human turn was spliced in for this provider.
|
||||
assert sum(isinstance(m, HumanMessage) for m in messages) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_step_screenshot_follows_as_a_human_message_for_openai(tmp_path):
|
||||
from langchain_core.messages import HumanMessage, ToolMessage
|
||||
|
||||
_, messages = await _run_with_screenshot(tmp_path, "openai")
|
||||
tool_msgs = [m for m in messages if isinstance(m, ToolMessage)]
|
||||
assert len(tool_msgs) == 1
|
||||
assert tool_msgs[0].content == "Screenshot of step 2 (pre-action) is attached."
|
||||
follower = messages[messages.index(tool_msgs[0]) + 1]
|
||||
assert isinstance(follower, HumanMessage)
|
||||
assert follower.content[0]["text"] == "[Screenshot returned by get_step_screenshot for step 2]"
|
||||
assert follower.content[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_replay_steps_result_is_a_plain_tool_message(tmp_path):
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
ctx = _mock_ctx()
|
||||
ctx.data_engine.get_agent_friendly_steps_in_range.return_value = [
|
||||
{
|
||||
"step_id": "s3",
|
||||
"step_number": 3,
|
||||
"relative_time": "9.0s",
|
||||
"summary": "Saved the alarm.",
|
||||
"action_taken": {"action": "click", "target": [0.5, 0.5], "target_text": "Save"},
|
||||
"last_execution_result": {"status": "success"},
|
||||
"interleaved_events": [],
|
||||
}
|
||||
]
|
||||
llm, acomplete = _screenshot_then_verdict(
|
||||
"google", {"name": "replay_steps", "args": {"start_step": 3}, "id": "tc-replay"}
|
||||
)
|
||||
with (
|
||||
patch("artemis.agents.checker.checker.get_llm", return_value=llm),
|
||||
patch("artemis.agents.checker.checker.acomplete", new=acomplete),
|
||||
):
|
||||
await run_checkpoint_check(
|
||||
ctx,
|
||||
check_items=[_ci(text="x")],
|
||||
anchor=EvidenceAnchor(anchor_step_id="sid", trigger_ts=0.0, plan_text="- [x] G"),
|
||||
goal="the goal",
|
||||
subgoal_text="G",
|
||||
)
|
||||
ctx.data_engine.get_agent_friendly_steps_in_range.assert_called_once_with(3, 3)
|
||||
messages = acomplete.call_args_list[1][0][1]
|
||||
tool_msg = next(m for m in messages if isinstance(m, ToolMessage))
|
||||
assert tool_msg.name == "replay_steps"
|
||||
assert "- **Step 3 (Start: 9.0s)**" in tool_msg.content
|
||||
assert "[Screen]: Saved the alarm." in tool_msg.content
|
||||
assert "[Planned Action]: Tapped 'Save' at [0.5, 0.5]" in tool_msg.content
|
||||
|
||||
|
||||
# --- Streamed-reasoning transcript (replay of the live Thought/Work text) -------------
|
||||
|
||||
|
||||
class _FakeBus:
|
||||
"""Minimal DataEngine bus: subscribers receive every published event."""
|
||||
|
||||
def __init__(self):
|
||||
self.subscribers = []
|
||||
|
||||
def subscribe(self, cb):
|
||||
self.subscribers.append(cb)
|
||||
|
||||
def unsubscribe(self, cb):
|
||||
self.subscribers.remove(cb)
|
||||
|
||||
def _publish(self, event_type, data):
|
||||
for cb in list(self.subscribers):
|
||||
cb(event_type, data)
|
||||
|
||||
|
||||
def _bus_ctx(tmp_path):
|
||||
"""Mock engine whose bus actually delivers events (the rest stays mocked)."""
|
||||
ctx = _mock_ctx()
|
||||
ctx.data_engine.base_dir = tmp_path
|
||||
bus = _FakeBus()
|
||||
ctx.data_engine.subscribers = bus.subscribers
|
||||
ctx.data_engine.subscribe = bus.subscribe
|
||||
ctx.data_engine.unsubscribe = bus.unsubscribe
|
||||
ctx.data_engine._publish = bus._publish
|
||||
return ctx
|
||||
|
||||
|
||||
def _stream(ctx, exec_id, chunk, *, stream_type="text", parent=None):
|
||||
from artemis.data_engine.context_vars import CURRENT_TRACE_ID
|
||||
|
||||
ctx.data_engine._publish(
|
||||
"llm_stream",
|
||||
{
|
||||
"execution_id": exec_id,
|
||||
"parent_trace_id": parent or str(CURRENT_TRACE_ID.get()),
|
||||
"chunk": chunk,
|
||||
"stream_type": stream_type,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _read_streams(tmp_path):
|
||||
from artemis.graph.checkpoints import read_attempt_streams
|
||||
|
||||
return read_attempt_streams(tmp_path)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_checker_persists_its_streamed_turns_as_timestamped_segments(tmp_path):
|
||||
"""Every llm_stream chunk emitted under the Checker's own trace is kept as
|
||||
one segment per (turn, kind) in first-chunk order; other agents' streams
|
||||
and a discarded (mid-stream reset) turn are left out."""
|
||||
ctx = _bus_ctx(tmp_path)
|
||||
|
||||
async def loop(ctx, messages, tools, items):
|
||||
_stream(ctx, "turn-1", "Looking at ", stream_type="thinking")
|
||||
_stream(ctx, "turn-1", "step 2.", stream_type="thinking")
|
||||
_stream(ctx, "turn-1", "Let me replay it.")
|
||||
_stream(ctx, "operator-turn", "not mine", parent="someone-else")
|
||||
_stream(ctx, "turn-2", "partial ")
|
||||
ctx.data_engine._publish(
|
||||
"llm_stream_reset", {"stream_exec_id": "turn-2", "action": "discard"}
|
||||
)
|
||||
_stream(ctx, "turn-3", "The alarm is there.")
|
||||
return CheckReport(verdicts=[])
|
||||
|
||||
with (
|
||||
patch("artemis.agents.checker.checker._run_check_loop", side_effect=loop),
|
||||
patch("artemis.agents.checker.checker.build_checker_tools", return_value=[]),
|
||||
):
|
||||
await run_checkpoint_check(
|
||||
ctx,
|
||||
check_items=[_ci(text="x")],
|
||||
anchor=EvidenceAnchor(anchor_step_id="sid", trigger_ts=0.0, plan_text="- [x] G"),
|
||||
goal="the goal",
|
||||
subgoal_text="G",
|
||||
attempt_id="abc#1",
|
||||
)
|
||||
|
||||
assert ctx.data_engine.subscribers == [] # detached after the run
|
||||
records = _read_streams(tmp_path)
|
||||
assert len(records) == 1
|
||||
rec = records[0]
|
||||
assert rec["attempt_id"] == "abc#1"
|
||||
assert rec["checkpoint_id"] == "abc"
|
||||
assert rec["phase"] == "checkpoint"
|
||||
assert rec["trace_id"]
|
||||
assert rec["truncated"] is False and rec["dropped_chars"] == 0
|
||||
assert isinstance(rec["ts"], float)
|
||||
segs = rec["segments"]
|
||||
assert [(s["execution_id"], s["role"], s["text"]) for s in segs] == [
|
||||
("turn-1", "thought", "Looking at step 2."),
|
||||
("turn-1", "answer", "Let me replay it."),
|
||||
("turn-3", "answer", "The alarm is there."),
|
||||
]
|
||||
assert all(isinstance(s["when"], float) for s in segs)
|
||||
assert segs[0]["when"] <= segs[1]["when"] <= segs[2]["when"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_entry_persists_transcript_even_when_the_loop_fails(tmp_path):
|
||||
"""The transcript is written in a ``finally``: an attempt that times out,
|
||||
raises or is superseded still leaves what it said."""
|
||||
ctx = _bus_ctx(tmp_path)
|
||||
|
||||
async def loop(ctx, messages, tools, items):
|
||||
_stream(ctx, "turn-1", "Checking the final screen…")
|
||||
raise RuntimeError("provider down")
|
||||
|
||||
with (
|
||||
patch("artemis.agents.checker.checker._run_check_loop", side_effect=loop),
|
||||
patch("artemis.agents.checker.checker.build_checker_tools", return_value=[]),
|
||||
patch(
|
||||
"artemis.agents.checker.checker._capture_final_screen",
|
||||
AsyncMock(return_value=(None, "")),
|
||||
),
|
||||
pytest.raises(RuntimeError),
|
||||
):
|
||||
await run_final_check(
|
||||
ctx,
|
||||
goal="g",
|
||||
plan_text="- [x] G",
|
||||
ledger=[],
|
||||
check_items=[_ci(text="x")],
|
||||
attempt_id="final#1",
|
||||
)
|
||||
|
||||
records = _read_streams(tmp_path)
|
||||
assert [r["attempt_id"] for r in records] == ["final#1"]
|
||||
assert records[0]["phase"] == "final" and records[0]["checkpoint_id"] == "final"
|
||||
assert records[0]["segments"][0]["text"] == "Checking the final screen…"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_transcript_without_attempt_id_or_streamed_text(tmp_path):
|
||||
ctx = _bus_ctx(tmp_path)
|
||||
|
||||
async def silent(ctx, messages, tools, items):
|
||||
return CheckReport(verdicts=[])
|
||||
|
||||
async def talkative(ctx, messages, tools, items):
|
||||
_stream(ctx, "turn-1", "text")
|
||||
return CheckReport(verdicts=[])
|
||||
|
||||
with patch("artemis.agents.checker.checker.build_checker_tools", return_value=[]):
|
||||
with patch("artemis.agents.checker.checker._run_check_loop", side_effect=talkative):
|
||||
await run_checkpoint_check(
|
||||
ctx,
|
||||
check_items=[_ci(text="x")],
|
||||
anchor=EvidenceAnchor(anchor_step_id="sid", trigger_ts=0.0, plan_text=""),
|
||||
goal="g",
|
||||
subgoal_text="G",
|
||||
)
|
||||
with patch("artemis.agents.checker.checker._run_check_loop", side_effect=silent):
|
||||
await run_checkpoint_check(
|
||||
ctx,
|
||||
check_items=[_ci(text="x")],
|
||||
anchor=EvidenceAnchor(anchor_step_id="sid", trigger_ts=0.0, plan_text=""),
|
||||
goal="g",
|
||||
subgoal_text="G",
|
||||
attempt_id="abc#2",
|
||||
)
|
||||
|
||||
assert _read_streams(tmp_path) == []
|
||||
assert not (tmp_path / "check_streams.jsonl").exists()
|
||||
@@ -138,3 +138,70 @@ def test_flash_runner_screenshot_pruning(mock_context):
|
||||
assert messages[2].content[0]["type"] == "text"
|
||||
assert messages[2].content[1]["type"] == "image_url"
|
||||
assert messages[2].content[1]["image_url"]["url"] == "data:image/jpeg;base64,SCREENSHOT_3"
|
||||
|
||||
|
||||
# --- Recorded action shape (coordinate space + ledger rendering) ---------------------
|
||||
|
||||
|
||||
def _record(runner, name, args):
|
||||
from types import SimpleNamespace
|
||||
|
||||
runner.ctx.data_engine = Mock()
|
||||
runner.ctx.data_engine.current_step_id = None
|
||||
runner.ctx.data_engine.record_step.return_value = "step-1"
|
||||
exec_result = SimpleNamespace(
|
||||
status="success", text_summary=f"{name} executed", metadata={}, ui_elements_text="ui"
|
||||
)
|
||||
step_id = runner._record_action_step(
|
||||
name, args, exec_result, "thought", {}, b"pre", ["xml"], b"post"
|
||||
)
|
||||
assert step_id == "step-1"
|
||||
return runner.ctx.data_engine.record_step.call_args.kwargs["action_taken"]
|
||||
|
||||
|
||||
def test_record_action_step_stamps_the_normalized_coordinate_space(mock_context):
|
||||
"""Flash stores the model's own 0–1000 target verbatim, marked as
|
||||
normalized, so the agent-friendly view never re-normalizes it."""
|
||||
from artemis.utils.coordinates import (
|
||||
COORDINATE_SPACE_KEY,
|
||||
COORDINATE_SPACE_NORMALIZED,
|
||||
normalize_any_structure,
|
||||
)
|
||||
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
recorded = _record(runner, "click", {"target": [320, 399]})
|
||||
assert recorded["coordinates"] == [320, 399]
|
||||
assert recorded["normalized_coordinates"] == [320, 399]
|
||||
assert recorded[COORDINATE_SPACE_KEY] == COORDINATE_SPACE_NORMALIZED
|
||||
assert normalize_any_structure(recorded, 1080, 2400)["coordinates"] == [320, 399]
|
||||
|
||||
|
||||
def test_flash_records_render_in_the_ledger_with_their_arguments(mock_context):
|
||||
"""The band-③ phrase reads the tool arguments Flash keeps under ``args``:
|
||||
app launches name the package, key presses the key, inputs the text,
|
||||
direction swipes the direction — never ``'None'``."""
|
||||
from artemis.utils.task_tree import format_actions_clean
|
||||
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
|
||||
launch = _record(runner, "manage_app", {"action": "launch", "app_name": "com.android.settings"})
|
||||
assert format_actions_clean(launch) == "Launched app 'com.android.settings'"
|
||||
stop = _record(runner, "manage_app", {"action": "stop", "app_name": "com.android.settings"})
|
||||
assert format_actions_clean(stop) == "Stopped app 'com.android.settings'"
|
||||
direct = _record(runner, "launch_app", {"app_name": "com.android.settings"})
|
||||
assert format_actions_clean(direct) == "Launched app 'com.android.settings'"
|
||||
|
||||
key = _record(runner, "press_key", {"key": "BACK"})
|
||||
assert format_actions_clean(key) == "Pressed key 'BACK'"
|
||||
|
||||
typed = _record(runner, "input_text", {"target": [500, 600], "text": "hello"})
|
||||
assert format_actions_clean(typed) == "Inputted 'hello' into field at [500, 600]"
|
||||
|
||||
swipe_dir = _record(runner, "swipe", {"direction": "up"})
|
||||
assert format_actions_clean(swipe_dir) == "Swiped up"
|
||||
swipe_fa = _record(runner, "swipe", {"action": "down"})
|
||||
assert format_actions_clean(swipe_fa) == "Swiped down"
|
||||
swipe_coords = _record(runner, "swipe", {"coordinates": [556, 289, 556, 124]})
|
||||
assert format_actions_clean(swipe_coords) == "Swiped from [556, 289] to [556, 124]"
|
||||
@@ -87,28 +87,55 @@ def test_video_analyzer_bound_only_when_recording_tools_enabled(mock_context):
|
||||
assert "video_analyzer" not in names
|
||||
|
||||
|
||||
def test_recall_history_bound_only_with_a_data_engine_session(mock_context):
|
||||
HISTORY_TOOLS = {"search_history", "replay_steps", "get_step_screenshot"}
|
||||
|
||||
|
||||
def test_history_tools_bound_only_with_a_data_engine_session(mock_context):
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
# No DataEngine: nothing to search, the tool stays out (as in Pro).
|
||||
# No DataEngine: nothing to read, the tools stay out (as in Pro).
|
||||
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
|
||||
assert "recall_history" not in names
|
||||
assert not (HISTORY_TOOLS & set(names))
|
||||
prompt = FlashRunner(mock_context, goal="g")._render_system_prompt(
|
||||
FlashRunner(mock_context, goal="g")._get_tools()
|
||||
)
|
||||
assert "search_history" not in prompt and "replay_steps" not in prompt
|
||||
|
||||
mock_context.data_engine = Mock()
|
||||
with patch("artemis.tools.history_recall._recall_config", return_value=None):
|
||||
with patch("artemis.tools.history._recall_config", return_value=None):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
tools = runner._get_tools()
|
||||
names = [t.name for t in tools]
|
||||
assert "recall_history" in names
|
||||
declaration = next(t for t in tools if t.name == "recall_history")
|
||||
assert set(declaration.parameters["properties"]) == {
|
||||
"query",
|
||||
"step_range",
|
||||
"include_details",
|
||||
"include_images",
|
||||
"max_results",
|
||||
}
|
||||
assert declaration.parameters["required"] == ["query"]
|
||||
assert "`recall_history`" in runner._render_system_prompt(tools)
|
||||
assert HISTORY_TOOLS <= set(names)
|
||||
assert names[-1] == "report_task_status"
|
||||
|
||||
# Same declarations as the LangChain tools: derived from one args schema.
|
||||
search = next(t for t in tools if t.name == "search_history")
|
||||
assert set(search.parameters["properties"]) == {"query", "step_range", "max_results"}
|
||||
assert search.parameters["required"] == []
|
||||
replay = next(t for t in tools if t.name == "replay_steps")
|
||||
assert set(replay.parameters["properties"]) == {"start_step", "end_step"}
|
||||
assert replay.parameters["required"] == ["start_step"]
|
||||
shot = next(t for t in tools if t.name == "get_step_screenshot")
|
||||
assert shot.parameters["properties"]["which"]["enum"] == ["pre", "post", "overlay"]
|
||||
|
||||
prompt = runner._render_system_prompt(tools)
|
||||
assert "`search_history`" in prompt
|
||||
assert "`replay_steps`" in prompt
|
||||
assert "`get_step_screenshot`" in prompt
|
||||
|
||||
|
||||
def test_search_history_alone_follows_the_recall_config_gate(mock_context):
|
||||
from types import SimpleNamespace
|
||||
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
mock_context.data_engine = Mock()
|
||||
with patch(
|
||||
"artemis.tools.history._recall_config",
|
||||
return_value=SimpleNamespace(enabled=False),
|
||||
):
|
||||
names = [t.name for t in FlashRunner(mock_context, goal="g")._get_tools()]
|
||||
assert "search_history" not in names
|
||||
assert {"replay_steps", "get_step_screenshot"} <= set(names)
|
||||
|
||||
|
||||
def test_video_analyzer_prompt_segment_follows_availability(mock_context):
|
||||
|
||||
@@ -95,7 +95,7 @@ async def test_hint_fires_on_old_matching_step():
|
||||
assert len(parts) == 1
|
||||
assert "Historical state hint" in parts[0]
|
||||
assert "Step 2" in parts[0]
|
||||
assert "recall_history" in parts[0]
|
||||
assert "search_history" in parts[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -108,12 +107,15 @@ async def test_history_analyzer_detailed_query_with_tool_call():
|
||||
},
|
||||
]
|
||||
mock_ctx.data_engine.get_agent_friendly_steps.return_value = steps
|
||||
mock_ctx.data_engine.get_agent_friendly_steps_in_range.side_effect = lambda s, e: [
|
||||
st for st in steps if s <= st["step_number"] <= e
|
||||
]
|
||||
|
||||
# Turn 1 LLM response: Tool call to get_step_details for step 2
|
||||
# Turn 1 LLM response: Tool call to replay_steps for step 2
|
||||
mock_response_turn_1 = MagicMock()
|
||||
mock_response_turn_1.content = ""
|
||||
mock_tool_call = {
|
||||
"name": "get_step_details",
|
||||
"name": "replay_steps",
|
||||
"args": {"start_step": 2, "end_step": 2},
|
||||
"id": "call_123456",
|
||||
}
|
||||
@@ -160,16 +162,14 @@ async def test_history_analyzer_detailed_query_with_tool_call():
|
||||
] # ToolMessage is second to last after final response is appended
|
||||
assert isinstance(tool_msg, ToolMessage)
|
||||
assert tool_msg.tool_call_id == "call_123456"
|
||||
assert tool_msg.name == "replay_steps"
|
||||
|
||||
# Parse the JSON in the ToolMessage content
|
||||
tool_result = json.loads(tool_msg.content)
|
||||
assert len(tool_result) == 1
|
||||
assert tool_result[0]["step_number"] == 2
|
||||
assert tool_result[0]["operator_raw_thinking"] == "Toggle the switch to turn wifi on"
|
||||
assert tool_result[0]["action_taken"] == {
|
||||
"action": "tap",
|
||||
"coordinates": [100, 200],
|
||||
}
|
||||
# The shared replay renderer: step header, screen description, thinking.
|
||||
mock_ctx.data_engine.get_agent_friendly_steps_in_range.assert_called_once_with(2, 2)
|
||||
assert "- **Step 2 (Start: 4.5s)**" in tool_msg.content
|
||||
assert "[Screen]: Toggled the wifi switch" in tool_msg.content
|
||||
assert "Toggle the switch to turn wifi on" in tool_msg.content
|
||||
assert "**Step 1" not in tool_msg.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -282,13 +282,14 @@ def test_history_analyzer_robust_tools_behavior():
|
||||
]
|
||||
|
||||
analyzer = HistoryAnalyzer(mock_ctx)
|
||||
mock_ctx.data_engine.get_agent_friendly_steps_in_range.return_value = steps
|
||||
|
||||
# 1. Test get_step_details with string inputs
|
||||
details_tool = analyzer._get_step_details_tool(steps)
|
||||
result_details = details_tool.invoke({"start_step": "1", "end_step": "1"})
|
||||
parsed_result = json.loads(result_details)
|
||||
assert len(parsed_result) == 1
|
||||
assert parsed_result[0]["step_number"] == 1
|
||||
# 1. The shared replay tool coerces string step numbers
|
||||
replay_tool = next(t for t in analyzer._build_tools() if t.name == "replay_steps")
|
||||
result_details = replay_tool.invoke({"start_step": "1", "end_step": "1"})
|
||||
mock_ctx.data_engine.get_agent_friendly_steps_in_range.assert_called_once_with(1, 1)
|
||||
assert "- **Step 1 (Start: 1.0s)**" in result_details
|
||||
assert "[Screen]: Action 1" in result_details
|
||||
|
||||
# 2. Test read_note with .md suffix
|
||||
from artemis.tools.scratchpad import get_read_note_tool_pure
|
||||
|
||||
@@ -113,6 +113,7 @@ async def test_perform_action_validation():
|
||||
{
|
||||
"action": "tap",
|
||||
"coordinates": [200, 300],
|
||||
"coordinate_space": "pixel",
|
||||
"normalized_coordinates": [185, 125],
|
||||
"times": 1,
|
||||
"delay_ms": 100,
|
||||
@@ -143,6 +144,7 @@ async def test_perform_action_validation():
|
||||
assert actions[0] == {
|
||||
"action": "focus_and_input_text",
|
||||
"coordinates": [400, 500],
|
||||
"coordinate_space": "pixel",
|
||||
"normalized_coordinates": [370, 208],
|
||||
"text": "hello",
|
||||
"clear_before_input": True,
|
||||
@@ -171,6 +173,7 @@ async def test_perform_action_validation():
|
||||
{
|
||||
"action": "tap",
|
||||
"coordinates": [400, 500],
|
||||
"coordinate_space": "pixel",
|
||||
"normalized_coordinates": [370, 208],
|
||||
"times": 1,
|
||||
"delay_ms": 100,
|
||||
@@ -263,6 +266,7 @@ async def test_bare_coordinate_click_enriched_by_hit_test():
|
||||
{
|
||||
"action": "tap",
|
||||
"coordinates": [540, 1440],
|
||||
"coordinate_space": "pixel",
|
||||
"normalized_coordinates": [500, 600],
|
||||
"times": 1,
|
||||
"delay_ms": 100,
|
||||
@@ -784,6 +788,7 @@ async def test_long_press_action_translation():
|
||||
{
|
||||
"action": "long_press_on",
|
||||
"coordinates": [200, 300],
|
||||
"coordinate_space": "pixel",
|
||||
"normalized_coordinates": [185, 125],
|
||||
"duration": 1500,
|
||||
"target_text": "Item 1",
|
||||
@@ -806,6 +811,7 @@ async def test_long_press_action_translation():
|
||||
{
|
||||
"action": "long_press_on",
|
||||
"coordinates": [540, 1440],
|
||||
"coordinate_space": "pixel",
|
||||
"normalized_coordinates": [500, 600],
|
||||
"duration": 1000,
|
||||
"target_text": None,
|
||||
|
||||
@@ -32,8 +32,12 @@ def test_operator_prompt_contract_matches_note_runtime_semantics():
|
||||
)
|
||||
assert "--- Execution Incident (OPEN) ---" in prompt
|
||||
assert "Failure Analyzer" not in prompt
|
||||
assert "(`read_note`, `list_notes`, `recall_history`)" in prompt
|
||||
assert "`save_note`, and `recall_history`" not in prompt
|
||||
assert (
|
||||
"(`read_note`, `list_notes`, `search_history`, `replay_steps`, `get_step_screenshot`)"
|
||||
in prompt
|
||||
)
|
||||
assert "recall_history" not in prompt
|
||||
assert "`save_note`, and `search_history`" not in prompt
|
||||
assert "Do NOT submit a Turn-Ending Action at the same time" not in prompt
|
||||
assert "memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`" not in prompt
|
||||
|
||||
|
||||
@@ -12,11 +12,11 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Operator dual prompt path (M2): legacy golden parity and transcript mode.
|
||||
"""Operator prompt snapshots and transcript mode.
|
||||
|
||||
Flag off (``agent.memory.transcript.enabled=false``): the legacy 2-message
|
||||
build must stay byte-for-byte identical — pinned by SHA-256 goldens recorded
|
||||
against the pre-split implementation.
|
||||
for the current template.
|
||||
|
||||
Flag on: ``_build_prompt`` renders ``S + F + A + tail`` from the session
|
||||
transcript ledger; the turn's messages are staged after the tool loop and
|
||||
@@ -51,20 +51,10 @@ from artemis.memory.transcript import (
|
||||
TranscriptLedger,
|
||||
)
|
||||
|
||||
# SHA-256 of the legacy system message rendered by TemplatePromptComponent
|
||||
# with the fixed inputs below. Originally recorded BEFORE the M2 template
|
||||
# split; re-recorded in M5 (2026-09-01) after the Short-Term Memory block was
|
||||
# removed, and again on 2026-09-02 when the Failure Analyzer was retired: the
|
||||
# operator.json template gained the Time-Sensitive Tasks principle, the two
|
||||
# execution tiers (vetted single action / fast-action burst), the Execution
|
||||
# Incident workflow section and the Phase 3 burst-discipline rule; and again
|
||||
# on 2026-09-02 after the prompt consolidation (every rule stated once, the
|
||||
# transcript/legacy history-marker split, the recited tool budget, the
|
||||
# recommended diagnosis triggers); and again on 2026-09-03 for the nested
|
||||
# plan-ledger contract and its validation gate. Any other drift means the
|
||||
# flag-off path is no longer byte-identical.
|
||||
GOLDEN_EMPTY_PLAN = "2ed35be503e3872f008ae5a12616af6f90cde65f23b63adbb6da782079ae4e7c"
|
||||
GOLDEN_SENTINEL_PLAN = "8bd6aa60cf3fad8d21cf5c284a01f586c57d209fcd1d2b4a517a791736f9b1de"
|
||||
# SHA-256 snapshots of the legacy system message with the fixed inputs below.
|
||||
# Update these when an intentional template change alters the rendered prompt.
|
||||
GOLDEN_EMPTY_PLAN = "dfcb15a164fa32172cf4bd2ab7b39409229a05dd1d5995eae26992784cfaa4f6"
|
||||
GOLDEN_SENTINEL_PLAN = "4b49a3c1ace6387df05a6f1f163dda007e4c2f8ff3d544dac12520317f17d21b"
|
||||
|
||||
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
|
||||
|
||||
@@ -166,13 +156,8 @@ def _no_action_llm(captured: list):
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_mode_two_turns_build_four_regions(tmp_path):
|
||||
ctx = _transcript_ctx()
|
||||
# Regression data shape (M2 cold-start bug, on-device A/B 2026-09-01):
|
||||
# turn 1 runs before any step record exists; by turn 2 the first step HAS
|
||||
# been recorded, so `steps` is non-empty while the previous turn is still
|
||||
# staged (turn_count == 0). The cold-start check must not read that
|
||||
# live-session state as a process restart — seeding would be rejected by
|
||||
# the empty-ledger invariant and silently demote every turn to the legacy
|
||||
# 2-message path.
|
||||
# Turn 2 has a recorded step but no committed ledger turn yet. This is
|
||||
# an active session, so it must not trigger cold-start history seeding.
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
ctx.data_engine.global_base_dir = str(tmp_path)
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock, mock_open, patch
|
||||
|
||||
from artemis.agents.outputter.tools import (
|
||||
get_search_history_tool,
|
||||
get_step_details_tool,
|
||||
get_step_screenshot_tool,
|
||||
)
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.global_base_dir = "/tmp/fake_traces"
|
||||
return ctx
|
||||
|
||||
|
||||
def test_get_step_details_tool():
|
||||
steps = [
|
||||
{
|
||||
"step_number": 1,
|
||||
"relative_time": "1s",
|
||||
"summary": "Step 1",
|
||||
"action_taken": "tap",
|
||||
},
|
||||
{
|
||||
"step_number": 2,
|
||||
"relative_time": "2s",
|
||||
"summary": "Step 2",
|
||||
"action_taken": "swipe",
|
||||
},
|
||||
]
|
||||
tool = get_step_details_tool(steps)
|
||||
|
||||
# Test valid range
|
||||
result = tool.invoke({"start_step": 1, "end_step": 1})
|
||||
parsed = json.loads(result)
|
||||
assert len(parsed) == 1
|
||||
assert parsed[0]["step_number"] == 1
|
||||
assert parsed[0]["action_taken"] == "tap"
|
||||
|
||||
# Test invalid range
|
||||
result_invalid = tool.invoke({"start_step": 3, "end_step": 4})
|
||||
assert "No steps found" in result_invalid
|
||||
|
||||
|
||||
def test_get_step_screenshot_tool(mock_context):
|
||||
steps = [
|
||||
{"step_number": 1, "pre_image_name": "img1", "post_image_name": "img2"},
|
||||
]
|
||||
tool = get_step_screenshot_tool(mock_context, steps)
|
||||
|
||||
# Mock open and exists
|
||||
with (
|
||||
patch("pathlib.Path.exists", return_value=True),
|
||||
patch("builtins.open", mock_open(read_data=b"fake_image_bytes")),
|
||||
):
|
||||
result = tool.invoke({"step_number": 1})
|
||||
assert isinstance(result, list)
|
||||
assert result[0]["type"] == "text"
|
||||
assert result[1]["type"] == "image_url"
|
||||
assert "data:image/jpeg;base64" in result[1]["image_url"]["url"]
|
||||
|
||||
|
||||
def test_search_history_tool(mock_context):
|
||||
tool = get_search_history_tool(mock_context)
|
||||
|
||||
# Mock steps returned by storage
|
||||
mock_step_1 = MagicMock()
|
||||
mock_step_1.step_number = 1
|
||||
mock_step_1.pre_image_name = "img1"
|
||||
mock_step_1.post_image_name = None
|
||||
|
||||
mock_context.data_engine.storage.get_steps.return_value = [mock_step_1]
|
||||
|
||||
# Mock search_ui_by_hash
|
||||
mock_context.data_engine.storage.search_ui_by_hash.return_value = [
|
||||
{"type": "ocr", "matched_text": "verification code: 1234", "score": 0.9}
|
||||
]
|
||||
|
||||
result = tool.invoke({"query": "verification"})
|
||||
parsed = json.loads(result)
|
||||
assert len(parsed) == 1
|
||||
assert parsed[0]["step_number"] == 1
|
||||
assert parsed[0]["matches"][0]["matched_text"] == "verification code: 1234"
|
||||
@@ -98,3 +98,6 @@ def test_prompt_rules_are_keyed_on_target_kind():
|
||||
assert "`Kind: coordinates only`" in prompt
|
||||
assert "hidden" in prompt
|
||||
assert "Identify what appears under the red dot" in prompt
|
||||
# Rule 1 judges the dot, not the screen: a control surviving elsewhere (or as
|
||||
# an active input field) does not make the tapped spot valid.
|
||||
assert "Judge the red dot, not the screen" in prompt
|
||||
@@ -85,7 +85,11 @@ def test_full_toolset_is_the_default():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer", "recall_history"))
|
||||
@pytest.mark.parametrize(
|
||||
"tool",
|
||||
OPERATOR_DEVICE_TOOLS
|
||||
+ ("video_analyzer", "search_history", "replay_steps", "get_step_screenshot"),
|
||||
)
|
||||
def test_removed_tool_leaves_no_reference(tool):
|
||||
"""The executable definition of 'an absent tool costs the model nothing'."""
|
||||
for name, template in load_operator_prompts().items():
|
||||
@@ -148,7 +152,10 @@ def test_full_set_enum_slots_render_verbatim():
|
||||
)
|
||||
assert "Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`)" in out
|
||||
assert "ADB/task tools (`run_adb_command`, `manage_task`)" in out
|
||||
assert "(`read_note`, `list_notes`, `recall_history`)" in out
|
||||
assert (
|
||||
"(`read_note`, `list_notes`, `search_history`, `replay_steps`, `get_step_screenshot`)"
|
||||
in out
|
||||
)
|
||||
# save_note is a write-through tool, never a result-dependent one; the on-demand
|
||||
# analyze_task_output is not advertised.
|
||||
assert "`save_note`, and" not in out
|
||||
|
||||
@@ -60,7 +60,14 @@ def mock_context(tmp_path):
|
||||
engine = Mock()
|
||||
engine.get_step_number.return_value = 7
|
||||
engine.get_step_record.return_value = SimpleNamespace(
|
||||
action_taken=[{"action": "tap", "coordinates": [500, 600], "target_text": "Search"}]
|
||||
action_taken=[
|
||||
{
|
||||
"action": "tap",
|
||||
"coordinates": [540, 1440],
|
||||
"coordinate_space": "pixel",
|
||||
"target_text": "Search",
|
||||
}
|
||||
]
|
||||
)
|
||||
engine.get_step_image_path.side_effect = lambda step_number, which="pre": (
|
||||
pre_path if which == "pre" else post_path
|
||||
@@ -84,7 +91,11 @@ async def test_summarizer_dispatches_visual_lens(mock_context):
|
||||
mock_context.step_memory.dispatch.assert_called_once_with(
|
||||
step_number=7,
|
||||
action_name="tap",
|
||||
action_args={"coordinates": [500, 600], "target_text": "Search"},
|
||||
action_args={
|
||||
"coordinates": [500, 600],
|
||||
"coordinate_space": "normalized",
|
||||
"target_text": "Search",
|
||||
},
|
||||
pre_img_bytes=b"PRE_IMAGE",
|
||||
post_img_bytes=b"POST_IMAGE",
|
||||
exec_outcome="success",
|
||||
@@ -123,8 +134,9 @@ async def test_summarizer_falls_back_to_structured_decisions(mock_context):
|
||||
|
||||
state = DummyState(
|
||||
structured_decisions=(
|
||||
'[{"action": "swipe", "coordinates": [1, 2, 3, 4]},'
|
||||
' {"action": "tap", "coordinates": [9, 9]}]'
|
||||
'[{"action": "swipe", "coordinates": [540, 1800, 540, 600],'
|
||||
' "coordinate_space": "pixel"},'
|
||||
' {"action": "tap", "coordinates": [540, 1200], "coordinate_space": "pixel"}]'
|
||||
),
|
||||
last_execution_result={"status": "success"},
|
||||
current_step_id="12345678-1234-5678-1234-567812345678",
|
||||
@@ -135,8 +147,10 @@ async def test_summarizer_falls_back_to_structured_decisions(mock_context):
|
||||
|
||||
kwargs = mock_context.step_memory.dispatch.call_args.kwargs
|
||||
assert kwargs["action_name"] == "swipe"
|
||||
assert kwargs["action_args"]["coordinates"] == [1, 2, 3, 4]
|
||||
assert kwargs["action_args"]["additional_actions"] == [{"action": "tap", "coordinates": [9, 9]}]
|
||||
assert kwargs["action_args"]["coordinates"] == [500, 750, 500, 250]
|
||||
assert kwargs["action_args"]["additional_actions"] == [
|
||||
{"action": "tap", "coordinates": [500, 500], "coordinate_space": "normalized"}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -28,6 +28,13 @@ from langchain_core.messages import AIMessage
|
||||
import pytest
|
||||
|
||||
from artemis.agents.validator.categories import ValidationErrorCategory
|
||||
from artemis.agents.validator.execution_loop import BURST_SKIPPED_MARKER
|
||||
from artemis.agents.validator.precondition_xml import (
|
||||
XML_STRONG_MIN_ELEMENTS,
|
||||
XML_VERDICT_STRONG,
|
||||
XML_VERDICT_WEAK,
|
||||
classify_xml_verdict,
|
||||
)
|
||||
from artemis.agents.validator.validator import ValidatorNode
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.mcp.action_types import ActionCode, ActionResult
|
||||
@@ -107,6 +114,28 @@ def temp_screenshot(tmp_path):
|
||||
return str(p)
|
||||
|
||||
|
||||
def _unrelated_hierarchy(count: int) -> list[dict]:
|
||||
"""``count`` labelled rows far below the target region (nothing matches it)."""
|
||||
return [
|
||||
{
|
||||
"text": f"Item {i}",
|
||||
"resource-id": f"row_{i}",
|
||||
"bounds": f"[0,{1000 + i * 100}][1080,{1080 + i * 100}]",
|
||||
"clickable": "true",
|
||||
}
|
||||
for i in range(count)
|
||||
]
|
||||
|
||||
|
||||
_LOGIN_TAP = {
|
||||
"action": "tap",
|
||||
"coordinates": [200, 250],
|
||||
"target_text": "Login",
|
||||
"target_bounds": [100, 200, 300, 300],
|
||||
"target_resource_id": "btn_login",
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_mcp():
|
||||
fake = FakeActionSession()
|
||||
@@ -175,17 +204,20 @@ async def test_validator_exec_error_opens_incident(mock_mcp, mock_context, temp_
|
||||
assert incident["reason"] == "Error: Element not found"
|
||||
assert incident["consecutive_failures"] == 1
|
||||
assert incident["burst_size"] == 1
|
||||
assert incident["action_description"] == "Tapped element at [105, 205]"
|
||||
# The Operator reads the incident: the pixel target [105, 205] is rendered
|
||||
# in its own 0-1000 space (1080x2400 frame).
|
||||
assert incident["action_description"] == "Tapped element at [97, 85]"
|
||||
# The same record rides in graph state for the Operator prompt.
|
||||
assert result["open_incident"] == incident
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_burst_skips_safety_net_and_aborts_on_first_failure(
|
||||
async def test_validator_burst_vets_first_member_only_and_aborts_on_first_failure(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Two or more actions form a fast-action burst: no precondition gate, a
|
||||
single dispatch per member, and the first failure aborts the rest."""
|
||||
"""Two or more actions form a fast-action burst: the precondition gate runs
|
||||
for the first member only, every member is dispatched once, and the first
|
||||
failure aborts the rest."""
|
||||
calls = []
|
||||
|
||||
def handler(name, args):
|
||||
@@ -213,9 +245,12 @@ async def test_validator_burst_skips_safety_net_and_aborts_on_first_failure(
|
||||
) as mock_gate,
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
|
||||
):
|
||||
mock_gate.return_value = (True, ValidationErrorCategory.NONE, "")
|
||||
result = await node(state)
|
||||
|
||||
mock_gate.assert_not_called()
|
||||
# Only the first member (decided on the still-live screen) is vetted.
|
||||
mock_gate.assert_called_once()
|
||||
assert mock_gate.call_args.args[2]["coordinates"] == [105, 205]
|
||||
|
||||
report = result["last_execution_result"]
|
||||
assert report["burst"] is True
|
||||
@@ -237,6 +272,42 @@ async def test_validator_burst_skips_safety_net_and_aborts_on_first_failure(
|
||||
assert result["open_incident"] == incident
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_burst_first_member_blocked_aborts_whole_burst(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""A refused first member opens a safety-net incident and skips the rest
|
||||
of the burst; nothing is dispatched to the device."""
|
||||
mock_mcp.hierarchy = _unrelated_hierarchy(XML_STRONG_MIN_ELEMENTS + 2)
|
||||
decisions = json.dumps([dict(_LOGIN_TAP), {"action": "press_key", "keycode": "ENTER"}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
node = ValidatorNode(mock_context)
|
||||
with (
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
|
||||
patch.object(
|
||||
ValidatorNode, "_validate_action_precondition_pixel", new_callable=AsyncMock
|
||||
) as mock_pixel,
|
||||
):
|
||||
result = await node(state)
|
||||
|
||||
report = result["last_execution_result"]
|
||||
assert report["burst"] is True
|
||||
assert report["status"] == "failed"
|
||||
assert report["execution"][0]["attempts"][0].startswith("Pre-execution validation failed")
|
||||
assert report["execution"][1]["attempts"] == [BURST_SKIPPED_MARKER]
|
||||
assert mock_mcp.calls_for("click") == []
|
||||
assert mock_mcp.calls_for("press_key") == []
|
||||
mock_pixel.assert_not_called()
|
||||
|
||||
incident = report["incident"]
|
||||
assert incident["kind"] == "safety_net"
|
||||
assert incident["category"] == ValidationErrorCategory.TARGET_DISAPPEARED.value
|
||||
assert incident["action_index"] == 0
|
||||
assert incident["burst_size"] == 2
|
||||
assert result["open_incident"] == incident
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_burst_success_executes_every_member(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
@@ -572,6 +643,126 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
|
||||
)
|
||||
|
||||
|
||||
def test_classify_xml_verdict_grades_strength():
|
||||
strong, why = classify_xml_verdict(XML_STRONG_MIN_ELEMENTS, "Login", None)
|
||||
assert (strong, why) == (XML_VERDICT_STRONG, "")
|
||||
assert classify_xml_verdict(XML_STRONG_MIN_ELEMENTS, None, "btn_login")[0] == XML_VERDICT_STRONG
|
||||
|
||||
weak, why = classify_xml_verdict(XML_STRONG_MIN_ELEMENTS - 1, "Login", "btn_login")
|
||||
assert weak == XML_VERDICT_WEAK and "sparse hierarchy" in why
|
||||
|
||||
weak, why = classify_xml_verdict(XML_STRONG_MIN_ELEMENTS, None, None)
|
||||
assert weak == XML_VERDICT_WEAK and "no resource-id/text" in why
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_strong_xml_missing_verdict_is_final_without_pixel_judge(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""A strong XML verdict blocks the action without consulting the pixel judge."""
|
||||
mock_mcp.hierarchy = _unrelated_hierarchy(XML_STRONG_MIN_ELEMENTS + 2)
|
||||
state = DummyState(
|
||||
structured_decisions=json.dumps([dict(_LOGIN_TAP)]), latest_screenshot=temp_screenshot
|
||||
)
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
patch.object(
|
||||
ValidatorNode, "_validate_action_precondition_pixel", new_callable=AsyncMock
|
||||
) as mock_pixel,
|
||||
):
|
||||
# A positive pixel verdict must not override strong XML evidence.
|
||||
mock_pixel.return_value = (True, ValidationErrorCategory.NONE, "")
|
||||
result = await node(state)
|
||||
|
||||
mock_pixel.assert_not_called()
|
||||
assert mock_mcp.calls_for("click") == []
|
||||
|
||||
incident = result["last_execution_result"]["incident"]
|
||||
assert incident["kind"] == "safety_net"
|
||||
assert incident["category"] == ValidationErrorCategory.TARGET_DISAPPEARED.value
|
||||
assert "was not found on the screen" in incident["reason"]
|
||||
assert "pixel judge not consulted" in incident["reason"]
|
||||
assert incident["evidence"]["xml_verdict"] == XML_VERDICT_STRONG
|
||||
assert incident["evidence"]["xml_element_count"] == XML_STRONG_MIN_ELEMENTS + 2
|
||||
assert incident["evidence"]["pixel_judge"] == "not consulted (strong XML verdict)"
|
||||
# The clause also rides in the action record the traces keep.
|
||||
assert (
|
||||
"pixel judge not consulted"
|
||||
in result["last_execution_result"]["execution"][0]["attempts"][0]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_weak_xml_sparse_tree_lets_pixel_judge_allow(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""A sparse hierarchy (WebView/game-like) cannot vouch for a missing target;
|
||||
the pixel judge decides, and a confident "present" lets the action run."""
|
||||
mock_mcp.hierarchy = _unrelated_hierarchy(2)
|
||||
state = DummyState(
|
||||
structured_decisions=json.dumps([dict(_LOGIN_TAP)]), latest_screenshot=temp_screenshot
|
||||
)
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
patch.object(
|
||||
ValidatorNode, "_validate_action_precondition_pixel", new_callable=AsyncMock
|
||||
) as mock_pixel,
|
||||
):
|
||||
mock_pixel.return_value = (True, ValidationErrorCategory.NONE, "")
|
||||
result = await node(state)
|
||||
|
||||
mock_pixel.assert_called_once()
|
||||
report = result["last_execution_result"]
|
||||
assert report["status"] == "success"
|
||||
assert report["incident"] is None
|
||||
assert len(mock_mcp.calls_for("click")) == 1
|
||||
assert "safety_net_evidence" not in report["execution"][0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_weak_xml_identifierless_target_pixel_judge_blocks(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Bare bounds from a hit test carry no identifiers to search for, so the
|
||||
XML "missing" is weak; the pixel judge is consulted and its refusal is
|
||||
recorded next to the XML verdict."""
|
||||
mock_mcp.hierarchy = _unrelated_hierarchy(XML_STRONG_MIN_ELEMENTS + 2)
|
||||
item = {
|
||||
"action": "tap",
|
||||
"coordinates": [200, 250],
|
||||
"target_class": "android.widget.ImageView",
|
||||
"target_bounds": [100, 200, 300, 300],
|
||||
"target_label_source": "hit_test",
|
||||
}
|
||||
state = DummyState(structured_decisions=json.dumps([item]), latest_screenshot=temp_screenshot)
|
||||
node = ValidatorNode(mock_context)
|
||||
|
||||
with (
|
||||
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
|
||||
patch.object(
|
||||
ValidatorNode, "_validate_action_precondition_pixel", new_callable=AsyncMock
|
||||
) as mock_pixel,
|
||||
):
|
||||
mock_pixel.return_value = (
|
||||
False,
|
||||
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
|
||||
"Pixel check failed",
|
||||
)
|
||||
result = await node(state)
|
||||
|
||||
mock_pixel.assert_called_once()
|
||||
incident = result["last_execution_result"]["incident"]
|
||||
assert incident["category"] == ValidationErrorCategory.TARGET_DISAPPEARED.value
|
||||
assert "XML evidence weak (target had no resource-id/text" in incident["reason"]
|
||||
assert "pixel judge did not confirm the target" in incident["reason"]
|
||||
assert incident["evidence"]["xml_verdict"] == XML_VERDICT_WEAK
|
||||
assert incident["evidence"]["pixel_judge"].startswith("consulted (weak XML verdict)")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_pixel_validation_success(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that the pixel safety net succeeds when Gemini reports target is present."""
|
||||
|
||||
@@ -438,3 +438,194 @@ def test_record_step_without_app_data_stamps_nothing(tmp_path):
|
||||
|
||||
step = engine.storage.get_steps(engine.current_session_id)[0]
|
||||
assert "foreground_app" not in (step.extra_metadata or {})
|
||||
|
||||
|
||||
# --- Agent-friendly single-step / range lookups (step replay backing) ----------------
|
||||
|
||||
|
||||
def _replay_engine(tmp_path):
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_execution_setup = MagicMock()
|
||||
mock_execution_setup.traces_path = str(tmp_path)
|
||||
mock_ctx.execution_setup = mock_execution_setup
|
||||
mock_ctx.device = None
|
||||
engine = DataEngine(mock_ctx)
|
||||
engine.start_session("replay lookup session")
|
||||
return engine
|
||||
|
||||
|
||||
def _join_pending(engine):
|
||||
for t in list(engine._pending_threads):
|
||||
t.join()
|
||||
|
||||
|
||||
def test_friendly_step_range_expands_tool_traces_and_matches_full_listing(tmp_path):
|
||||
engine = _replay_engine(tmp_path)
|
||||
engine.record_step(
|
||||
summary="Asked the explorer about the toggle.",
|
||||
action_taken={"action": "click", "target": [540, 1200], "target_text": "Save"},
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
engine.record_trace(
|
||||
type="tool",
|
||||
name="ask_explorer",
|
||||
payload={"args": {"question": "toggle?"}, "result": "The toggle is ON"},
|
||||
step_id=engine.last_recorded_step_id,
|
||||
)
|
||||
engine.record_step(summary="Second step.", action_taken={"action": "swipe"})
|
||||
engine.record_step(summary="Third step.", action_taken={"action": "press_key", "key": "BACK"})
|
||||
_join_pending(engine)
|
||||
|
||||
rows = engine.get_agent_friendly_steps_in_range(1, 2)
|
||||
assert [r["step_number"] for r in rows] == [1, 2]
|
||||
calls = rows[0]["tool_calls"]
|
||||
assert calls and calls[0]["name"] == "ask_explorer"
|
||||
assert calls[0]["args"] == {"question": "toggle?"}
|
||||
assert calls[0]["result"] == "The toggle is ON"
|
||||
assert any(e["type"] == "tool_call" for e in rows[0]["interleaved_events"])
|
||||
|
||||
# The single-step path renders exactly what the whole-session listing does
|
||||
# (same trace expansion, same coordinate normalization).
|
||||
full = engine.get_agent_friendly_steps()
|
||||
assert rows[0]["action_taken"] == full[0]["action_taken"]
|
||||
assert rows[0]["interleaved_events"] == full[0]["interleaved_events"]
|
||||
|
||||
# Reversed bounds are tolerated; a single step defaults end to start.
|
||||
assert [r["step_number"] for r in engine.get_agent_friendly_steps_in_range(3, 2)] == [2, 3]
|
||||
assert [r["step_number"] for r in engine.get_agent_friendly_steps_in_range(3)] == [3]
|
||||
|
||||
one = engine.get_agent_friendly_step(3)
|
||||
assert one is not None and one["summary"] == "Third step."
|
||||
assert engine.get_agent_friendly_step(99) is None
|
||||
|
||||
|
||||
def test_friendly_step_lookups_without_session_are_empty(tmp_path):
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_execution_setup = MagicMock()
|
||||
mock_execution_setup.traces_path = str(tmp_path)
|
||||
mock_ctx.execution_setup = mock_execution_setup
|
||||
mock_ctx.device = None
|
||||
engine = DataEngine(mock_ctx)
|
||||
assert engine.get_agent_friendly_steps_in_range(1, 5) == []
|
||||
assert engine.get_agent_friendly_step(1) is None
|
||||
|
||||
|
||||
def test_tool_result_images_are_described_as_step_screenshots(tmp_path):
|
||||
"""An image a tool result embedded (stored as an <ImageRef: sha256=…>
|
||||
placeholder) is rewritten into the description of the step screenshot it
|
||||
is, never dropped silently; unknown images get the generic label."""
|
||||
import hashlib
|
||||
import io
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from artemis.data_engine.engine import GENERIC_IMAGE_LABEL
|
||||
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (32, 64), "red").save(buf, format="JPEG")
|
||||
pre_bytes = buf.getvalue()
|
||||
pre_hash = hashlib.sha256(pre_bytes).hexdigest()
|
||||
|
||||
engine = _replay_engine(tmp_path)
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=pre_bytes,
|
||||
summary="Opened the alarms list.",
|
||||
action_taken={"action": "click"},
|
||||
ocr_result=[{"text": "Alarm 07:30 needleocr", "bounds": [0, 0, 10, 10]}],
|
||||
)
|
||||
engine.record_step(summary="Recalled the earlier screen.", action_taken={"action": "click"})
|
||||
engine.record_trace(
|
||||
type="tool",
|
||||
name="get_step_screenshot",
|
||||
payload={
|
||||
"args": {"step_number": 1, "which": "pre"},
|
||||
"result": [
|
||||
{"type": "text", "text": "Screenshot of step 1 (pre-action) is attached."},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"<ImageRef: sha256={pre_hash} length=99999>"},
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "<ImageRef: sha256=" + "0" * 64 + " length=5>"},
|
||||
},
|
||||
],
|
||||
},
|
||||
step_id=engine.last_recorded_step_id,
|
||||
)
|
||||
_join_pending(engine)
|
||||
|
||||
step2 = engine.get_agent_friendly_step(2)
|
||||
result = step2["tool_calls"][0]["result"]
|
||||
assert all(block["type"] == "text" for block in result)
|
||||
assert result[1]["text"] == "[screenshot: pre-action of Step 1]"
|
||||
assert result[1]["image_name"] == pre_hash
|
||||
assert result[2]["text"] == GENERIC_IMAGE_LABEL
|
||||
# Whole-session listing describes the same way.
|
||||
assert engine.get_agent_friendly_steps()[1]["tool_calls"][0]["result"] == result
|
||||
|
||||
|
||||
# --- Coordinate space of agent-friendly steps -------------------------
|
||||
|
||||
|
||||
def test_flash_normalized_record_survives_agent_friendly_steps_unchanged(tmp_path):
|
||||
"""Flash records the model's 0–1000 target with an explicit space marker;
|
||||
the agent-friendly view must not normalize it a second time."""
|
||||
from artemis.utils.coordinates import COORDINATE_SPACE_KEY, COORDINATE_SPACE_NORMALIZED
|
||||
|
||||
engine = _replay_engine(tmp_path)
|
||||
engine.record_step(
|
||||
action_taken={
|
||||
"action": "click",
|
||||
"coordinates": [320, 399],
|
||||
"normalized_coordinates": [320, 399],
|
||||
COORDINATE_SPACE_KEY: COORDINATE_SPACE_NORMALIZED,
|
||||
"args": {"target": [320, 399]},
|
||||
},
|
||||
last_execution_result={"status": "success", "result": "click executed"},
|
||||
)
|
||||
_join_pending(engine)
|
||||
|
||||
row = engine.get_agent_friendly_steps()[0]
|
||||
assert row["action_taken"]["coordinates"] == [320, 399]
|
||||
assert row["action_taken"][COORDINATE_SPACE_KEY] == COORDINATE_SPACE_NORMALIZED
|
||||
assert engine.get_agent_friendly_step(1)["action_taken"]["coordinates"] == [320, 399]
|
||||
|
||||
|
||||
def test_pro_pixel_record_is_normalized_exactly_once(tmp_path):
|
||||
"""Pro records physical pixels (with the frame size in extra_metadata);
|
||||
the friendly view converts them once, and the result is stamped so a
|
||||
downstream pass (MCP inspector, replay) leaves it alone."""
|
||||
from artemis.utils.coordinates import (
|
||||
COORDINATE_SPACE_KEY,
|
||||
COORDINATE_SPACE_NORMALIZED,
|
||||
COORDINATE_SPACE_PIXEL,
|
||||
normalize_any_structure,
|
||||
)
|
||||
|
||||
engine = _replay_engine(tmp_path)
|
||||
action = {
|
||||
"action": "tap",
|
||||
"coordinates": [540, 1200],
|
||||
"normalized_coordinates": [500, 500],
|
||||
COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
|
||||
"target_text": "Battery",
|
||||
}
|
||||
engine.record_step(
|
||||
action_taken=[action],
|
||||
last_execution_result={
|
||||
"status": "success",
|
||||
"execution": [dict(action, attempts=["Success"])],
|
||||
},
|
||||
extra_metadata={"width": 1080, "height": 2400},
|
||||
)
|
||||
_join_pending(engine)
|
||||
|
||||
row = engine.get_agent_friendly_steps()[0]
|
||||
friendly = row["action_taken"][0]
|
||||
assert friendly["coordinates"] == [500, 500]
|
||||
assert friendly[COORDINATE_SPACE_KEY] == COORDINATE_SPACE_NORMALIZED
|
||||
assert row["last_execution_result"]["execution"][0]["coordinates"] == [500, 500]
|
||||
|
||||
again = normalize_any_structure(row["action_taken"], 1080, 2400)
|
||||
assert again == row["action_taken"]
|
||||
@@ -0,0 +1,139 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""OfflineHistoryReader: the MCP inspector's read-only twin of the live engine.
|
||||
|
||||
The offline reader must produce exactly the records the live ``DataEngine``
|
||||
produces (same ``friendly_step``), open the database read-only, and satisfy
|
||||
the ``HistoryReader`` protocol the history tools are written against.
|
||||
"""
|
||||
|
||||
import io
|
||||
import sqlite3
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
from artemis.data_engine.history_reader import HistoryReader, OfflineHistoryReader
|
||||
|
||||
|
||||
def _jpeg(color: str) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (64, 128), color).save(buf, format="JPEG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _engine(tmp_path):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.execution_setup = MagicMock()
|
||||
ctx.execution_setup.traces_path = str(tmp_path)
|
||||
ctx.device = None
|
||||
engine = DataEngine(ctx)
|
||||
engine.start_session("offline reader test")
|
||||
return engine
|
||||
|
||||
|
||||
def _flush(engine):
|
||||
for t in list(engine._pending_threads):
|
||||
t.join()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def recorded(tmp_path):
|
||||
engine = _engine(tmp_path)
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=_jpeg("red"),
|
||||
post_screenshot_bytes=_jpeg("blue"),
|
||||
summary="Tapped Save after asking the explorer.",
|
||||
action_taken={"action": "click", "target": [540, 1200], "target_text": "Save"},
|
||||
last_execution_result={"status": "success"},
|
||||
operator_raw_thinking="Confirm the toggle first.",
|
||||
extra_metadata={"width": 1080, "height": 2400},
|
||||
)
|
||||
engine.record_trace(
|
||||
type="tool",
|
||||
name="ask_explorer",
|
||||
payload={"args": {"question": "toggle?"}, "result": "The toggle is ON."},
|
||||
step_id=engine.last_recorded_step_id,
|
||||
)
|
||||
engine.record_step(
|
||||
summary="Went back.",
|
||||
action_taken={"action": "press_key", "key": "back"},
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
engine.record_history_chunk(
|
||||
start_step_number=1,
|
||||
end_step_number=2,
|
||||
version=1,
|
||||
status="ready",
|
||||
band2=" - Steps 1–2: saved the toggle",
|
||||
band3="- Step 1 (T+00:00): click -> executed",
|
||||
rendered_text="[Chunk 1 | Steps 1–2]",
|
||||
)
|
||||
_flush(engine)
|
||||
return engine
|
||||
|
||||
|
||||
def _reader(engine) -> OfflineHistoryReader:
|
||||
return OfflineHistoryReader(engine.db_path, engine.global_base_dir, engine.current_session_id)
|
||||
|
||||
|
||||
def test_offline_reader_matches_live_friendly_steps(recorded):
|
||||
reader = _reader(recorded)
|
||||
assert isinstance(reader, HistoryReader)
|
||||
assert reader.session_start_time == recorded.session_start_time
|
||||
assert reader.base_dir == recorded.base_dir
|
||||
|
||||
live = recorded.get_agent_friendly_steps()
|
||||
offline = reader.get_agent_friendly_steps()
|
||||
assert offline == live
|
||||
assert offline[0]["tool_calls"][0]["name"] == "ask_explorer"
|
||||
# Coordinates normalized exactly like the live engine.
|
||||
assert offline[0]["action_taken"]["target"] == live[0]["action_taken"]["target"]
|
||||
assert offline[0]["relative_time"] == live[0]["relative_time"]
|
||||
|
||||
assert reader.get_agent_friendly_steps_in_range(2, 1) == live
|
||||
assert reader.get_agent_friendly_step(2) == live[1]
|
||||
assert reader.get_agent_friendly_step(9) is None
|
||||
|
||||
|
||||
def test_offline_reader_step_records_images_and_chunks(recorded):
|
||||
reader = _reader(recorded)
|
||||
record = reader.get_step_record(1)
|
||||
assert record is not None and record.step_number == 1
|
||||
assert reader.get_step_record(7) is None
|
||||
|
||||
pre = reader.get_step_image_path(1, "pre")
|
||||
post = reader.get_step_image_path(1, "post")
|
||||
assert pre is not None and pre.exists()
|
||||
assert post is not None and post.exists()
|
||||
assert pre == recorded.get_image_path(record.pre_image_name)
|
||||
assert reader.get_step_image_path(2, "pre") is None
|
||||
|
||||
chunks = reader.get_history_chunks()
|
||||
assert len(chunks) == 1
|
||||
assert chunks[0].start_step_number == 1
|
||||
|
||||
|
||||
def test_offline_reader_is_read_only(recorded):
|
||||
reader = _reader(recorded)
|
||||
with reader.storage._get_connection() as conn:
|
||||
with pytest.raises(sqlite3.OperationalError):
|
||||
conn.execute("DELETE FROM steps")
|
||||
# No directories or tables are created for a missing database.
|
||||
with pytest.raises(FileNotFoundError):
|
||||
OfflineHistoryReader(recorded.global_base_dir / "nope.db", recorded.global_base_dir, "x")
|
||||
@@ -702,3 +702,95 @@ async def test_exception_fail_open_releases_but_books_inconclusive(tmp_path):
|
||||
for r in read_ledger(tmp_path):
|
||||
assert r["status"] == "inconclusive"
|
||||
assert "checker blew up" in r["evidence"]
|
||||
|
||||
|
||||
# --- Attempt stream transcripts: bounded, append-only, replayable ------------------------
|
||||
|
||||
|
||||
def _seg(exec_id, text, role="answer", when=1.0):
|
||||
return {"execution_id": exec_id, "role": role, "when": when, "text": text}
|
||||
|
||||
|
||||
def test_clamp_keeps_short_transcripts_verbatim():
|
||||
from artemis.graph.checkpoints import clamp_stream_segments
|
||||
|
||||
segs = [_seg("a", "hello", "thought"), _seg("b", "world")]
|
||||
out, dropped = clamp_stream_segments(segs, limit=20)
|
||||
assert out == segs and dropped == 0
|
||||
assert out[0] is not segs[0] # copies, the input is never mutated
|
||||
|
||||
|
||||
def test_clamp_cuts_the_middle_once_and_keeps_head_and_tail():
|
||||
from artemis.graph.checkpoints import clamp_stream_segments
|
||||
|
||||
segs = [
|
||||
_seg("a", "A" * 40, "thought", when=1.0),
|
||||
_seg("b", "B" * 40, when=2.0),
|
||||
_seg("c", "C" * 40, when=3.0),
|
||||
_seg("d", "D" * 40, when=4.0),
|
||||
]
|
||||
out, dropped = clamp_stream_segments(segs, limit=100)
|
||||
|
||||
assert dropped == 60
|
||||
# head: 50 chars = all of a + 10 of b; tail: 50 chars = 10 of c + all of d
|
||||
assert [s["execution_id"] for s in out] == ["a", "b", "c", "d"]
|
||||
assert out[0]["text"] == "A" * 40 and out[0]["role"] == "thought"
|
||||
assert out[1]["text"] == "B" * 10 + "\n…[60 chars truncated]…\n"
|
||||
assert out[2]["text"] == "C" * 10
|
||||
assert out[3]["text"] == "D" * 40
|
||||
joined = "".join(s["text"] for s in out)
|
||||
assert joined.count("chars truncated") == 1
|
||||
assert len(joined) == 100 + len("\n…[60 chars truncated]…\n")
|
||||
# Timestamps and the other fields ride along untouched.
|
||||
assert [s["when"] for s in out] == [1.0, 2.0, 3.0, 4.0]
|
||||
|
||||
|
||||
def test_clamp_drops_segments_that_fall_entirely_into_the_cut():
|
||||
from artemis.graph.checkpoints import clamp_stream_segments
|
||||
|
||||
segs = [_seg("a", "A" * 10), _seg("mid", "M" * 100), _seg("z", "Z" * 10)]
|
||||
out, dropped = clamp_stream_segments(segs, limit=20)
|
||||
assert dropped == 100
|
||||
assert [s["execution_id"] for s in out] == ["a", "z"]
|
||||
assert out[0]["text"] == "A" * 10
|
||||
assert out[1]["text"] == "\n…[100 chars truncated]…\n" + "Z" * 10
|
||||
|
||||
|
||||
def test_clamp_single_huge_segment_keeps_both_ends():
|
||||
from artemis.graph.checkpoints import clamp_stream_segments
|
||||
|
||||
out, dropped = clamp_stream_segments([_seg("a", "H" * 30 + "M" * 100 + "T" * 30)], limit=60)
|
||||
assert dropped == 100
|
||||
assert len(out) == 1
|
||||
assert out[0]["text"] == "H" * 30 + "\n…[100 chars truncated]…\n" + "T" * 30
|
||||
|
||||
|
||||
def test_stream_records_round_trip_and_carry_truncation_flags(tmp_path):
|
||||
from artemis.graph.checkpoints import (
|
||||
STREAM_TEXT_LIMIT,
|
||||
append_attempt_stream_record,
|
||||
read_attempt_streams,
|
||||
streams_path,
|
||||
)
|
||||
|
||||
assert STREAM_TEXT_LIMIT == 20_000
|
||||
append_attempt_stream_record(
|
||||
tmp_path,
|
||||
{"attempt_id": "abc#1", "trace_id": "t1", "segments": [_seg("a", "short")]},
|
||||
)
|
||||
append_attempt_stream_record(
|
||||
tmp_path,
|
||||
{"attempt_id": "abc#2", "trace_id": "t2", "segments": [_seg("b", "x" * 50)]},
|
||||
limit=10,
|
||||
)
|
||||
with streams_path(tmp_path).open("a", encoding="utf-8") as f:
|
||||
f.write("not json\n\n")
|
||||
|
||||
records = read_attempt_streams(tmp_path)
|
||||
assert [r["attempt_id"] for r in records] == ["abc#1", "abc#2"]
|
||||
assert records[0]["truncated"] is False and records[0]["dropped_chars"] == 0
|
||||
assert records[0]["segments"] == [_seg("a", "short")]
|
||||
assert isinstance(records[0]["ts"], float)
|
||||
assert records[1]["truncated"] is True and records[1]["dropped_chars"] == 40
|
||||
assert records[1]["segments"][0]["text"] == "x" * 5 + "\n…[40 chars truncated]…\n" + "x" * 5
|
||||
assert read_attempt_streams(tmp_path / "nowhere") == []
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Helper-tool routing in McpActionExecutor (Flash profile).
|
||||
|
||||
The Flash runner binds ``ask_explorer``, ``video_analyzer`` and
|
||||
``recall_history`` declarations next to its device actions; the executor must
|
||||
The Flash runner binds ``ask_explorer``, ``video_analyzer`` and the history
|
||||
tools (``search_history`` / ``replay_steps`` / ``get_step_screenshot``) next to its device actions; the executor must
|
||||
route them to the same implementations the Pro operator's LangChain tools
|
||||
delegate to, as agent-side text tools (no screenshot capture, no device
|
||||
action), and report their status explicitly instead of sniffing the text for
|
||||
@@ -38,7 +38,13 @@ def _state():
|
||||
|
||||
def test_helper_tools_are_not_device_actions():
|
||||
executor = _make_executor()
|
||||
assert {"ask_explorer", "video_analyzer", "recall_history"} <= AGENT_TOOL_NAMES
|
||||
assert {
|
||||
"ask_explorer",
|
||||
"video_analyzer",
|
||||
"search_history",
|
||||
"replay_steps",
|
||||
"get_step_screenshot",
|
||||
} <= AGENT_TOOL_NAMES
|
||||
assert not (AGENT_TOOL_NAMES & executor.action_tool_names)
|
||||
|
||||
|
||||
@@ -231,22 +237,32 @@ async def test_video_analyzer_exception_is_contained():
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# recall_history
|
||||
# history tools (search_history / replay_steps / get_step_screenshot)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _history_tool_double(return_value=None, side_effect=None):
|
||||
from artemis.tools.history import SearchHistoryArgs
|
||||
|
||||
tool = Mock()
|
||||
tool.args_schema = SearchHistoryArgs
|
||||
tool.execute = AsyncMock(return_value=return_value, side_effect=side_effect)
|
||||
return tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recall_history_routes_to_the_shared_tool_with_text_result():
|
||||
async def test_history_tool_routes_to_the_shared_tool_with_text_result():
|
||||
executor = _make_executor()
|
||||
with patch("artemis.tools.history_recall.recall_history") as tool:
|
||||
tool.execute = AsyncMock(return_value="- Step 3 (T+00:30): tap btn3 -> executed")
|
||||
tool = _history_tool_double(return_value="- Step 3 (T+00:30): tap btn3 -> executed")
|
||||
with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool) as by_name:
|
||||
result = await executor.execute(
|
||||
"recall_history",
|
||||
"search_history",
|
||||
{"query": "login Error timeout", "step_range": [1, 5], "unknown_arg": 1},
|
||||
"tc-recall",
|
||||
"tc-search",
|
||||
None,
|
||||
)
|
||||
|
||||
by_name.assert_called_once_with("search_history")
|
||||
tool.execute.assert_awaited_once_with(
|
||||
ctx=executor.ctx, query="login Error timeout", step_range=[1, 5]
|
||||
)
|
||||
@@ -257,27 +273,40 @@ async def test_recall_history_routes_to_the_shared_tool_with_text_result():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recall_history_forwards_multimodal_blocks():
|
||||
async def test_history_tool_forwards_multimodal_blocks():
|
||||
executor = _make_executor()
|
||||
blocks = [
|
||||
{"type": "text", "text": "Step 4 matched."},
|
||||
{"type": "text", "text": "Screenshot of step 4 (pre-action) is attached."},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,AAA"}},
|
||||
]
|
||||
with patch("artemis.tools.history_recall.recall_history") as tool:
|
||||
tool.execute = AsyncMock(return_value=blocks)
|
||||
result = await executor.execute(
|
||||
"recall_history", {"query": "settings", "include_images": True}, "tc", None
|
||||
)
|
||||
tool = _history_tool_double(return_value=blocks)
|
||||
with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool):
|
||||
result = await executor.execute("get_step_screenshot", {"query": "unused"}, "tc", None)
|
||||
assert result.status == "success"
|
||||
assert result.text_summary == "Step 4 matched."
|
||||
assert result.text_summary == "Screenshot of step 4 (pre-action) is attached."
|
||||
assert result.raw_result == blocks
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recall_history_exception_is_contained():
|
||||
async def test_history_tool_exception_is_contained():
|
||||
executor = _make_executor()
|
||||
with patch("artemis.tools.history_recall.recall_history") as tool:
|
||||
tool.execute = AsyncMock(side_effect=RuntimeError("db gone"))
|
||||
result = await executor.execute("recall_history", {"query": "x"}, "tc", None)
|
||||
tool = _history_tool_double(side_effect=RuntimeError("db gone"))
|
||||
with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool):
|
||||
result = await executor.execute("replay_steps", {"query": "x"}, "tc", None)
|
||||
assert result.status == "success" # contained: the answer explains the failure
|
||||
assert result.text_summary == "recall_history failed: db gone"
|
||||
assert result.text_summary == "replay_steps failed: db gone"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_tools_dispatch_to_the_real_shared_instances():
|
||||
"""No per-tool branch: every history tool name resolves through the package."""
|
||||
executor = _make_executor() # ctx.data_engine is None -> tools degrade to text
|
||||
for name in ("search_history", "replay_steps", "get_step_screenshot"):
|
||||
result = await executor.execute(
|
||||
name, {"query": "x", "start_step": 1, "step_number": 1}, "tc", None
|
||||
)
|
||||
assert result.status == "success"
|
||||
assert (
|
||||
"no active execution history" in result.text_summary
|
||||
or "no execution history" in result.text_summary
|
||||
)
|
||||
@@ -77,6 +77,10 @@ def test_tool_signatures():
|
||||
assert "action" in sig_inspect.parameters
|
||||
assert "trace_id" in sig_inspect.parameters
|
||||
assert "step_number" in sig_inspect.parameters
|
||||
assert "query" in sig_inspect.parameters
|
||||
assert "step_range" in sig_inspect.parameters
|
||||
assert sig_inspect.parameters["max_results"].default == 5
|
||||
assert "'search'" in (mobile_inspect_trace.__doc__ or "")
|
||||
|
||||
|
||||
def test_mobile_run_task_invalid_model():
|
||||
@@ -501,3 +505,152 @@ def test_mobile_manage_task_stop_via_daemon(temp_trace_env, monkeypatch):
|
||||
mock_stop.assert_called_once_with(trace_id)
|
||||
assert res["status"] == "cancelled"
|
||||
assert trace_store.read_status(trace_id)["status"] == "cancelled"
|
||||
|
||||
|
||||
def _seed_trace(temp_dir: str, trace_id: str, *, with_image: bool = True) -> str:
|
||||
"""Writes one Flash-style step (with a tool trace and a pre screenshot)
|
||||
through the real storage layer, exactly as a live run would."""
|
||||
import io
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from artemis.data_engine.models import (
|
||||
ImageRecord,
|
||||
SessionMetadata,
|
||||
StepRecord,
|
||||
TraceRecord,
|
||||
)
|
||||
from artemis.data_engine.storage import StorageManager
|
||||
|
||||
storage = StorageManager(os.path.join(temp_dir, "data_engine.db"), temp_dir)
|
||||
storage.create_session(
|
||||
SessionMetadata(session_id=trace_id, initial_goal="Replay test", start_time=1000.0)
|
||||
)
|
||||
image_name = None
|
||||
if with_image:
|
||||
image_name = "a" * 64
|
||||
images_dir = os.path.join(temp_dir, "images")
|
||||
os.makedirs(images_dir, exist_ok=True)
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (1080, 2400), "white").save(buf, format="JPEG")
|
||||
with open(os.path.join(images_dir, f"{image_name}.jpg"), "wb") as f:
|
||||
f.write(buf.getvalue())
|
||||
storage.create_image(
|
||||
ImageRecord(
|
||||
image_name=image_name,
|
||||
ocr_result=[{"text": "Save ocrneedle", "bounds": [0, 0, 10, 10]}],
|
||||
)
|
||||
)
|
||||
step_id = str(uuid.uuid4())
|
||||
storage.create_step(
|
||||
StepRecord(
|
||||
step_id=step_id,
|
||||
session_id=trace_id,
|
||||
step_number=1,
|
||||
timestamp=1012.0,
|
||||
pre_image_name=image_name,
|
||||
summary="Asked the explorer, then tapped Save.",
|
||||
action_taken={"action": "click", "target": [540, 1200], "target_text": "Save"},
|
||||
operator_raw_thinking="Confirm the toggle first.",
|
||||
last_execution_result={"status": "success"},
|
||||
extra_metadata={"width": 1080, "height": 2400},
|
||||
)
|
||||
)
|
||||
storage.create_trace(
|
||||
TraceRecord(
|
||||
trace_id=str(uuid.uuid4()),
|
||||
session_id=trace_id,
|
||||
step_id=step_id,
|
||||
type="tool",
|
||||
name="ask_explorer",
|
||||
status="success",
|
||||
timestamp=1011.0,
|
||||
duration=0.8,
|
||||
payload={"args": {"question": "toggle?"}, "result": "The toggle is ON. " * 100},
|
||||
)
|
||||
)
|
||||
return step_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mobile_inspect_trace_step_details_replays_tool_calls_for_flash(temp_trace_env):
|
||||
"""view_step_details works for Flash runs too and replays the recorded
|
||||
tool calls through the same friendly-step path the live DataEngine uses."""
|
||||
trace_id = str(uuid.uuid4())
|
||||
trace_store.init_trace(trace_id, "Replay test", "Flash", "conv-1", device_serial="pixel-7")
|
||||
_seed_trace(temp_trace_env, trace_id)
|
||||
|
||||
res = await mobile_inspect_trace(action="view_step_details", trace_id=trace_id, step_number=1)
|
||||
assert "error" not in res
|
||||
assert res["device_serial"] == "pixel-7"
|
||||
details = res["details"]
|
||||
assert "- **Step 1 (Start: 12.0s)**" in details
|
||||
assert "[Screen]: Asked the explorer, then tapped Save." in details
|
||||
assert "Confirm the toggle first." in details
|
||||
assert "`ask_explorer(" in details
|
||||
assert "The toggle is ON. " * 100 in details
|
||||
# Coordinates are replayed normalized, not as stored pixels.
|
||||
assert "[540, 1200]" not in details
|
||||
assert "Tapped 'Save'" in details
|
||||
|
||||
missing = await mobile_inspect_trace(
|
||||
action="view_step_details", trace_id=trace_id, step_number=9
|
||||
)
|
||||
assert missing["error"] == "Step not found"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mobile_inspect_trace_step_screenshots_draw_the_overlay(temp_trace_env):
|
||||
trace_id = str(uuid.uuid4())
|
||||
trace_store.init_trace(trace_id, "Shot test", "Flash", "conv-1", device_serial="pixel-7")
|
||||
_seed_trace(temp_trace_env, trace_id)
|
||||
|
||||
res = await mobile_inspect_trace(
|
||||
action="view_step_screenshots", trace_id=trace_id, step_number=1
|
||||
)
|
||||
assert "error" not in res
|
||||
assert res["before_screenshot"].startswith("file://")
|
||||
assert res["before_screenshot"].endswith(f"{'a' * 64}.jpg")
|
||||
assert res["after_screenshot"] is None
|
||||
overlay = res["action_overlay_screenshot"]
|
||||
assert overlay is not None
|
||||
overlay_path = overlay[len("file://") :]
|
||||
assert os.path.exists(overlay_path)
|
||||
assert overlay_path.endswith("step_1_overlay.jpg")
|
||||
# The overlay differs from the plain pre screenshot: the action was drawn.
|
||||
with open(overlay_path, "rb") as f, open(res["before_screenshot"][7:], "rb") as g:
|
||||
assert f.read() != g.read()
|
||||
|
||||
# Second call reuses the file it already wrote.
|
||||
again = await mobile_inspect_trace(
|
||||
action="view_step_screenshots", trace_id=trace_id, step_number=1
|
||||
)
|
||||
assert again["action_overlay_screenshot"] == overlay
|
||||
|
||||
no_step = await mobile_inspect_trace(
|
||||
action="view_step_screenshots", trace_id=trace_id, step_number=None
|
||||
)
|
||||
assert no_step["error"] == "Missing parameter"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mobile_inspect_trace_search_uses_the_shared_history_search(temp_trace_env):
|
||||
trace_id = str(uuid.uuid4())
|
||||
trace_store.init_trace(trace_id, "Search test", "Flash", "conv-1", device_serial="pixel-7")
|
||||
_seed_trace(temp_trace_env, trace_id)
|
||||
|
||||
res = await mobile_inspect_trace(action="search", trace_id=trace_id, query="toggle")
|
||||
assert "error" not in res
|
||||
assert res["device_serial"] == "pixel-7"
|
||||
assert "[Step 1 " in res["results"]
|
||||
assert "Screen: Asked the explorer, then tapped Save." in res["results"]
|
||||
|
||||
# On-screen OCR text of the step's screenshot is part of the search surface.
|
||||
res = await mobile_inspect_trace(action="search", trace_id=trace_id, query="ocrneedle")
|
||||
assert "[Step 1 " in res["results"]
|
||||
|
||||
ledger = await mobile_inspect_trace(action="search", trace_id=trace_id, step_range=[1, 1])
|
||||
assert "Action ledger for Steps 1–1" in ledger["results"]
|
||||
|
||||
empty = await mobile_inspect_trace(action="search", trace_id=trace_id)
|
||||
assert empty["error"] == "Missing parameter"
|
||||
@@ -358,7 +358,7 @@ def _hashes(boundary_after: int):
|
||||
|
||||
|
||||
def test_milestone_switch_closes_previous_segment_whole_and_gates_swap():
|
||||
"""Ready-gated swap (user decision 2026-09-01): a milestone switch closes
|
||||
"""Ready-gated swap: a milestone switch closes
|
||||
the previous segment whole and dispatches its capsule, but the original
|
||||
turns stay in the transcript until the capsule header is ready; the swap
|
||||
happens at the first render after readiness."""
|
||||
@@ -726,7 +726,7 @@ def test_era_overflow_compresses_oldest_era_to_period_paragraph():
|
||||
assert "Verified: " in match.group(1)
|
||||
# Recall guidance line is retained.
|
||||
assert (
|
||||
f" (Step-level ledger via recall_history for steps"
|
||||
f" (Step-level ledger via search_history for steps"
|
||||
f" {era.start_step_number}–{era.end_step_number})"
|
||||
) in frozen
|
||||
# Never-evict: the user injection survives even the period paragraph.
|
||||
@@ -799,7 +799,7 @@ def test_period_paragraph_degrades_to_milestone_list_when_all_band1_missing():
|
||||
|
||||
block = render_era_block(era)
|
||||
assert block.startswith(f"[Era 3 | Steps 1–8 | T+01:00 → T+08:59] {paragraph}")
|
||||
assert "(Step-level ledger via recall_history for steps 1–8)" in block
|
||||
assert "(Step-level ledger via search_history for steps 1–8)" in block
|
||||
assert 'User @ Step 6: "stay logged in"' in block
|
||||
|
||||
|
||||
@@ -1034,7 +1034,7 @@ async def test_flush_harvests_late_capsule_and_persists_ready_version():
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ready-gated swap (user decision 2026-09-01): degradation ladder
|
||||
# Ready-gated swap: degradation ladder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -1098,7 +1098,7 @@ def test_hard_threshold_force_swaps_pending_chunks_into_l3():
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Band ① note-linkage machine check (user decision 2026-09-01)
|
||||
# Band ① note-linkage machine check
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -1176,3 +1176,299 @@ def test_chunk_lists_every_step_of_a_multi_action_turn():
|
||||
assert chunk.source_step_ids == ["s1", "s2", "s3"]
|
||||
assert chunk.start_step_number == 1 and chunk.end_step_number == 3
|
||||
assert "- Step 2 (T+00:20)" in chunk.band3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Capsule source = the operator's own transcript
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _rich_turn(i: int) -> list:
|
||||
"""A turn that asks the explorer before acting, with a UI list and a
|
||||
native thinking block — the parts the old per-step projection dropped."""
|
||||
from artemis.memory.transcript import PRO_UI_LIST_MARKER
|
||||
|
||||
return [
|
||||
HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": f"# CURRENT OBSERVATION [T+00:{i % 60:02d}]"},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG_{i}"}},
|
||||
{"type": "text", "text": f"{PRO_UI_LIST_MARKER}\n[0] Buy button"},
|
||||
]
|
||||
),
|
||||
AIMessage(
|
||||
content=[
|
||||
{"type": "thinking", "thinking": "need the price first"},
|
||||
{"type": "text", "text": "Ask the explorer for the price."},
|
||||
],
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "ask_explorer",
|
||||
"args": {"question": "price?"},
|
||||
"id": f"tcx{i}",
|
||||
"type": "tool_call",
|
||||
}
|
||||
],
|
||||
),
|
||||
ToolMessage(tool_call_id=f"tcx{i}", content="Price shown: ¥39"),
|
||||
AIMessage(
|
||||
content=f"thinking {i}",
|
||||
tool_calls=[
|
||||
{"name": "click", "args": {"target": i}, "id": f"tc{i}", "type": "tool_call"}
|
||||
],
|
||||
),
|
||||
ToolMessage(tool_call_id=f"tc{i}", content="Action Recorded"),
|
||||
]
|
||||
|
||||
|
||||
def _payload_with_step(capsule: StubCapsuleService, step_number: int) -> dict:
|
||||
for payload in capsule._step_inputs.values():
|
||||
if any(s.get("step_number") == step_number for s in payload.get("steps", [])):
|
||||
return payload
|
||||
raise AssertionError(f"no capsule payload carries step {step_number}")
|
||||
|
||||
|
||||
def test_capsule_payload_carries_each_turn_transcript_as_the_operator_saw_it():
|
||||
"""The segment capsule digests exactly the text it replaces: every turn's
|
||||
post-scrub transcript — reasoning (native thinking included), every tool
|
||||
call with its arguments and returned result, the validator result — not
|
||||
a hand-picked projection. Scrubbed parts (UI list, screenshot) are gone
|
||||
from the source too, because the operator no longer saw them either."""
|
||||
steps = [_step(i) for i in range(1, 7)]
|
||||
ledger, chunker, _engine, capsule = _make(steps, min_active=2, max_steps=3)
|
||||
for i in range(1, 7):
|
||||
ledger.commit_staged(
|
||||
step_key=f"s{i - 1}" if i > 1 else None,
|
||||
validator_result={"status": "success"} if i > 1 else None,
|
||||
)
|
||||
ledger.render([_observation(i)])
|
||||
ledger.stage_turn(_rich_turn(i) if i == 2 else _turn(i))
|
||||
chunker.on_step_stamped(f"s{i}", "hash-a")
|
||||
|
||||
payload = _payload_with_step(capsule, 2)
|
||||
turn = next(t for t in payload["turns"] if t["steps"][0]["step_number"] == 2)
|
||||
transcript = turn["transcript"]
|
||||
|
||||
# What the operator saw during that turn, verbatim and in order.
|
||||
assert "(thinking) need the price first" in transcript
|
||||
assert "Ask the explorer for the price." in transcript
|
||||
assert '[tool call] ask_explorer({"question": "price?"})' in transcript
|
||||
assert "[tool result ask_explorer]\nPrice shown: ¥39" in transcript
|
||||
assert '[tool call] click({"target": 2})' in transcript
|
||||
assert "--- Action Execution Result" in transcript and "Status: success" in transcript
|
||||
assert transcript.index("Price shown") < transcript.index("click(")
|
||||
# What the scrub edge already removed from the operator's view.
|
||||
assert "Buy button" not in transcript and "IMG_2" not in transcript
|
||||
|
||||
# The lens request renders the turn block: step facts, then the transcript.
|
||||
lens = StepCapsuleLens(model_name="test", llm=object())
|
||||
text = lens.build_messages(payload)[1].content
|
||||
assert "## Step 2 (T+00:20)" in text
|
||||
assert "- Step 2 (T+00:20): " in text and "Visual transition of step 2." in text
|
||||
assert "Transcript as seen by the operator during this turn:" in text
|
||||
assert text.index("Recorded steps:") < text.index("Price shown: ¥39")
|
||||
assert "Reasoning excerpt" not in text # the transcript supersedes the excerpt
|
||||
|
||||
|
||||
def test_chunk_without_ledger_falls_back_to_per_step_projection():
|
||||
"""A chunk built without a live ledger (no transcripts) keeps the
|
||||
per-step fallback rendering so the capsule can still be produced."""
|
||||
steps = [_step(i) for i in range(1, 3)]
|
||||
_ledger, chunker, _engine, capsule = _make(steps)
|
||||
turns = [{"step_key": f"s{i}", "step_keys": [f"s{i}"], "start": 0, "end": 3} for i in (1, 2)]
|
||||
chunk = chunker._create_chunk(turns, None, {s["step_id"]: s for s in steps})
|
||||
assert chunk is not None
|
||||
payload = capsule._step_inputs[chunk.capsule_key]
|
||||
assert "turns" not in payload
|
||||
|
||||
text = StepCapsuleLens(model_name="test", llm=object()).build_messages(payload)[1].content
|
||||
assert "## Step 1 (T+00:10)" in text and "## Step 2 (T+00:20)" in text
|
||||
assert "Reasoning excerpt: thinking 1" in text
|
||||
assert "Transcript as seen by the operator" not in text
|
||||
|
||||
|
||||
def test_turn_transcript_is_capped_in_place_inside_the_capsule_request():
|
||||
lens = StepCapsuleLens(model_name="test", llm=object())
|
||||
cap = StepCapsuleLens.MAX_TURN_TRANSCRIPT_CHARS
|
||||
step = {"step_number": 4, "offset": "T+00:40", "action": "tap", "outcome": "executed"}
|
||||
payload = {
|
||||
"start_step": 4,
|
||||
"end_step": 4,
|
||||
"steps": [step],
|
||||
"turns": [{"steps": [step], "transcript": "x" * (cap + 500)}],
|
||||
}
|
||||
text = lens.build_messages(payload)[1].content
|
||||
assert "x" * (cap + 1) not in text
|
||||
assert "500 characters cut here" in text
|
||||
|
||||
|
||||
def test_capped_transcript_keeps_its_tail_so_the_validator_result_survives():
|
||||
"""The cut lands in the middle: the head (observation + reasoning) and the
|
||||
tail (the validator result that closes every turn) both stay verbatim."""
|
||||
cap = StepCapsuleLens.MAX_TURN_TRANSCRIPT_CHARS
|
||||
tail = "\n[observation]\n--- Action Execution Result (T+09:59) ---\nStatus: failed\nError: boom"
|
||||
transcript = "HEAD-START " + "h" * (cap + 3000) + tail
|
||||
capped = StepCapsuleLens._cap_transcript(transcript)
|
||||
assert capped.startswith("HEAD-START ")
|
||||
assert capped.endswith(tail)
|
||||
assert "characters cut here" in capped
|
||||
assert len(capped) <= cap + 200 # cap plus the announcement line
|
||||
|
||||
|
||||
def test_visual_transition_line_is_skipped_when_the_transcript_already_has_it():
|
||||
"""The resolved visual summary sits verbatim in the transcript once the
|
||||
scrub edge replaced the screenshot; only an unresolved turn (placeholder
|
||||
still there) gets the DataEngine summary as a separate line."""
|
||||
from artemis.agents.flash.context_compressor import HISTORY_SUMMARY_PREFIX
|
||||
|
||||
step = {
|
||||
"step_number": 4,
|
||||
"offset": "T+00:40",
|
||||
"action": "tap",
|
||||
"outcome": "executed",
|
||||
"visual_summary": "The cart badge changed from 0 to 1.",
|
||||
}
|
||||
resolved = f"[observation]\n{HISTORY_SUMMARY_PREFIX}The cart badge changed from 0 to 1."
|
||||
unresolved = "[observation]\n[screenshot]"
|
||||
for transcript, expected_count in ((resolved, 1), (unresolved, 1), ("", 1)):
|
||||
payload = {"start_step": 4, "end_step": 4, "steps": [step]}
|
||||
if transcript:
|
||||
payload["turns"] = [{"steps": [step], "transcript": transcript}]
|
||||
text = StepCapsuleLens(model_name="test", llm=object()).build_messages(payload)[1].content
|
||||
assert text.count("The cart badge changed from 0 to 1.") == expected_count, transcript
|
||||
resolved_text = (
|
||||
StepCapsuleLens(model_name="test", llm=object())
|
||||
.build_messages(
|
||||
{
|
||||
"start_step": 4,
|
||||
"end_step": 4,
|
||||
"steps": [step],
|
||||
"turns": [{"steps": [step], "transcript": resolved}],
|
||||
}
|
||||
)[1]
|
||||
.content
|
||||
)
|
||||
assert "Visual transition:" not in resolved_text
|
||||
unresolved_text = (
|
||||
StepCapsuleLens(model_name="test", llm=object())
|
||||
.build_messages(
|
||||
{
|
||||
"start_step": 4,
|
||||
"end_step": 4,
|
||||
"steps": [step],
|
||||
"turns": [{"steps": [step], "transcript": unresolved}],
|
||||
}
|
||||
)[1]
|
||||
.content
|
||||
)
|
||||
assert "Visual transition: The cart badge" in unresolved_text
|
||||
|
||||
|
||||
def test_size_trigger_measures_the_rendered_transcript_the_capsule_receives():
|
||||
"""``turn_text_chars`` sizes the exact text the lens gets — tool-call
|
||||
arguments included — so the source-token trigger and the capsule payload
|
||||
never disagree about how big a segment is."""
|
||||
steps = [_step(i) for i in range(1, 5)]
|
||||
ledger, chunker, _engine, capsule = _make(steps, min_active=1, target_tokens=10**9)
|
||||
_run_turns(ledger, chunker, 1, 4)
|
||||
turns = ledger.unchunked_turns()
|
||||
assert ledger.turn_text_chars(turns) == sum(len(ledger.turn_transcript(t)) for t in turns)
|
||||
assert ledger.turn_text_chars(turns[:1]) == len(ledger.turn_transcript(turns[0]))
|
||||
assert '[tool call] click({"target": 1})' in ledger.turn_transcript(turns[0])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ready_capsule_releases_its_turn_transcripts():
|
||||
"""Once a capsule landed the job is never re-run, so the service drops the
|
||||
bulky ``turns`` from the retained payload (the flat step facts stay)."""
|
||||
from artemis.memory.chunking import ChunkCapsuleService
|
||||
|
||||
class EchoLens(StepCapsuleLens):
|
||||
async def render(self, key, payload):
|
||||
return json.dumps(_capsule(1, 1))
|
||||
|
||||
service = ChunkCapsuleService(None, EchoLens(model_name="test", llm=object()))
|
||||
payload = {
|
||||
"start_step": 1,
|
||||
"end_step": 1,
|
||||
"step_number": 1,
|
||||
"steps": [{"step_number": 1}],
|
||||
"turns": [{"steps": [{"step_number": 1}], "transcript": "big text"}],
|
||||
}
|
||||
service.submit("chunk:1-1", payload)
|
||||
await service.flush()
|
||||
assert service.has_summary("chunk:1-1")
|
||||
assert "turns" not in service.get_job_payload("chunk:1-1")
|
||||
assert service.get_job_payload("chunk:1-1")["steps"] == [{"step_number": 1}]
|
||||
|
||||
|
||||
def test_multi_action_turn_renders_one_block_with_every_recorded_step():
|
||||
lens = StepCapsuleLens(model_name="test", llm=object())
|
||||
steps = [
|
||||
{"step_number": n, "offset": f"T+00:{n}0", "action": f"tap btn{n}", "outcome": "executed"}
|
||||
for n in (7, 8)
|
||||
]
|
||||
payload = {"start_step": 7, "end_step": 8, "steps": steps}
|
||||
payload["turns"] = [{"steps": steps, "transcript": "[operator]\nfired two taps"}]
|
||||
text = lens.build_messages(payload)[1].content
|
||||
assert "## Steps 7–8 (T+00:70 → T+00:80)" in text
|
||||
assert "- Step 7 (T+00:70): tap btn7 -> executed" in text
|
||||
assert "- Step 8 (T+00:80): tap btn8 -> executed" in text
|
||||
assert "fired two taps" in text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Band ② must not restate band ③
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_band2_coordinates_are_machine_checked():
|
||||
"""A coordinate literal in any band-② interval text fails the attempt
|
||||
(regenerate), same rank as the coverage check: band ③ already carries
|
||||
every action's target, so ② repeating it is the ledger written twice."""
|
||||
from artemis.memory.chunking import band2_intervals_carry_coordinates
|
||||
|
||||
lens = StepCapsuleLens(model_name="test", llm=object())
|
||||
payload = {"start_step": 3, "end_step": 6, "steps": []}
|
||||
|
||||
with_coords = dict(
|
||||
_capsule(3, 6),
|
||||
intervals=[
|
||||
{"start_step": 3, "end_step": 5, "text": "opened three sections in turn"},
|
||||
{"start_step": 6, "end_step": 6, "text": "tapped 'Battery' at [320, 399]"},
|
||||
],
|
||||
)
|
||||
assert band2_intervals_carry_coordinates(with_coords["intervals"])
|
||||
assert lens.parse_capsule(json.dumps(with_coords), payload) is None
|
||||
|
||||
swipe_literal = dict(
|
||||
_capsule(3, 6),
|
||||
intervals=[{"start_step": 3, "end_step": 6, "text": "swiped [556, 289, 556, 124]"}],
|
||||
)
|
||||
assert lens.parse_capsule(json.dumps(swipe_literal), payload) is None
|
||||
|
||||
behavior_only = dict(
|
||||
_capsule(3, 6),
|
||||
intervals=[
|
||||
{
|
||||
"start_step": 3,
|
||||
"end_step": 6,
|
||||
"text": "Opened 'Apps' and 'Battery' and returned to the main list each time;"
|
||||
" the Battery page showed 100% and 3 rows.",
|
||||
}
|
||||
],
|
||||
)
|
||||
assert not band2_intervals_carry_coordinates(behavior_only["intervals"])
|
||||
assert lens.parse_capsule(json.dumps(behavior_only), payload) is not None
|
||||
|
||||
|
||||
def test_capsule_prompt_forbids_band2_coordinates_and_sets_a_length_target():
|
||||
"""The prompt carries the ② contract (behavior + observed result, no
|
||||
coordinates, homogeneous runs merged) and the ①+② soft length target,
|
||||
while keeping the preserve-don't-abstract rule."""
|
||||
prompt = StepCapsuleLens(model_name="test", llm=object())._prompt
|
||||
assert "NEVER include coordinates" in prompt
|
||||
assert "NEVER narrate step by step" in prompt
|
||||
assert "Merge homogeneous consecutive actions" in prompt
|
||||
assert "one third of the source text" in prompt
|
||||
assert "Preserve, don't abstract" in prompt
|
||||
@@ -320,3 +320,101 @@ def test_commit_records_every_step_key_of_a_multi_action_turn():
|
||||
ledger.stage_turn(_turn(2))
|
||||
ledger.commit_staged(step_key=None)
|
||||
assert ledger.unchunked_turns()[1]["step_keys"] == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Turn transcript rendering (chunk-capsule source)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_render_turn_transcript_is_lossless_over_text_thinking_tools_and_results():
|
||||
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
|
||||
|
||||
from artemis.memory.transcript import SCREENSHOT_PLACEHOLDER, render_turn_transcript
|
||||
|
||||
messages = [
|
||||
HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "# CURRENT OBSERVATION [T+01:00]"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG"}},
|
||||
]
|
||||
),
|
||||
AIMessage(
|
||||
content=[
|
||||
{"type": "thinking", "thinking": "compare the two prices"},
|
||||
{"type": "text", "text": "I will recall the earlier price."},
|
||||
],
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "search_history",
|
||||
"args": {"query": "price"},
|
||||
"id": "c1",
|
||||
"type": "tool_call",
|
||||
}
|
||||
],
|
||||
),
|
||||
ToolMessage(tool_call_id="c1", content="Step 3: price ¥39", status="success"),
|
||||
ToolMessage(
|
||||
tool_call_id="c2", name="adb_shell", content="permission denied", status="error"
|
||||
),
|
||||
ToolMessage(tool_call_id="c9", content="orphan result"),
|
||||
SystemMessage(content="You have reached the maximum number of tool calls"),
|
||||
AIMessage(
|
||||
content="tap now",
|
||||
tool_calls=[{"name": "click", "args": {"target": [0.5, 0.5]}, "id": "c3"}],
|
||||
),
|
||||
]
|
||||
text = render_turn_transcript(messages)
|
||||
assert text.splitlines()[0] == "[observation]"
|
||||
assert "# CURRENT OBSERVATION [T+01:00]" in text
|
||||
assert SCREENSHOT_PLACEHOLDER in text and "IMG" not in text
|
||||
assert "[operator]\n(thinking) compare the two prices\nI will recall the earlier price." in text
|
||||
assert '[tool call] search_history({"query": "price"})' in text
|
||||
# A Pro ToolMessage carries no name: resolved through the turn's own call.
|
||||
assert "[tool result search_history]\nStep 3: price ¥39" in text
|
||||
assert "[tool result adb_shell (error)]\npermission denied" in text
|
||||
assert "[tool result c9]\norphan result" in text # unresolvable: id shown
|
||||
assert "[system]\nYou have reached the maximum number of tool calls" in text
|
||||
assert '[tool call] click({"target": [0.5, 0.5]})' in text
|
||||
# Order is message order.
|
||||
assert text.index("search_history(") < text.index("price ¥39") < text.index("tap now")
|
||||
|
||||
|
||||
def test_ledger_turn_transcript_reflects_the_scrubbed_active_region():
|
||||
"""The transcript handed to chunk compression is the turn as it stands
|
||||
after the scrub edge: the UI list stripped, the validator result present."""
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
|
||||
from artemis.memory.transcript import PRO_UI_LIST_MARKER, TranscriptLedger
|
||||
|
||||
ledger = TranscriptLedger(clock=lambda: 0.0)
|
||||
|
||||
def turn(i: int) -> list:
|
||||
return [
|
||||
HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": f"# CURRENT OBSERVATION [T+00:0{i}]"},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG{i}"}},
|
||||
{"type": "text", "text": f"{PRO_UI_LIST_MARKER}\n[0] Button {i}"},
|
||||
]
|
||||
),
|
||||
AIMessage(
|
||||
content=f"thought {i}",
|
||||
tool_calls=[{"name": "click", "args": {}, "id": f"t{i}"}],
|
||||
),
|
||||
ToolMessage(tool_call_id=f"t{i}", content="Action Recorded"),
|
||||
]
|
||||
|
||||
ledger.stage_turn(turn(1))
|
||||
ledger.commit_staged(step_key="s1", validator_result={"status": "success"})
|
||||
ledger.stage_turn(turn(2))
|
||||
ledger.commit_staged(step_key="s2", validator_result={"status": "failed", "error": "boom"})
|
||||
ledger.render([HumanMessage(content=[{"type": "text", "text": "tail"}])])
|
||||
|
||||
first, second = ledger.unchunked_turns()
|
||||
text = ledger.turn_transcript(first)
|
||||
assert "thought 1" in text and "[tool call] click({})" in text
|
||||
assert "Button 1" not in text # depth-1 strip already applied
|
||||
assert "--- Action Execution Result (T+00:00) ---\nStatus: success" in text
|
||||
assert "Status: failed" in ledger.turn_transcript(second)
|
||||
assert "thought 2" not in text # a turn transcript never leaks into another turn
|
||||
@@ -0,0 +1,41 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""The Pro tuning summary the SDK persists with every session's device_info."""
|
||||
|
||||
from artemis.sdk.agent import run_tuning_summary
|
||||
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
|
||||
|
||||
|
||||
def test_flash_runs_persist_no_tuning():
|
||||
config = AgentConfigBuilder().build(validate_profiles=False)
|
||||
assert run_tuning_summary(config, "flash") is None
|
||||
|
||||
|
||||
def test_pro_runs_persist_the_two_launcher_sliders():
|
||||
config = (
|
||||
AgentConfigBuilder()
|
||||
.with_verification_level("checkpoints")
|
||||
.with_explorer(pro_mode="ultra")
|
||||
.build(validate_profiles=False)
|
||||
)
|
||||
assert run_tuning_summary(config, "pro") == {
|
||||
"verification_level": "checkpoints",
|
||||
"explorer_mode": "ultra",
|
||||
}
|
||||
|
||||
|
||||
def test_pro_summary_reflects_checker_switched_off():
|
||||
config = AgentConfigBuilder().with_verification_level("off").build(validate_profiles=False)
|
||||
assert run_tuning_summary(config, "pro")["verification_level"] == "off"
|
||||
@@ -808,3 +808,42 @@ class TestVerificationLevelPresets:
|
||||
)
|
||||
== "off"
|
||||
)
|
||||
|
||||
|
||||
class TestRunTuningSummary:
|
||||
"""``run_tuning_for_profile``: the per-run tuning persisted with a session."""
|
||||
|
||||
def test_flash_has_no_tuning(self):
|
||||
from artemis.config import CheckerConfig, ExplorerConfig, run_tuning_for_profile
|
||||
|
||||
kwargs = {"checker": CheckerConfig(), "explorer": ExplorerConfig()}
|
||||
assert run_tuning_for_profile("flash", **kwargs) is None
|
||||
assert run_tuning_for_profile(None, **kwargs) is None
|
||||
|
||||
def test_pro_reports_both_sliders(self):
|
||||
from artemis.config import (
|
||||
CheckerConfig,
|
||||
ExplorerConfig,
|
||||
checker_overrides_for_level,
|
||||
run_tuning_for_profile,
|
||||
)
|
||||
|
||||
summary = run_tuning_for_profile(
|
||||
" Pro ",
|
||||
checker=CheckerConfig(**checker_overrides_for_level("strict")),
|
||||
explorer=ExplorerConfig(pro_mode="ultra"),
|
||||
)
|
||||
assert summary == {"verification_level": "strict", "explorer_mode": "ultra"}
|
||||
|
||||
def test_pro_defaults_match_launcher_defaults(self):
|
||||
from artemis.config import (
|
||||
CheckerConfig,
|
||||
ExplorerConfig,
|
||||
run_tuning_for_profile,
|
||||
verification_level_for_checker,
|
||||
)
|
||||
|
||||
checker, explorer = CheckerConfig(), ExplorerConfig()
|
||||
summary = run_tuning_for_profile("pro", checker=checker, explorer=explorer)
|
||||
assert summary["verification_level"] == verification_level_for_checker(checker)
|
||||
assert summary["explorer_mode"] == explorer.resolve(profile="pro")
|
||||
@@ -239,4 +239,3 @@ async def test_mid_stream_failure_records_stream_reset_payload(monkeypatch):
|
||||
assert payload["reason"] == "mid_stream_failure"
|
||||
assert "stream_exec_id" in payload
|
||||
assert "lower API priority" in payload["message"]
|
||||
|
||||
@@ -93,7 +93,10 @@ def test_recall_and_similarity_defaults():
|
||||
assert cfg.memory.recall.enabled is True
|
||||
assert cfg.memory.recall.max_results == 5
|
||||
assert cfg.memory.recall.max_text_tokens == 2000
|
||||
assert cfg.memory.recall.max_image_steps == 1
|
||||
assert cfg.memory.recall.screen_scan_steps == 150
|
||||
assert not hasattr(cfg.memory.recall, "max_image_steps")
|
||||
assert cfg.memory.replay.max_steps == 5
|
||||
assert cfg.memory.replay.max_tokens == 12000
|
||||
assert cfg.memory.transcript.similarity_hint is True
|
||||
assert cfg.memory.transcript.similarity_max_distance == 5
|
||||
assert cfg.memory.policies == {}
|
||||
@@ -116,7 +119,8 @@ def test_recall_and_policies_overridable():
|
||||
cfg = AgentGlobalConfig.model_validate(
|
||||
{
|
||||
"memory": {
|
||||
"recall": {"enabled": False, "max_results": 3},
|
||||
"recall": {"enabled": False, "max_results": 3, "screen_scan_steps": 20},
|
||||
"replay": {"max_steps": 2, "max_tokens": 4000},
|
||||
"transcript": {"similarity_hint": False},
|
||||
"policies": {"planner": {"last_n_detailed": 3}},
|
||||
}
|
||||
@@ -124,5 +128,8 @@ def test_recall_and_policies_overridable():
|
||||
)
|
||||
assert cfg.memory.recall.enabled is False
|
||||
assert cfg.memory.recall.max_results == 3
|
||||
assert cfg.memory.recall.screen_scan_steps == 20
|
||||
assert cfg.memory.replay.max_steps == 2
|
||||
assert cfg.memory.replay.max_tokens == 4000
|
||||
assert cfg.memory.transcript.similarity_hint is False
|
||||
assert cfg.memory.policies == {"planner": {"last_n_detailed": 3}}
|
||||
@@ -169,3 +169,25 @@ def test_record_llm_usage_lens_calls_do_not_touch_last_prompt_tokens():
|
||||
assert meter.last_prompt_tokens == 15000 # untouched by the lens call
|
||||
assert meter.llm_calls == 2
|
||||
assert meter.prompt_tokens == 15300
|
||||
|
||||
|
||||
def test_record_llm_usage_tags_the_traced_node():
|
||||
"""Each usage trace names the traced node that issued the call so the
|
||||
session usage endpoint can pick the executor's (Operator/FlashRunner)
|
||||
prompt size out as the live context."""
|
||||
from artemis.data_engine.context_vars import CURRENT_NODE_NAME
|
||||
|
||||
engine = Mock()
|
||||
engine.current_session_id = uuid4()
|
||||
engine.current_step_id = None
|
||||
|
||||
token = CURRENT_NODE_NAME.set("operator")
|
||||
try:
|
||||
payload = record_llm_usage(engine, _msg_with_usage(prompt=42000, completion=90))
|
||||
finally:
|
||||
CURRENT_NODE_NAME.reset(token)
|
||||
assert payload["node"] == "operator"
|
||||
assert payload["context_base_tokens"] == 42000
|
||||
|
||||
untagged = record_llm_usage(engine, _msg_with_usage(prompt=10))
|
||||
assert "node" not in untagged
|
||||
@@ -6,12 +6,21 @@
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
import io
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
|
||||
from artemis.interfaces.cli.commands import ui
|
||||
from artemis.interfaces.cli.commands.ui import (
|
||||
_npm_install_required,
|
||||
_resolve_npm_executable,
|
||||
_run_build_step,
|
||||
_showcase_build_required,
|
||||
ensure_showcase_built,
|
||||
)
|
||||
|
||||
|
||||
@@ -64,3 +73,111 @@ def test_showcase_build_not_required_when_build_is_current(tmp_path):
|
||||
os.utime(built_index, (200, 200))
|
||||
|
||||
assert _showcase_build_required(showcase_dir) is False
|
||||
|
||||
|
||||
def _showcase_with_manifests(tmp_path, *, stamp_mtime=None):
|
||||
showcase_dir = tmp_path / "showcase_ui"
|
||||
showcase_dir.mkdir()
|
||||
for name in ("package.json", "package-lock.json"):
|
||||
path = showcase_dir / name
|
||||
path.write_text("{}", encoding="utf-8")
|
||||
os.utime(path, (100, 100))
|
||||
if stamp_mtime is not None:
|
||||
stamp = showcase_dir / "node_modules" / ".package-lock.json"
|
||||
stamp.parent.mkdir()
|
||||
stamp.write_text("{}", encoding="utf-8")
|
||||
os.utime(stamp, (stamp_mtime, stamp_mtime))
|
||||
return showcase_dir
|
||||
|
||||
|
||||
def test_npm_install_required_when_node_modules_missing(tmp_path):
|
||||
assert _npm_install_required(_showcase_with_manifests(tmp_path)) is True
|
||||
|
||||
|
||||
def test_npm_install_not_required_when_lock_stamp_is_current(tmp_path):
|
||||
assert _npm_install_required(_showcase_with_manifests(tmp_path, stamp_mtime=200)) is False
|
||||
|
||||
|
||||
def test_npm_install_required_when_package_lock_is_newer(tmp_path):
|
||||
showcase_dir = _showcase_with_manifests(tmp_path, stamp_mtime=200)
|
||||
os.utime(showcase_dir / "package-lock.json", (300, 300))
|
||||
|
||||
assert _npm_install_required(showcase_dir) is True
|
||||
|
||||
|
||||
def _recording_console():
|
||||
return Console(file=io.StringIO(), force_terminal=False, width=120)
|
||||
|
||||
|
||||
def test_ensure_showcase_built_skips_install_when_deps_current(monkeypatch):
|
||||
steps = []
|
||||
monkeypatch.setattr(ui, "_showcase_build_required", lambda showcase_dir: True)
|
||||
monkeypatch.setattr(ui, "_npm_install_required", lambda showcase_dir: False)
|
||||
monkeypatch.setattr(ui, "_resolve_npm_executable", lambda: "npm")
|
||||
monkeypatch.setattr(
|
||||
ui, "_run_build_step", lambda console, label, cmd, cwd: steps.append((label, cmd))
|
||||
)
|
||||
console = _recording_console()
|
||||
|
||||
ensure_showcase_built(console)
|
||||
|
||||
assert [label for label, _ in steps] == ["② ng build"]
|
||||
assert steps[0][1] == ["npm", "run", "build"]
|
||||
assert "Showcase UI built in" in console.file.getvalue()
|
||||
|
||||
|
||||
def test_ensure_showcase_built_runs_install_first_when_deps_changed(monkeypatch):
|
||||
steps = []
|
||||
monkeypatch.setattr(ui, "_showcase_build_required", lambda showcase_dir: True)
|
||||
monkeypatch.setattr(ui, "_npm_install_required", lambda showcase_dir: True)
|
||||
monkeypatch.setattr(ui, "_resolve_npm_executable", lambda: "npm")
|
||||
monkeypatch.setattr(
|
||||
ui, "_run_build_step", lambda console, label, cmd, cwd: steps.append((label, cmd))
|
||||
)
|
||||
|
||||
ensure_showcase_built(_recording_console())
|
||||
|
||||
assert [label for label, _ in steps] == ["① npm install (dependencies changed)", "② ng build"]
|
||||
assert steps[0][1] == ["npm", "install", "--no-audit", "--no-fund", "--loglevel=warn"]
|
||||
|
||||
|
||||
def test_ensure_showcase_built_reports_failure_with_manual_command(monkeypatch):
|
||||
monkeypatch.setattr(ui, "_showcase_build_required", lambda showcase_dir: True)
|
||||
monkeypatch.setattr(ui, "_npm_install_required", lambda showcase_dir: False)
|
||||
monkeypatch.setattr(ui, "_resolve_npm_executable", lambda: "npm")
|
||||
|
||||
def failing_step(console, label, cmd, cwd):
|
||||
raise subprocess.CalledProcessError(1, cmd)
|
||||
|
||||
monkeypatch.setattr(ui, "_run_build_step", failing_step)
|
||||
console = _recording_console()
|
||||
|
||||
ensure_showcase_built(console) # must not raise: the server still starts
|
||||
|
||||
output = console.file.getvalue()
|
||||
assert "Failed to auto-build Showcase UI" in output
|
||||
assert "npm install && npm run build" in output
|
||||
|
||||
|
||||
def test_run_build_step_streams_child_output_and_raises_on_failure(tmp_path):
|
||||
console = _recording_console()
|
||||
cmd = [sys.executable, "-c", "print('hello from child'); import sys; sys.exit(3)"]
|
||||
|
||||
with pytest.raises(subprocess.CalledProcessError) as excinfo:
|
||||
_run_build_step(console, "② ng build", cmd, tmp_path)
|
||||
|
||||
assert excinfo.value.returncode == 3
|
||||
output = console.file.getvalue()
|
||||
assert "② ng build" in output
|
||||
assert "hello from child" in output
|
||||
|
||||
|
||||
def test_run_build_step_strips_ansi_and_succeeds(tmp_path):
|
||||
console = _recording_console()
|
||||
cmd = [sys.executable, "-c", "print('\\x1b[32mdone\\x1b[0m')"]
|
||||
|
||||
_run_build_step(console, "step", cmd, tmp_path)
|
||||
|
||||
output = console.file.getvalue()
|
||||
assert "done" in output
|
||||
assert "\x1b[32m" not in output
|
||||
@@ -621,3 +621,85 @@ async def test_next_segment_anchors_at_first_frame(mock_ctx, tmp_path):
|
||||
assert session.android_segment_started_at >= before
|
||||
assert session.android_rotation == 1
|
||||
assert session.local_video_path == tmp_path / "recording_001.mkv"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_recording_passes_session_offsets_to_manifest(mock_ctx, tmp_path):
|
||||
"""The playlist manifest must carry each segment's real session-relative start."""
|
||||
controller = UnifiedMobileController(mock_ctx)
|
||||
remove_active_session("emulator-5554")
|
||||
|
||||
seg0 = tmp_path / "recording.mkv"
|
||||
seg0.write_bytes(b"segment 0")
|
||||
seg1 = tmp_path / "recording_001.mkv"
|
||||
seg1.write_bytes(b"segment 1")
|
||||
|
||||
now = time.time()
|
||||
session_t0 = now - 40.0
|
||||
# First frame landed 1.2s after the session started; rotation restarted
|
||||
# scrcpy so segment 1 begins 11.5s after that first frame (10s of video + 1.5s gap).
|
||||
recording_t0 = session_t0 + 1.2
|
||||
session = RecordingSession(
|
||||
video_id=uuid4(),
|
||||
device_id="emulator-5554",
|
||||
start_time=recording_t0,
|
||||
data_engine_start_time=session_t0,
|
||||
local_video_path=seg1,
|
||||
android_video_segments=[seg0],
|
||||
android_segment_records=[
|
||||
{
|
||||
"path": seg0,
|
||||
"output_path": tmp_path / "recording.mp4",
|
||||
"start": 0.0,
|
||||
"end": 10.0,
|
||||
}
|
||||
],
|
||||
android_segment_index=1,
|
||||
android_segment_started_at=recording_t0 + 11.5,
|
||||
process=MagicMock(returncode=None, terminate=MagicMock(), wait=AsyncMock()),
|
||||
)
|
||||
set_active_session("emulator-5554", session)
|
||||
|
||||
manifest = AsyncMock(return_value=tmp_path / "recording.json")
|
||||
with (
|
||||
patch(
|
||||
"artemis.controllers.unified_controller.remux_recording_to_mp4",
|
||||
AsyncMock(side_effect=lambda src, dst: (dst.write_bytes(b"mp4"), True)[1]),
|
||||
),
|
||||
patch("artemis.controllers.unified_controller.write_recording_manifest", manifest),
|
||||
):
|
||||
res = await controller.stop_video_recording()
|
||||
|
||||
assert res.success is True
|
||||
manifest.assert_awaited_once()
|
||||
output_dir, mp4_paths, offsets = manifest.await_args.args
|
||||
assert output_dir == tmp_path
|
||||
assert mp4_paths == [tmp_path / "recording.mp4", tmp_path / "recording_001.mp4"]
|
||||
assert abs(offsets[tmp_path / "recording.mp4"] - 1.2) < 0.01
|
||||
assert abs(offsets[tmp_path / "recording_001.mp4"] - 12.7) < 0.01
|
||||
|
||||
|
||||
def test_segment_session_offsets_without_data_engine_anchor(tmp_path):
|
||||
session = RecordingSession(
|
||||
video_id=uuid4(),
|
||||
device_id="emulator-5554",
|
||||
start_time=1000.0,
|
||||
data_engine_start_time=None,
|
||||
local_video_path=tmp_path / "recording.mkv",
|
||||
android_segment_records=[
|
||||
{"path": tmp_path / "a.mkv", "output_path": tmp_path / "recording.mp4", "start": 0.0},
|
||||
{
|
||||
"path": tmp_path / "b.mkv",
|
||||
"output_path": tmp_path / "recording_001.mp4",
|
||||
"start": 7.5,
|
||||
},
|
||||
],
|
||||
)
|
||||
fallback = tmp_path / "recording_converted.mp4"
|
||||
offsets = UnifiedMobileController._segment_session_offsets(
|
||||
session, [tmp_path / "recording.mp4", tmp_path / "recording_001.mp4", fallback]
|
||||
)
|
||||
assert offsets[tmp_path / "recording.mp4"] == 0.0
|
||||
assert offsets[tmp_path / "recording_001.mp4"] == 7.5
|
||||
# An emergency fallback remux has no record; it starts at the recording anchor.
|
||||
assert offsets[fallback] == 0.0
|
||||
@@ -0,0 +1,99 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""One args schema, two contracts: the Flash ``ToolDeclaration`` and the
|
||||
LangChain tool of every history tool are derived from the same Pydantic model."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.core.tool_declaration import ToolDeclaration
|
||||
from artemis.tools.history import (
|
||||
GET_STEP_SCREENSHOT_TOOL,
|
||||
HISTORY_TOOL_DECLARATIONS,
|
||||
HISTORY_TOOL_NAMES,
|
||||
HISTORY_TOOL_WRAPPERS,
|
||||
HISTORY_TOOLS,
|
||||
REPLAY_STEPS_TOOL,
|
||||
SEARCH_HISTORY_TOOL,
|
||||
get_history_tools,
|
||||
history_tool_by_name,
|
||||
history_tool_declarations,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool", HISTORY_TOOLS, ids=lambda t: t.name)
|
||||
def test_tool_declaration_mirrors_the_args_schema(tool):
|
||||
declaration = tool.to_tool_declaration()
|
||||
assert isinstance(declaration, ToolDeclaration)
|
||||
assert declaration.name == tool.name
|
||||
assert declaration.description == tool.description
|
||||
|
||||
properties = declaration.parameters["properties"]
|
||||
assert list(properties) == list(tool.args_schema.model_fields)
|
||||
for field_name, info in tool.args_schema.model_fields.items():
|
||||
assert properties[field_name]["description"] == info.description
|
||||
assert declaration.parameters["required"] == [
|
||||
name for name, info in tool.args_schema.model_fields.items() if info.is_required()
|
||||
]
|
||||
|
||||
# The LangChain export binds the identical schema.
|
||||
lc_tool = tool.to_langchain_tool(SimpleNamespace(data_engine=None))
|
||||
assert lc_tool.name == tool.name
|
||||
assert lc_tool.args_schema is tool.args_schema
|
||||
|
||||
# And the GenAI export agrees on names and required fields.
|
||||
genai = tool.to_genai_declaration()
|
||||
assert set(genai.parameters.properties) == set(properties)
|
||||
assert list(genai.parameters.required or []) == declaration.parameters["required"]
|
||||
|
||||
|
||||
def test_declared_types_and_enums():
|
||||
search = SEARCH_HISTORY_TOOL.parameters["properties"]
|
||||
assert search["query"]["type"] == "string"
|
||||
assert search["step_range"] == {
|
||||
"type": "array",
|
||||
"items": {"type": "integer"},
|
||||
"description": search["step_range"]["description"],
|
||||
}
|
||||
assert search["max_results"]["type"] == "integer"
|
||||
assert search["max_results"]["default"] == 5
|
||||
assert SEARCH_HISTORY_TOOL.parameters["required"] == []
|
||||
|
||||
replay = REPLAY_STEPS_TOOL.parameters["properties"]
|
||||
assert replay["start_step"]["type"] == "integer"
|
||||
assert replay["end_step"]["type"] == "integer"
|
||||
assert REPLAY_STEPS_TOOL.parameters["required"] == ["start_step"]
|
||||
|
||||
shot = GET_STEP_SCREENSHOT_TOOL.parameters["properties"]
|
||||
assert shot["step_number"]["type"] == "integer"
|
||||
assert shot["which"]["type"] == "string"
|
||||
assert shot["which"]["enum"] == ["pre", "post", "overlay"]
|
||||
assert shot["which"]["default"] == "pre"
|
||||
assert GET_STEP_SCREENSHOT_TOOL.parameters["required"] == ["step_number"]
|
||||
|
||||
|
||||
def test_exports_are_consistent():
|
||||
assert HISTORY_TOOL_NAMES == {"search_history", "replay_steps", "get_step_screenshot"}
|
||||
assert [d.name for d in HISTORY_TOOL_DECLARATIONS] == [t.name for t in HISTORY_TOOLS]
|
||||
assert len(HISTORY_TOOL_WRAPPERS) == 3
|
||||
assert history_tool_by_name("default_api:replay_steps").name == "replay_steps"
|
||||
assert history_tool_by_name("nope") is None
|
||||
|
||||
ctx = SimpleNamespace(data_engine=object())
|
||||
expected = [t.name for t in HISTORY_TOOLS]
|
||||
assert [t.name for t in get_history_tools(ctx)] == expected
|
||||
assert [d.name for d in history_tool_declarations(ctx)] == expected
|
||||
assert history_tool_declarations(SimpleNamespace(data_engine=None)) == []
|
||||
@@ -0,0 +1,150 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""replay_steps: range replay, per-call step cap, whole-step token budget,
|
||||
``[Screen]`` status lines, and the no-history degradation."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.tools.history import ReplayStepsTool, replay_steps, replay_steps_text
|
||||
|
||||
|
||||
def _friendly_step(number, tool_result="The toggle is ON", **overrides):
|
||||
step = {
|
||||
"step_id": f"step_{number}",
|
||||
"step_number": number,
|
||||
"relative_time": f"{number * 3}.0s",
|
||||
"summary": f"Step {number} summary.",
|
||||
"action_taken": {"action": "click", "target": [0.5, 0.5], "target_text": "Save"},
|
||||
"last_execution_result": {"status": "success"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": f"Thinking at step {number}."},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "ask_explorer",
|
||||
"args": {"question": "toggle?"},
|
||||
"result": tool_result,
|
||||
},
|
||||
],
|
||||
}
|
||||
step.update(overrides)
|
||||
return step
|
||||
|
||||
|
||||
def _reader(steps):
|
||||
reader = MagicMock()
|
||||
reader.get_agent_friendly_steps_in_range.side_effect = lambda s, e: [
|
||||
st for st in steps if s <= st["step_number"] <= e
|
||||
]
|
||||
return reader
|
||||
|
||||
|
||||
def test_replay_range_renders_every_step_in_full():
|
||||
reader = _reader([_friendly_step(3), _friendly_step(4, tool_result="The toggle is OFF " * 100)])
|
||||
out = replay_steps_text(reader, 3, 4)
|
||||
|
||||
reader.get_agent_friendly_steps_in_range.assert_called_once_with(3, 4)
|
||||
assert "- **Step 3 (Start: 9.0s)**" in out
|
||||
assert "- **Step 4 (Start: 12.0s)**" in out
|
||||
assert "[Screen]: Step 3 summary." in out
|
||||
assert "Thinking at step 4." in out
|
||||
assert "`ask_explorer(" in out
|
||||
# Loose clamp: a ~1.8k-char tool result is replayed in full.
|
||||
assert "The toggle is OFF " * 100 in out
|
||||
assert "[Planned Action]: Tapped 'Save' at [0.5, 0.5]" in out
|
||||
|
||||
|
||||
def test_replay_single_step_reversed_bounds_and_missing_step():
|
||||
reader = _reader([_friendly_step(2)])
|
||||
assert "- **Step 2 (Start: 6.0s)**" in replay_steps_text(reader, 2)
|
||||
reader.get_agent_friendly_steps_in_range.assert_called_once_with(2, 2)
|
||||
|
||||
reader = _reader([_friendly_step(2), _friendly_step(3)])
|
||||
out = replay_steps_text(reader, 3, 2)
|
||||
reader.get_agent_friendly_steps_in_range.assert_called_once_with(2, 3)
|
||||
assert out.index("**Step 2") < out.index("**Step 3")
|
||||
|
||||
assert replay_steps_text(_reader([]), 9) == "Error: step 9 not found."
|
||||
assert replay_steps_text(_reader([]), 9, 12) == "Error: no recorded steps in range 9–12."
|
||||
assert "must be integers" in replay_steps_text(_reader([]), "x")
|
||||
|
||||
|
||||
def test_replay_caps_the_range_per_call():
|
||||
reader = _reader([_friendly_step(n) for n in range(1, 30)])
|
||||
out = replay_steps_text(reader, 1, 20, max_steps=5)
|
||||
reader.get_agent_friendly_steps_in_range.assert_called_once_with(1, 5)
|
||||
assert "**Step 5" in out
|
||||
assert "**Step 6" not in out
|
||||
assert "Call again from Step 6" in out
|
||||
|
||||
|
||||
def test_replay_token_budget_drops_whole_trailing_steps():
|
||||
big = "x" * 4000 # ~1k tokens per step
|
||||
reader = _reader([_friendly_step(n, tool_result=big) for n in range(1, 6)])
|
||||
out = replay_steps_text(reader, 1, 5, max_steps=5, max_tokens=2500)
|
||||
|
||||
# Two steps fit; the rest are dropped as whole steps, never cut mid-step.
|
||||
assert "**Step 1" in out and "**Step 2" in out
|
||||
assert "**Step 3" not in out
|
||||
assert out.count(big) == 2
|
||||
assert "Token budget reached: Steps 1–2 replayed; 3 more step(s)" in out
|
||||
assert "Call again from Step 3" in out
|
||||
|
||||
|
||||
def test_replay_always_shows_at_least_the_first_step():
|
||||
reader = _reader([_friendly_step(1, tool_result="y" * 4000), _friendly_step(2)])
|
||||
out = replay_steps_text(reader, 1, 2, max_tokens=10)
|
||||
assert "**Step 1" in out
|
||||
assert "y" * 4000 in out
|
||||
assert "**Step 2" not in out
|
||||
|
||||
|
||||
def test_replay_screen_line_states():
|
||||
with_text = replay_steps_text(_reader([_friendly_step(1)]), 1)
|
||||
assert "[Screen]: Step 1 summary." in with_text
|
||||
assert "[Summary]" not in with_text
|
||||
|
||||
pending = replay_steps_text(
|
||||
_reader([_friendly_step(1, summary=None, extra_metadata={"summary_status": "pending"})]),
|
||||
1,
|
||||
)
|
||||
assert "[Screen]: (screen description pending)" in pending
|
||||
|
||||
failed = replay_steps_text(
|
||||
_reader([_friendly_step(1, summary="", extra_metadata={"summary_status": "failed"})]),
|
||||
1,
|
||||
)
|
||||
assert "[Screen]: (screen description unavailable)" in failed
|
||||
|
||||
no_status = replay_steps_text(_reader([_friendly_step(1, summary=None)]), 1)
|
||||
assert "[Screen]" not in no_status
|
||||
|
||||
|
||||
def test_replay_without_history():
|
||||
assert "no execution history" in replay_steps_text(None, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_execute_reads_the_context_engine_and_degrades_without_one():
|
||||
reader = _reader([_friendly_step(2)])
|
||||
out = await ReplayStepsTool().execute(ctx=SimpleNamespace(data_engine=reader), start_step=2)
|
||||
assert "- **Step 2 (Start: 6.0s)**" in out
|
||||
|
||||
out = await replay_steps.execute(ctx=SimpleNamespace(data_engine=None), start_step=1)
|
||||
assert "no execution history" in out
|
||||
assert replay_steps.is_available(SimpleNamespace(data_engine=None)) is False
|
||||
assert replay_steps.is_available(SimpleNamespace(data_engine=reader)) is True
|
||||
@@ -0,0 +1,120 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""tool_result_messages: one helper decides how a tool result — with or
|
||||
without images — enters the conversation, per LLM provider."""
|
||||
|
||||
from enum import StrEnum
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import HumanMessage, ToolMessage
|
||||
|
||||
from artemis.tools.tool_wrapper import (
|
||||
resolve_image_carrier,
|
||||
split_multimodal_result,
|
||||
tool_result_messages,
|
||||
)
|
||||
|
||||
IMAGE = {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,AAAA"}}
|
||||
SHOT = [{"type": "text", "text": "Screenshot of step 7 (pre-action) is attached."}, IMAGE]
|
||||
|
||||
|
||||
class _Provider(StrEnum):
|
||||
GOOGLE = "google"
|
||||
VERTEX_AI = "vertexai"
|
||||
|
||||
|
||||
def _llm(provider):
|
||||
return SimpleNamespace(endpoint=SimpleNamespace(provider=provider))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("provider", "carrier"),
|
||||
[
|
||||
("google", "tool"),
|
||||
("anthropic", "tool"),
|
||||
("vertexai", "human"),
|
||||
("openai", "human"),
|
||||
("openrouter", "human"),
|
||||
("xai", "human"),
|
||||
("ollama", "human"),
|
||||
(None, "human"),
|
||||
],
|
||||
)
|
||||
def test_resolve_image_carrier_by_provider_name(provider, carrier):
|
||||
assert resolve_image_carrier(provider) == carrier
|
||||
assert resolve_image_carrier(_llm(provider)) == carrier
|
||||
|
||||
|
||||
def test_resolve_image_carrier_accepts_enums_and_endpoint_less_llms():
|
||||
assert resolve_image_carrier(_Provider.GOOGLE) == "tool"
|
||||
assert resolve_image_carrier(_llm(_Provider.VERTEX_AI)) == "human"
|
||||
assert resolve_image_carrier(SimpleNamespace()) == "human"
|
||||
|
||||
|
||||
def test_split_multimodal_result():
|
||||
assert split_multimodal_result("plain") == ("plain", [])
|
||||
assert split_multimodal_result(None) == ("", [])
|
||||
assert split_multimodal_result(ToolMessage(content="x", tool_call_id="t")) == ("x", [])
|
||||
text, images = split_multimodal_result(SHOT)
|
||||
assert text == "Screenshot of step 7 (pre-action) is attached."
|
||||
assert images == [IMAGE]
|
||||
|
||||
|
||||
def test_text_only_results_stay_a_single_tool_message():
|
||||
msgs = tool_result_messages("tc-1", "Step 3 replay", name="replay_steps", status="error")
|
||||
assert len(msgs) == 1
|
||||
assert isinstance(msgs[0], ToolMessage)
|
||||
assert msgs[0].content == "Step 3 replay"
|
||||
assert msgs[0].tool_call_id == "tc-1"
|
||||
assert msgs[0].name == "replay_steps"
|
||||
assert msgs[0].status == "error"
|
||||
|
||||
# Text-block lists pass through unchanged; other objects are stringified.
|
||||
blocks = [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]
|
||||
assert tool_result_messages("tc", blocks)[0].content == blocks
|
||||
assert tool_result_messages("tc", {"k": 1})[0].content == "{'k': 1}"
|
||||
|
||||
|
||||
def test_tool_carrier_puts_the_image_inside_the_tool_message():
|
||||
msgs = tool_result_messages("tc-2", SHOT, name="get_step_screenshot", llm=_llm("google"))
|
||||
assert len(msgs) == 1
|
||||
assert isinstance(msgs[0], ToolMessage)
|
||||
assert msgs[0].content == SHOT
|
||||
assert msgs[0].name == "get_step_screenshot"
|
||||
|
||||
|
||||
def test_human_carrier_splits_text_and_image_with_a_caption():
|
||||
msgs = tool_result_messages(
|
||||
"tc-3", SHOT, name="get_step_screenshot", status="success", llm=_llm("openai")
|
||||
)
|
||||
assert len(msgs) == 2
|
||||
tool_msg, human_msg = msgs
|
||||
assert isinstance(tool_msg, ToolMessage)
|
||||
assert tool_msg.content == "Screenshot of step 7 (pre-action) is attached."
|
||||
assert tool_msg.tool_call_id == "tc-3"
|
||||
assert isinstance(human_msg, HumanMessage)
|
||||
assert human_msg.content[0] == {
|
||||
"type": "text",
|
||||
"text": "[Screenshot returned by get_step_screenshot for step 7]",
|
||||
}
|
||||
assert human_msg.content[1] == IMAGE
|
||||
|
||||
|
||||
def test_explicit_carrier_wins_over_the_llm():
|
||||
msgs = tool_result_messages("tc", SHOT, image_carrier="human", llm=_llm("google"))
|
||||
assert len(msgs) == 2
|
||||
msgs = tool_result_messages("tc", SHOT, image_carrier="tool", llm=_llm("openai"))
|
||||
assert len(msgs) == 1
|
||||
@@ -0,0 +1,109 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""get_step_screenshot: pre / post / overlay, missing files, invalid variants."""
|
||||
|
||||
import base64
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from artemis.tools.history import get_step_screenshot, load_step_screenshot
|
||||
|
||||
|
||||
def _jpeg_file(tmp_path, name="pre.jpg", color="white"):
|
||||
path = tmp_path / name
|
||||
Image.new("RGB", (200, 400), color).save(path, format="JPEG")
|
||||
return path
|
||||
|
||||
|
||||
def _reader(pre=None, post=None, action=None, record=True):
|
||||
reader = MagicMock()
|
||||
reader.get_step_image_path.side_effect = lambda n, which="pre": pre if which == "pre" else post
|
||||
reader.get_step_record.return_value = SimpleNamespace(action_taken=action) if record else None
|
||||
return reader
|
||||
|
||||
|
||||
def test_pre_and_post_screenshots_are_attached_as_content_blocks(tmp_path):
|
||||
pre = _jpeg_file(tmp_path, "pre.jpg", "white")
|
||||
post = _jpeg_file(tmp_path, "post.jpg", "black")
|
||||
reader = _reader(pre=pre, post=post)
|
||||
|
||||
shot = load_step_screenshot(reader, 2, "pre")
|
||||
reader.get_step_image_path.assert_called_with(2, "pre")
|
||||
assert shot.image_bytes == pre.read_bytes()
|
||||
assert shot.description == "Screenshot of step 2 (pre-action) is attached."
|
||||
blocks = shot.to_content_blocks()
|
||||
assert blocks[0] == {"type": "text", "text": shot.description}
|
||||
assert blocks[1]["type"] == "image_url"
|
||||
url = blocks[1]["image_url"]["url"]
|
||||
assert url.startswith("data:image/jpeg;base64,")
|
||||
assert base64.b64decode(url.split(",", 1)[1]) == pre.read_bytes()
|
||||
|
||||
shot = load_step_screenshot(reader, 2, "post")
|
||||
reader.get_step_image_path.assert_called_with(2, "post")
|
||||
assert shot.image_bytes == post.read_bytes()
|
||||
assert "post-action" in shot.description
|
||||
|
||||
|
||||
def test_overlay_draws_the_recorded_raw_action(tmp_path):
|
||||
pre = _jpeg_file(tmp_path)
|
||||
reader = _reader(pre=pre, action={"action": "click", "target": [100, 200]})
|
||||
|
||||
shot = load_step_screenshot(reader, 2, "overlay")
|
||||
reader.get_step_image_path.assert_called_once_with(2, "pre")
|
||||
assert shot.overlay_drawn is True
|
||||
assert "action drawn on it" in shot.description
|
||||
assert shot.image_bytes != pre.read_bytes()
|
||||
|
||||
|
||||
def test_overlay_falls_back_to_plain_pre_without_action(tmp_path):
|
||||
pre = _jpeg_file(tmp_path)
|
||||
shot = load_step_screenshot(_reader(pre=pre, action=None), 2, "overlay")
|
||||
assert shot.overlay_drawn is False
|
||||
assert shot.image_bytes == pre.read_bytes()
|
||||
assert "plain screenshot" in shot.description
|
||||
|
||||
|
||||
def test_missing_file_and_missing_step_return_plain_text():
|
||||
shot = load_step_screenshot(_reader(pre=None, post=None), 3, "post")
|
||||
assert shot.image_bytes is None
|
||||
assert shot.to_content_blocks() == "No post-action screenshot recorded for step 3."
|
||||
|
||||
shot = load_step_screenshot(_reader(pre=None, record=False), 9, "pre")
|
||||
assert shot.to_content_blocks() == "Error: step 9 not found."
|
||||
|
||||
|
||||
def test_invalid_variant_and_step_number_are_rejected():
|
||||
shot = load_step_screenshot(_reader(), 1, "bogus")
|
||||
assert shot.image_bytes is None
|
||||
assert "'pre', 'post' or 'overlay'" in shot.description
|
||||
assert "must be an integer" in load_step_screenshot(_reader(), "one").description
|
||||
assert "no execution history" in load_step_screenshot(None, 1).description
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_execute_returns_blocks_or_text(tmp_path):
|
||||
pre = _jpeg_file(tmp_path)
|
||||
ctx = SimpleNamespace(data_engine=_reader(pre=pre))
|
||||
out = await get_step_screenshot.execute(ctx=ctx, step_number=1, which="pre")
|
||||
assert isinstance(out, list) and out[1]["type"] == "image_url"
|
||||
|
||||
out = await get_step_screenshot.execute(ctx=ctx, step_number=1, which="post")
|
||||
assert out == "No post-action screenshot recorded for step 1."
|
||||
|
||||
out = await get_step_screenshot.execute(ctx=SimpleNamespace(data_engine=None), step_number=1)
|
||||
assert "no execution history" in out
|
||||
@@ -0,0 +1,334 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""search_history boundaries and search correctness.
|
||||
|
||||
Hard boundaries under test: bounded result count, bounded response tokens,
|
||||
every result carrying a step number. Plus: keyword/step-range/notes/chunk
|
||||
filter correctness, the step-range ledger re-entry (full-width
|
||||
``build_action_ledger`` rows), and the screen-text surface: the step's own
|
||||
pre AND post screenshots plus screenshots its tool results embedded.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import re
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
from artemis.tools.history import (
|
||||
SearchHistoryTool,
|
||||
search_history,
|
||||
search_history_available,
|
||||
search_history_text,
|
||||
)
|
||||
from artemis.tools.history.search import result_search_text
|
||||
from artemis.utils.notes import save_note_content
|
||||
|
||||
|
||||
def _jpeg(color: str) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (64, 128), color).save(buf, format="JPEG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _make_engine(tmp_path):
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_execution_setup = MagicMock()
|
||||
mock_execution_setup.traces_path = str(tmp_path)
|
||||
mock_ctx.execution_setup = mock_execution_setup
|
||||
mock_ctx.device = None
|
||||
engine = DataEngine(mock_ctx)
|
||||
engine.start_session("search test session")
|
||||
return engine
|
||||
|
||||
|
||||
def _flush(engine):
|
||||
for t in list(engine._pending_threads):
|
||||
t.join()
|
||||
|
||||
|
||||
def _cfg(**overrides):
|
||||
values = {"enabled": True, "max_results": 5, "max_text_tokens": 2000, "screen_scan_steps": 150}
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def engine(tmp_path):
|
||||
engine = _make_engine(tmp_path)
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=_jpeg("red"),
|
||||
summary="Opened the login page.",
|
||||
action_taken={"action": "click", "target_text": "Login entry"},
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=_jpeg("green"),
|
||||
post_screenshot_bytes=_jpeg("blue"),
|
||||
summary="Typed the verification code 4711 into the input.",
|
||||
action_taken={"action": "input_text", "text": "4711"},
|
||||
last_execution_result={"status": "success"},
|
||||
operator_raw_thinking="The code arrived via SMS; entering 4711 now.",
|
||||
)
|
||||
engine.record_step(
|
||||
summary="A promo popup appeared and was dismissed.",
|
||||
action_taken={"action": "click", "target_text": "Close popup"},
|
||||
last_execution_result={"status": "failed", "error": "element vanished"},
|
||||
)
|
||||
_flush(engine)
|
||||
return engine
|
||||
|
||||
|
||||
def _result_blocks(text: str) -> list[str]:
|
||||
return [b for b in text.split("\n") if b.startswith("[")]
|
||||
|
||||
|
||||
def test_keyword_search_finds_step_and_carries_step_number(engine):
|
||||
out = search_history_text(engine, query="verification code", recall_config=_cfg())
|
||||
assert isinstance(out, str)
|
||||
assert "Step 2" in out
|
||||
assert "4711" in out
|
||||
# Every result header names a step / step range / step anchor.
|
||||
for header in _result_blocks(out):
|
||||
assert re.search(r"Step[s]? \d", header), header
|
||||
|
||||
|
||||
def test_step_range_filters_matches(engine):
|
||||
out = search_history_text(engine, query="popup", step_range=[1, 2], recall_config=_cfg())
|
||||
# Step 3 (the popup step) is outside the range: only the range ledger and
|
||||
# a no-match line may appear.
|
||||
assert "A promo popup appeared" not in out
|
||||
assert "No matches" in out
|
||||
|
||||
|
||||
def test_step_range_returns_full_width_ledger_rows(engine):
|
||||
"""The compressed-history marker line's re-entry point: a step range
|
||||
returns that range's full per-step action ledger."""
|
||||
out = search_history_text(engine, query="", step_range=[3, 1], recall_config=_cfg())
|
||||
assert "Action ledger for Steps 1–3" in out
|
||||
# Full-width build_action_ledger rows: step number + T+mm:ss offset +
|
||||
# semantic action + result phrase.
|
||||
assert re.search(r"- Step 1 \(T\+\d{2}:\d{2}\): .*Login entry", out)
|
||||
assert re.search(r"- Step 2 \(T\+\d{2}:\d{2}\): ", out)
|
||||
assert re.search(r"- Step 3 \(T\+\d{2}:\d{2}\): ", out)
|
||||
|
||||
|
||||
def test_result_count_is_clamped_by_config(engine):
|
||||
for i in range(8):
|
||||
engine.record_step(
|
||||
summary=f"Scrolled the anchorterm feed page {i}.",
|
||||
action_taken={"action": "swipe"},
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
_flush(engine)
|
||||
out = search_history_text(
|
||||
engine, query="anchorterm", max_results=50, recall_config=_cfg(max_results=3)
|
||||
)
|
||||
assert out.count("[Step ") == 3
|
||||
assert "more not shown" in out
|
||||
|
||||
|
||||
def test_response_is_truncated_at_token_budget(engine):
|
||||
for i in range(6):
|
||||
engine.record_step(
|
||||
summary=f"needleword page {i} " + "filler content " * 60,
|
||||
action_taken={"action": "click"},
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
_flush(engine)
|
||||
out = search_history_text(
|
||||
engine, query="needleword", max_results=6, recall_config=_cfg(max_text_tokens=150)
|
||||
)
|
||||
assert len(out) <= 150 * 4 + 200 # budget + truncation notice
|
||||
assert "truncated at the search token budget" in out
|
||||
|
||||
|
||||
def test_notes_are_searchable_and_anchored_to_a_step(engine):
|
||||
save_note_content(engine.base_dir, "login_flow", "The OTP entry lives behind the SMS tab.")
|
||||
out = search_history_text(engine, query="OTP entry", recall_config=_cfg())
|
||||
assert "[Note 'login_flow'" in out
|
||||
assert re.search(r"as of Step \d|last written at Step \d", out)
|
||||
|
||||
|
||||
def test_chunk_rows_are_searchable(engine):
|
||||
engine.record_history_chunk(
|
||||
start_step_number=1,
|
||||
end_step_number=2,
|
||||
version=1,
|
||||
status="ready",
|
||||
band2=" - Steps 1–2: walked the chunkneedle flow",
|
||||
band3="- Step 1 (T+00:01): click -> executed",
|
||||
rendered_text=(
|
||||
"[Chunk 1 | Steps 1–2]\n② Compressed step summary\n"
|
||||
" - Steps 1–2: walked the chunkneedle flow\n"
|
||||
"③ Step action ledger\n- Step 1 (T+00:01): click -> executed"
|
||||
),
|
||||
)
|
||||
_flush(engine)
|
||||
out = search_history_text(engine, query="chunkneedle", recall_config=_cfg())
|
||||
assert "[History chunk | Steps 1–2" in out
|
||||
|
||||
|
||||
def test_foreground_app_stamp_joins_search_surface(tmp_path):
|
||||
"""A package-name query hits the record_step foreground_app stamp."""
|
||||
engine = _make_engine(tmp_path)
|
||||
engine.record_step(
|
||||
summary="Opened the alarms list.",
|
||||
action_taken={"action": "click", "target_text": "Alarm"},
|
||||
ui_tree=[{"package": "com.google.android.deskclock", "bounds": "[0,0][9,9]"}],
|
||||
)
|
||||
_flush(engine)
|
||||
|
||||
out = search_history_text(engine, query="deskclock", recall_config=_cfg())
|
||||
assert "Step 1" in out
|
||||
assert "No matches" not in out
|
||||
|
||||
|
||||
def test_tool_calls_of_a_step_are_searchable(engine):
|
||||
engine.record_step(
|
||||
summary="Asked the explorer.",
|
||||
action_taken={"action": "click", "target_text": "Save"},
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
engine.record_trace(
|
||||
type="tool",
|
||||
name="ask_explorer",
|
||||
payload={"args": {"question": "toolneedle?"}, "result": "The resultneedle is ON."},
|
||||
step_id=engine.last_recorded_step_id,
|
||||
)
|
||||
_flush(engine)
|
||||
assert "[Step 4 " in search_history_text(engine, query="toolneedle", recall_config=_cfg())
|
||||
assert "[Step 4 " in search_history_text(engine, query="resultneedle", recall_config=_cfg())
|
||||
|
||||
|
||||
# --- Screen text: pre + post screenshots and referenced screenshots ----------------------
|
||||
|
||||
|
||||
def test_post_screenshot_text_joins_the_search_surface(engine):
|
||||
"""OCR text that only exists on the step's post-action screenshot is found."""
|
||||
record = engine.get_step_record(2)
|
||||
engine.storage.update_image_data(
|
||||
record.post_image_name,
|
||||
ocr_result=[{"text": "Welcome back postneedle", "bounds": [0, 0, 10, 10]}],
|
||||
ui_tree=None,
|
||||
)
|
||||
out = search_history_text(engine, query="postneedle", recall_config=_cfg())
|
||||
assert "[Step 2 " in out
|
||||
|
||||
|
||||
def test_screen_scan_cap_limits_the_screenshot_sweep(engine):
|
||||
record = engine.get_step_record(1)
|
||||
engine.storage.update_image_data(
|
||||
record.pre_image_name,
|
||||
ocr_result=[{"text": "capneedle", "bounds": [0, 0, 10, 10]}],
|
||||
ui_tree=None,
|
||||
)
|
||||
# Only the most recent step is scanned: Step 1's OCR is out of reach ...
|
||||
out = search_history_text(engine, query="capneedle", recall_config=_cfg(screen_scan_steps=1))
|
||||
assert "No matches" in out
|
||||
# ... unless the range narrows the sweep onto it.
|
||||
out = search_history_text(
|
||||
engine, query="capneedle", step_range=[1, 1], recall_config=_cfg(screen_scan_steps=1)
|
||||
)
|
||||
assert "[Step 1 " in out
|
||||
|
||||
|
||||
def test_embedded_screenshot_description_joins_the_search_surface(tmp_path):
|
||||
"""A step whose tool result embedded an earlier screenshot is found by
|
||||
that screenshot's OCR text: the image is described, not stripped."""
|
||||
engine = _make_engine(tmp_path)
|
||||
pre_bytes = _jpeg("red")
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=pre_bytes,
|
||||
summary="Opened the alarms list.",
|
||||
action_taken={"action": "click", "target_text": "Alarm"},
|
||||
ocr_result=[{"text": "Alarm 07:30 ocrneedle", "bounds": [0, 0, 10, 10]}],
|
||||
)
|
||||
engine.record_step(
|
||||
summary="Looked back at the alarm screen.",
|
||||
action_taken={"action": "click", "target_text": "Save"},
|
||||
)
|
||||
engine.record_trace(
|
||||
type="tool",
|
||||
name="get_step_screenshot",
|
||||
payload={
|
||||
"args": {"step_number": 1, "which": "pre"},
|
||||
"result": [
|
||||
{"type": "text", "text": "Screenshot of step 1 (pre-action) is attached."},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"<ImageRef: sha256={hashlib.sha256(pre_bytes).hexdigest()} length=1>"
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
step_id=engine.last_recorded_step_id,
|
||||
)
|
||||
_flush(engine)
|
||||
|
||||
out = search_history_text(engine, query="ocrneedle", recall_config=_cfg())
|
||||
assert "[Step 1 " in out
|
||||
assert "[Step 2 " in out
|
||||
assert "base64" not in out
|
||||
|
||||
|
||||
def test_result_search_text_strips_image_blocks():
|
||||
blocks = [
|
||||
{"type": "text", "text": "hello needle"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,QUJDRA=="}},
|
||||
]
|
||||
text = result_search_text(blocks)
|
||||
assert "hello needle" in text
|
||||
assert "base64" not in text and "QUJDRA" not in text
|
||||
assert result_search_text({"answer": "yes"}) == '{"answer": "yes"}'
|
||||
assert result_search_text(None) == ""
|
||||
|
||||
|
||||
# --- Tool surface ---------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_requires_query_or_range(engine):
|
||||
tool = SearchHistoryTool()
|
||||
out = await tool.execute(ctx=SimpleNamespace(data_engine=engine), query="", step_range=None)
|
||||
assert "needs a query and/or a step_range" in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_searches_and_never_returns_images(engine):
|
||||
out = await search_history.execute(
|
||||
ctx=SimpleNamespace(data_engine=engine), query="verification code"
|
||||
)
|
||||
assert isinstance(out, str)
|
||||
assert "Step 2" in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_without_engine_degrades():
|
||||
out = await search_history.execute(ctx=SimpleNamespace(data_engine=None), query="x")
|
||||
assert "no active execution history" in out
|
||||
|
||||
|
||||
def test_availability_requires_engine_and_config():
|
||||
assert search_history.is_available(None) is False
|
||||
assert search_history.is_available(SimpleNamespace(data_engine=None)) is False
|
||||
assert search_history.is_available(SimpleNamespace(data_engine=object())) is True
|
||||
assert search_history_available(SimpleNamespace(data_engine=object())) is True
|
||||
@@ -89,7 +89,7 @@ class TestCommitteeTool(unittest.IsolatedAsyncioTestCase):
|
||||
mock_response_tool_call = MagicMock()
|
||||
mock_response_tool_call.tool_calls = [
|
||||
{
|
||||
"name": "get_step_details",
|
||||
"name": "replay_steps",
|
||||
"args": {"start_step": 1, "end_step": 1},
|
||||
"id": "call_1",
|
||||
}
|
||||
|
||||
@@ -1,268 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""recall_history boundaries and search correctness (M4, design §10).
|
||||
|
||||
Hard boundaries under test: bounded result count, bounded response tokens,
|
||||
image recall limited to the real stored screenshots of a single step, and
|
||||
every result carrying a step number. Plus: keyword/step-range/notes filter
|
||||
correctness and the era-marker re-entry (a step_range returns the full-width
|
||||
``build_action_ledger`` rows for that range).
|
||||
"""
|
||||
|
||||
import io
|
||||
import re
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
from artemis.tools.history_recall import (
|
||||
RecallHistoryTool,
|
||||
recall_history,
|
||||
search_history,
|
||||
)
|
||||
from artemis.utils.notes import save_note_content
|
||||
|
||||
|
||||
def _jpeg(color: str) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (64, 128), color).save(buf, format="JPEG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _make_engine(tmp_path):
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_execution_setup = MagicMock()
|
||||
mock_execution_setup.traces_path = str(tmp_path)
|
||||
mock_ctx.execution_setup = mock_execution_setup
|
||||
mock_ctx.device = None
|
||||
engine = DataEngine(mock_ctx)
|
||||
engine.start_session("recall test session")
|
||||
return engine
|
||||
|
||||
|
||||
def _flush(engine):
|
||||
for t in list(engine._pending_threads):
|
||||
t.join()
|
||||
|
||||
|
||||
def _cfg(**overrides):
|
||||
values = {"enabled": True, "max_results": 5, "max_text_tokens": 2000, "max_image_steps": 1}
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def engine(tmp_path):
|
||||
engine = _make_engine(tmp_path)
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=_jpeg("red"),
|
||||
summary="Opened the login page.",
|
||||
action_taken={"action": "click", "target_text": "Login entry"},
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=_jpeg("green"),
|
||||
post_screenshot_bytes=_jpeg("blue"),
|
||||
summary="Typed the verification code 4711 into the input.",
|
||||
action_taken={"action": "input_text", "text": "4711"},
|
||||
last_execution_result={"status": "success"},
|
||||
operator_raw_thinking="The code arrived via SMS; entering 4711 now.",
|
||||
)
|
||||
engine.record_step(
|
||||
summary="A promo popup appeared and was dismissed.",
|
||||
action_taken={"action": "click", "target_text": "Close popup"},
|
||||
last_execution_result={"status": "failed", "error": "element vanished"},
|
||||
)
|
||||
_flush(engine)
|
||||
return engine
|
||||
|
||||
|
||||
def _result_blocks(text: str) -> list[str]:
|
||||
return [b for b in text.split("\n") if b.startswith("[")]
|
||||
|
||||
|
||||
def test_keyword_search_finds_step_and_carries_step_number(engine):
|
||||
out = search_history(engine, query="verification code", recall_config=_cfg())
|
||||
assert isinstance(out, str)
|
||||
assert "Step 2" in out
|
||||
assert "4711" in out
|
||||
# Every result header names a step / step range / step anchor.
|
||||
for header in _result_blocks(out):
|
||||
assert re.search(r"Step[s]? \d", header), header
|
||||
|
||||
|
||||
def test_step_range_filters_matches(engine):
|
||||
out = search_history(engine, query="popup", step_range=[1, 2], recall_config=_cfg())
|
||||
# Step 3 (the popup step) is outside the range: only the range ledger and
|
||||
# a no-match line may appear.
|
||||
assert "A promo popup appeared" not in out
|
||||
assert "No matches" in out
|
||||
|
||||
|
||||
def test_step_range_returns_full_width_ledger_rows(engine):
|
||||
"""The recall-only era period paragraph's '(Step-level ledger via
|
||||
recall_history for steps a–b)' re-entry point."""
|
||||
out = search_history(engine, query="", step_range=[1, 3], recall_config=_cfg())
|
||||
assert "Action ledger for Steps 1–3" in out
|
||||
# Full-width build_action_ledger rows: step number + T+mm:ss offset +
|
||||
# semantic action + result phrase.
|
||||
assert re.search(r"- Step 1 \(T\+\d{2}:\d{2}\): .*Login entry", out)
|
||||
assert re.search(r"- Step 2 \(T\+\d{2}:\d{2}\): ", out)
|
||||
assert re.search(r"- Step 3 \(T\+\d{2}:\d{2}\): ", out)
|
||||
|
||||
|
||||
def test_result_count_is_clamped_by_config(engine):
|
||||
for i in range(8):
|
||||
engine.record_step(
|
||||
summary=f"Scrolled the anchorterm feed page {i}.",
|
||||
action_taken={"action": "swipe"},
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
_flush(engine)
|
||||
out = search_history(
|
||||
engine, query="anchorterm", max_results=50, recall_config=_cfg(max_results=3)
|
||||
)
|
||||
assert out.count("[Step ") == 3
|
||||
assert "more not shown" in out
|
||||
|
||||
|
||||
def test_response_is_truncated_at_token_budget(engine):
|
||||
engine.record_step(
|
||||
summary="needleword " + "filler content " * 400,
|
||||
action_taken={"action": "click"},
|
||||
last_execution_result={"status": "success"},
|
||||
)
|
||||
_flush(engine)
|
||||
out = search_history(
|
||||
engine,
|
||||
query="needleword",
|
||||
include_details=True,
|
||||
recall_config=_cfg(max_text_tokens=150),
|
||||
)
|
||||
assert len(out) <= 150 * 4 + 200 # budget + truncation notice
|
||||
assert "truncated at the recall token budget" in out
|
||||
|
||||
|
||||
def test_notes_are_searchable_and_anchored_to_a_step(engine):
|
||||
save_note_content(engine.base_dir, "login_flow", "The OTP entry lives behind the SMS tab.")
|
||||
out = search_history(engine, query="OTP entry", recall_config=_cfg())
|
||||
assert "[Note 'login_flow'" in out
|
||||
assert re.search(r"as of Step \d|last written at Step \d", out)
|
||||
|
||||
|
||||
def test_chunk_rows_are_searchable(engine):
|
||||
engine.record_history_chunk(
|
||||
start_step_number=1,
|
||||
end_step_number=2,
|
||||
version=1,
|
||||
status="ready",
|
||||
band2=" - Steps 1–2: walked the chunkneedle flow",
|
||||
band3="- Step 1 (T+00:01): click -> executed",
|
||||
rendered_text=(
|
||||
"[Chunk 1 | Steps 1–2]\n② Compressed step summary\n"
|
||||
" - Steps 1–2: walked the chunkneedle flow\n"
|
||||
"③ Step action ledger\n- Step 1 (T+00:01): click -> executed"
|
||||
),
|
||||
)
|
||||
_flush(engine)
|
||||
out = search_history(engine, query="chunkneedle", recall_config=_cfg())
|
||||
assert "[History chunk | Steps 1–2" in out
|
||||
|
||||
|
||||
def test_images_only_for_a_single_step_and_only_real_files(engine):
|
||||
out = search_history(
|
||||
engine,
|
||||
query="verification code",
|
||||
include_images=True,
|
||||
recall_config=_cfg(),
|
||||
)
|
||||
assert isinstance(out, list)
|
||||
images = [b for b in out if b.get("type") == "image_url"]
|
||||
# Step 2 has a real pre and post screenshot on disk — but never more than
|
||||
# one step's worth of images.
|
||||
assert 1 <= len(images) <= 2
|
||||
labels = [
|
||||
b["text"]
|
||||
for b in out
|
||||
if b.get("type") == "text" and "screenshot of Step" in b.get("text", "")
|
||||
]
|
||||
assert all("Step 2" in label for label in labels)
|
||||
for block in images:
|
||||
assert block["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
||||
|
||||
|
||||
def test_images_disabled_by_config_cap(engine):
|
||||
out = search_history(
|
||||
engine,
|
||||
query="verification code",
|
||||
include_images=True,
|
||||
recall_config=_cfg(max_image_steps=0),
|
||||
)
|
||||
assert isinstance(out, str)
|
||||
|
||||
|
||||
def test_missing_image_files_yield_note_not_blocks(engine):
|
||||
# Step 3 recorded no screenshots at all.
|
||||
out = search_history(
|
||||
engine,
|
||||
query="promo popup",
|
||||
include_images=True,
|
||||
recall_config=_cfg(),
|
||||
)
|
||||
assert isinstance(out, list)
|
||||
assert not [b for b in out if b.get("type") == "image_url"]
|
||||
assert any(
|
||||
"no stored screenshots exist" in b.get("text", "") for b in out if b.get("type") == "text"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_requires_query_or_range(engine):
|
||||
tool = RecallHistoryTool()
|
||||
ctx = SimpleNamespace(data_engine=engine)
|
||||
out = await tool.execute(ctx=ctx, query="", step_range=None)
|
||||
assert "needs a query and/or a step_range" in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_without_engine_degrades():
|
||||
tool = RecallHistoryTool()
|
||||
out = await tool.execute(ctx=SimpleNamespace(data_engine=None), query="x")
|
||||
assert "no active DataEngine session" in out
|
||||
|
||||
|
||||
def test_availability_requires_engine_and_config():
|
||||
assert recall_history.is_available(None) is False
|
||||
assert recall_history.is_available(SimpleNamespace(data_engine=None)) is False
|
||||
assert recall_history.is_available(SimpleNamespace(data_engine=object())) is True
|
||||
|
||||
|
||||
def test_foreground_app_stamp_joins_search_surface(tmp_path):
|
||||
"""M5: a package-name query hits the record_step foreground_app stamp."""
|
||||
engine = _make_engine(tmp_path)
|
||||
engine.record_step(
|
||||
summary="Opened the alarms list.",
|
||||
action_taken={"action": "click", "target_text": "Alarm"},
|
||||
ui_tree=[{"package": "com.google.android.deskclock", "bounds": "[0,0][9,9]"}],
|
||||
)
|
||||
_flush(engine)
|
||||
|
||||
out = search_history(engine, query="deskclock", recall_config=_cfg())
|
||||
assert "Step 1" in out
|
||||
assert "No matches" not in out
|
||||
@@ -0,0 +1,104 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Coordinate-space contract of recorded actions.
|
||||
|
||||
Pro records pixels (``coordinate_space="pixel"``), Flash records the model's
|
||||
own 0–1000 values (``coordinate_space="normalized"``); every model-facing
|
||||
render normalizes, and the explicit marker — never a magnitude heuristic —
|
||||
is what makes that pass idempotent.
|
||||
"""
|
||||
|
||||
from artemis.utils.coordinates import (
|
||||
COORDINATE_SPACE_KEY,
|
||||
COORDINATE_SPACE_NORMALIZED,
|
||||
COORDINATE_SPACE_PIXEL,
|
||||
normalize_action_dict,
|
||||
normalize_any_structure,
|
||||
)
|
||||
|
||||
W, H = 1080, 2400
|
||||
|
||||
|
||||
def test_pixel_record_is_normalized_once_and_stamped():
|
||||
pixel = {
|
||||
"action": "tap",
|
||||
"coordinates": [540, 1200],
|
||||
"normalized_coordinates": [500, 500],
|
||||
COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
|
||||
}
|
||||
once = normalize_action_dict(pixel, W, H)
|
||||
assert once["coordinates"] == [500, 500]
|
||||
assert once[COORDINATE_SPACE_KEY] == COORDINATE_SPACE_NORMALIZED
|
||||
assert pixel["coordinates"] == [540, 1200] # input untouched
|
||||
|
||||
twice = normalize_action_dict(once, W, H)
|
||||
assert twice == once
|
||||
|
||||
|
||||
def test_normalized_record_is_returned_verbatim_even_when_values_look_like_pixels():
|
||||
"""No magnitude guessing: a Flash record of [320, 399] stays [320, 399]
|
||||
(the buggy double pass used to turn it into [296, 166])."""
|
||||
flash = {
|
||||
"action": "click",
|
||||
"coordinates": [320, 399],
|
||||
COORDINATE_SPACE_KEY: COORDINATE_SPACE_NORMALIZED,
|
||||
"args": {"target": [320, 399]},
|
||||
}
|
||||
out = normalize_action_dict(flash, W, H)
|
||||
assert out == flash
|
||||
assert out is not flash
|
||||
|
||||
big = dict(flash, coordinates=[1080, 2400])
|
||||
assert normalize_action_dict(big, W, H)["coordinates"] == [1080, 2400]
|
||||
|
||||
|
||||
def test_legacy_unstamped_record_is_converted_once_then_idempotent():
|
||||
legacy = {"action": "swipe", "coordinates": [540, 1800, 540, 600]}
|
||||
once = normalize_action_dict(legacy, W, H)
|
||||
assert once["coordinates"] == [500, 750, 500, 250]
|
||||
assert once[COORDINATE_SPACE_KEY] == COORDINATE_SPACE_NORMALIZED
|
||||
assert normalize_action_dict(once, W, H) == once
|
||||
|
||||
# A dict without coordinate fields gets no marker (nothing was converted).
|
||||
assert COORDINATE_SPACE_KEY not in normalize_action_dict({"action": "press_key"}, W, H)
|
||||
|
||||
|
||||
def test_normalize_any_structure_recurses_into_incident_containers():
|
||||
"""An execution incident holds an ``action`` *dict* — it is a container,
|
||||
not an action item, so its nested action is normalized too."""
|
||||
report = {
|
||||
"status": "failed",
|
||||
"execution": [
|
||||
{
|
||||
"action": "tap",
|
||||
"coordinates": [540, 1200],
|
||||
COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
|
||||
"attempts": ["Error"],
|
||||
}
|
||||
],
|
||||
"incident": {
|
||||
"kind": "exec_error",
|
||||
"action": {
|
||||
"action": "tap",
|
||||
"coordinates": [540, 1200],
|
||||
COORDINATE_SPACE_KEY: COORDINATE_SPACE_PIXEL,
|
||||
},
|
||||
},
|
||||
}
|
||||
out = normalize_any_structure(report, W, H)
|
||||
assert out["execution"][0]["coordinates"] == [500, 500]
|
||||
assert out["incident"]["action"]["coordinates"] == [500, 500]
|
||||
assert out["incident"]["kind"] == "exec_error"
|
||||
assert normalize_any_structure(out, W, H) == out
|
||||
@@ -621,3 +621,133 @@ def test_get_recent_subgoal_hashes_robust_milestone_exclusion(tmp_path):
|
||||
|
||||
# 5. Older failed subgoals from previously completed milestones (hash_failed_old)
|
||||
assert hash_failed_old not in keep_hashes
|
||||
|
||||
|
||||
# --- Step replay (shared renderer for replay_steps / MCP view_step_details) ----------
|
||||
|
||||
|
||||
def _replay_step(tool_result, **overrides):
|
||||
step = {
|
||||
"step_id": "step_4",
|
||||
"step_number": 4,
|
||||
"relative_time": "12.0s",
|
||||
"summary": "Confirmed the Wi-Fi toggle state via the explorer.",
|
||||
"action_taken": {"action": "click", "target": [0.5, 0.5], "target_text": "Save"},
|
||||
"last_execution_result": {"status": "success"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": "Need to confirm the toggle before saving."},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "ask_explorer",
|
||||
"args": {"question": "Is the Wi-Fi toggle on?"},
|
||||
"result": tool_result,
|
||||
},
|
||||
{"type": "tool_call", "name": "click", "args": {"target": [0.5, 0.5]}, "result": "ok"},
|
||||
],
|
||||
}
|
||||
step.update(overrides)
|
||||
return step
|
||||
|
||||
|
||||
def test_render_step_replay_shows_summary_thoughts_and_full_tool_results():
|
||||
from artemis.utils.task_tree import render_step_replay
|
||||
|
||||
long_result = "The Wi-Fi toggle is ON and the SSID row reads 'HomeNet'. " * 60 # ~3.5k chars
|
||||
out = render_step_replay(_replay_step(long_result))
|
||||
|
||||
assert "- **Step 4 (Start: 12.0s)**" in out
|
||||
assert "[Screen]: Confirmed the Wi-Fi toggle state via the explorer." in out
|
||||
assert "Need to confirm the toggle before saving." in out
|
||||
# The tool call is replayed verbatim: name, args and the whole result.
|
||||
assert "`ask_explorer(" in out
|
||||
assert long_result in out
|
||||
# Action tools are the planned action, not a tool-call line.
|
||||
assert "[Planned Action]: Tapped 'Save' at [0.5, 0.5]" in out
|
||||
assert "`click(" not in out
|
||||
|
||||
|
||||
def test_render_step_replay_clamps_only_pathological_results():
|
||||
from artemis.utils.task_tree import REPLAY_RESULT_CHARS, render_step_replay
|
||||
|
||||
huge = "y" * (REPLAY_RESULT_CHARS + 500)
|
||||
out = render_step_replay(_replay_step(huge))
|
||||
assert huge not in out
|
||||
assert ("y" * REPLAY_RESULT_CHARS) + "..." in out
|
||||
|
||||
|
||||
def test_live_window_keeps_the_tight_tool_result_clamp():
|
||||
"""The Operator's live context window still clamps tool results tightly;
|
||||
only the replay renderer is loose."""
|
||||
from artemis.utils.task_tree import LIVE_RESULT_CHARS
|
||||
|
||||
long_result = "z" * 3000
|
||||
out = build_plan_and_history(
|
||||
"- [ ] Check toggle", [_replay_step(long_result)], "default", last_n_detailed=1
|
||||
)
|
||||
assert long_result not in out
|
||||
assert ("z" * LIVE_RESULT_CHARS) + "..." in out
|
||||
# No screen-description line in the live detailed view.
|
||||
assert "[Screen]:" not in out
|
||||
|
||||
|
||||
def test_render_step_replay_reports_missing_screen_description_status():
|
||||
from artemis.utils.task_tree import render_step_replay
|
||||
|
||||
pending = render_step_replay(
|
||||
_replay_step("ok", summary=None, extra_metadata={"summary_status": "pending"})
|
||||
)
|
||||
assert "[Screen]: (screen description pending)" in pending
|
||||
failed = render_step_replay(
|
||||
_replay_step("ok", summary="", extra_metadata={"summary_status": "failed"})
|
||||
)
|
||||
assert "[Screen]: (screen description unavailable)" in failed
|
||||
assert "[Screen]" not in render_step_replay(_replay_step("ok", summary=None))
|
||||
|
||||
|
||||
def test_render_step_replay_shows_described_images_in_tool_results():
|
||||
from artemis.utils.task_tree import render_step_replay
|
||||
|
||||
result = [
|
||||
{"type": "text", "text": "--- pre-action screenshot of Step 1 ---"},
|
||||
{"type": "text", "text": "[screenshot: pre-action of Step 1]", "image_name": "ab" * 32},
|
||||
]
|
||||
out = render_step_replay(_replay_step(result))
|
||||
assert "[screenshot: pre-action of Step 1]" in out
|
||||
|
||||
|
||||
def test_format_action_clean_reads_flash_arguments_and_maps_manage_app():
|
||||
"""Read Flash action arguments from ``args`` when formatting app launches."""
|
||||
from artemis.utils.task_tree import format_action_clean
|
||||
|
||||
assert (
|
||||
format_action_clean(
|
||||
{
|
||||
"action": "manage_app",
|
||||
"coordinates": None,
|
||||
"args": {"action": "launch", "app_name": "com.android.settings"},
|
||||
}
|
||||
)
|
||||
== "Launched app 'com.android.settings'"
|
||||
)
|
||||
assert (
|
||||
format_action_clean({"action": "manage_app", "args": {"action": "stop", "app_name": "x"}})
|
||||
== "Stopped app 'x'"
|
||||
)
|
||||
# Pro's own shape keeps rendering as before.
|
||||
assert format_action_clean({"action": "launch_app", "app_name": "x"}) == "Launched app 'x'"
|
||||
assert (
|
||||
format_action_clean({"action": "manage_app", "app_name": "x", "intent": "clear"})
|
||||
== "Managed app 'x' (action: clear)"
|
||||
)
|
||||
# Top-level fields win over args; args only fill the gaps.
|
||||
assert (
|
||||
format_action_clean(
|
||||
{
|
||||
"action": "click",
|
||||
"coordinates": [1, 2],
|
||||
"target_text": "Save",
|
||||
"args": {"target": [9, 9]},
|
||||
}
|
||||
)
|
||||
== "Tapped 'Save' at [1, 2]"
|
||||
)
|
||||
@@ -0,0 +1,132 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Recording playlist manifest: session-relative segment offsets."""
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.utils.video import RECORDING_MANIFEST_VERSION, write_recording_manifest
|
||||
|
||||
|
||||
def _probe(durations: dict[str, float]):
|
||||
async def probe(path):
|
||||
return {"duration": durations[path.name], "width": 1080, "height": 1920}
|
||||
|
||||
return AsyncMock(side_effect=probe)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manifest_carries_session_offsets_and_restart_gaps(tmp_path):
|
||||
seg0 = tmp_path / "recording.mp4"
|
||||
seg1 = tmp_path / "recording_001.mp4"
|
||||
seg2 = tmp_path / "recording_002.mp4"
|
||||
for path in (seg0, seg1, seg2):
|
||||
path.write_bytes(b"mp4")
|
||||
|
||||
# Recording started 1.2s after the session; scrcpy restart gaps of 1.5s and 0.8s.
|
||||
offsets = {seg0: 1.2, seg1: 12.7, seg2: 18.5}
|
||||
|
||||
with patch(
|
||||
"artemis.utils.video.probe_video_segment",
|
||||
_probe({seg0.name: 10.0, seg1.name: 5.0, seg2.name: 4.0}),
|
||||
):
|
||||
manifest_path = await write_recording_manifest(tmp_path, [seg0, seg1, seg2], offsets)
|
||||
|
||||
assert manifest_path == tmp_path / "recording.json"
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
assert payload["version"] == RECORDING_MANIFEST_VERSION
|
||||
# Legacy back-to-back timeline is preserved for the existing player.
|
||||
assert payload["duration"] == 19.0
|
||||
assert [s["start"] for s in payload["segments"]] == [0.0, 10.0, 15.0]
|
||||
assert [s["duration"] for s in payload["segments"]] == [10.0, 5.0, 4.0]
|
||||
# Real session-relative timeline, including the gaps between segments.
|
||||
assert [s["offset_ms"] for s in payload["segments"]] == [1200, 12700, 18500]
|
||||
assert [s["duration_ms"] for s in payload["segments"]] == [10000, 5000, 4000]
|
||||
assert payload["session_offset_ms"] == 1200
|
||||
assert payload["session_end_ms"] == 22500
|
||||
assert [s["file"] for s in payload["segments"]] == [seg0.name, seg1.name, seg2.name]
|
||||
assert all(s["width"] == 1080 and s["height"] == 1920 for s in payload["segments"])
|
||||
assert not (tmp_path / "recording.part.json").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manifest_without_offsets_falls_back_to_back_to_back(tmp_path):
|
||||
seg0 = tmp_path / "recording.mp4"
|
||||
seg1 = tmp_path / "recording_001.mp4"
|
||||
seg0.write_bytes(b"mp4")
|
||||
seg1.write_bytes(b"mp4")
|
||||
|
||||
with patch(
|
||||
"artemis.utils.video.probe_video_segment",
|
||||
_probe({seg0.name: 3.25, seg1.name: 2.0}),
|
||||
):
|
||||
manifest_path = await write_recording_manifest(tmp_path, [seg0, seg1])
|
||||
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
assert [s["offset_ms"] for s in payload["segments"]] == [0, 3250]
|
||||
assert [s["duration_ms"] for s in payload["segments"]] == [3250, 2000]
|
||||
assert payload["session_offset_ms"] == 0
|
||||
assert payload["session_end_ms"] == 5250
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manifest_fills_missing_offset_after_known_segment(tmp_path):
|
||||
seg0 = tmp_path / "recording.mp4"
|
||||
seg1 = tmp_path / "recording_001.mp4"
|
||||
seg0.write_bytes(b"mp4")
|
||||
seg1.write_bytes(b"mp4")
|
||||
|
||||
with patch(
|
||||
"artemis.utils.video.probe_video_segment",
|
||||
_probe({seg0.name: 10.0, seg1.name: 5.0}),
|
||||
):
|
||||
manifest_path = await write_recording_manifest(tmp_path, [seg0, seg1], {seg0: 2.0})
|
||||
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
# Unknown offsets continue right after the previous segment (no gap assumed).
|
||||
assert [s["offset_ms"] for s in payload["segments"]] == [2000, 12000]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manifest_skips_invalid_segments_and_clamps_negative_offsets(tmp_path):
|
||||
good = tmp_path / "recording.mp4"
|
||||
broken = tmp_path / "recording_001.mp4"
|
||||
missing = tmp_path / "recording_002.mp4"
|
||||
good.write_bytes(b"mp4")
|
||||
broken.write_bytes(b"mp4")
|
||||
|
||||
with patch(
|
||||
"artemis.utils.video.probe_video_segment",
|
||||
_probe({good.name: 4.0, broken.name: 0.0}),
|
||||
):
|
||||
manifest_path = await write_recording_manifest(
|
||||
tmp_path, [good, broken, missing], {good: -0.4}
|
||||
)
|
||||
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
assert [s["file"] for s in payload["segments"]] == [good.name]
|
||||
assert payload["segments"][0]["offset_ms"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manifest_returns_none_without_valid_segments(tmp_path):
|
||||
with patch(
|
||||
"artemis.utils.video.probe_video_segment",
|
||||
AsyncMock(return_value={"duration": 0, "width": 0, "height": 0}),
|
||||
):
|
||||
assert await write_recording_manifest(tmp_path, [tmp_path / "nope.mp4"]) is None
|
||||
assert not (tmp_path / "recording.json").exists()
|
||||
Reference in new issue
Block a user