refactor: unify agent history and harden runtime diagnostics
Preserve user guidance through history compression, stabilize Flash index targets, and retire withdrawn checks without applying stale verdicts. Improve runtime diagnostics, tool responses, and history rendering. Simplify documentation and comments. Validation: 2076 tests passed in the full suite; 381 affected tests and 26 frontend tests passed after cleanup. Ruff and the quality ratchet passed.
This commit is contained in:
somew1nd committed
1 parent
e983180b5b
commit
5d92939d91
127 files changed
+13738
-1988
No files matched your search
@@ -0,0 +1,302 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""A Flash multi-action turn resolves every index against one element list.
|
||||
|
||||
Defect: each action's post-observation replaced ``state.indexed_elements``
|
||||
and the executor resolved index targets against the state at call time, so
|
||||
``click(3)`` then ``click(5)`` decided on one screen sent the second tap to
|
||||
whatever the screen after the first click listed as ``[5]``. The runner now
|
||||
snapshots the list once per turn and threads it through
|
||||
``McpActionExecutor.execute(index_elements=...)``; the state's list is still
|
||||
refreshed after every action (that is what the next observation shows).
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
from artemis.agents.flash.runner import FlashRunner, _TurnRecord
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.mcp.action_executor import McpActionExecutor, _ArgError
|
||||
from artemis.mcp.action_types import ActionResult, ObserveResult
|
||||
|
||||
|
||||
def _el(index: int, text: str, cx: int, cy: int) -> dict:
|
||||
return {
|
||||
"index": index,
|
||||
"center": [cx, cy],
|
||||
"text": text,
|
||||
"bounds": [cx - 100, cy - 50, cx + 100, cy + 50],
|
||||
"class": "android.widget.TextView",
|
||||
"resource_id": f"id/{text.lower().replace(' ', '_')}",
|
||||
"is_ocr": False,
|
||||
}
|
||||
|
||||
|
||||
# The list of the observation the model decided on (1080x2400 device).
|
||||
ORIGINAL = [
|
||||
_el(1, "Home", 540, 200),
|
||||
_el(2, "Search", 540, 600),
|
||||
_el(3, "Wi-Fi", 540, 1000),
|
||||
_el(4, "Bluetooth", 540, 1400),
|
||||
_el(5, "Display", 540, 1800),
|
||||
]
|
||||
# The screen after the first click: fewer elements, different numbering.
|
||||
REFRESHED = [_el(1, "Back", 100, 100), _el(2, "Wi-Fi toggle", 900, 300)]
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""An action session whose observations replay a scripted list sequence."""
|
||||
|
||||
started = True
|
||||
|
||||
def __init__(self, observations: list[list[dict]]):
|
||||
self.calls: list[tuple[str, dict]] = []
|
||||
self._observations = list(observations)
|
||||
|
||||
async def call(self, name, args):
|
||||
self.calls.append((name, dict(args)))
|
||||
return ActionResult.success(name, f"{name} dispatched")
|
||||
|
||||
async def observe(self, settle_ms=0):
|
||||
elements = self._observations.pop(0) if self._observations else []
|
||||
return ObserveResult(
|
||||
ok=True,
|
||||
elements_text="\n".join(f"[{e['index']}] {e['text']}" for e in elements),
|
||||
elements=elements,
|
||||
hierarchy_ok=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.llm_config = Mock()
|
||||
llm_cfg = Mock()
|
||||
llm_cfg.model = "gemini-2.5-flash"
|
||||
llm_cfg.temperature = 0.1
|
||||
ctx.llm_config.get_agent.return_value = llm_cfg
|
||||
ctx.device = Mock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
ctx.data_engine = None
|
||||
ctx.adb_client = None
|
||||
ctx.driver = Mock()
|
||||
return ctx
|
||||
|
||||
|
||||
def _runner(mock_context, observations):
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="Open Wi-Fi and Display")
|
||||
runner.summarizer = None
|
||||
runner.executor._session = _FakeSession(observations)
|
||||
# No device: the executor's observation already refreshed the state.
|
||||
runner._capture_post_screenshot = AsyncMock(return_value=None)
|
||||
return runner
|
||||
|
||||
|
||||
def _state(elements):
|
||||
state = State(initial_goal="g")
|
||||
state.indexed_elements = list(elements)
|
||||
state.indexed_points = [e["center"] for e in elements]
|
||||
return state
|
||||
|
||||
|
||||
def _click(index, tc_id):
|
||||
return {"name": "click", "args": {"target": index}, "id": tc_id}
|
||||
|
||||
|
||||
# --- Executor: the explicit list wins over the state ---------------------------------
|
||||
|
||||
|
||||
def _executor():
|
||||
ctx = Mock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
actuator = Mock()
|
||||
actuator.controller = Mock()
|
||||
return McpActionExecutor(ctx, actuator=actuator)
|
||||
|
||||
|
||||
def test_resolve_index_uses_the_given_list_and_falls_back_to_the_state():
|
||||
executor = _executor()
|
||||
state = _state(REFRESHED)
|
||||
|
||||
# None -> the state's current list (Pro operator / validator / explorer callers).
|
||||
point, recorded = executor._resolve_index(2, "click", state)
|
||||
assert point == [833, 125] and recorded["target_text"] == "Wi-Fi toggle"
|
||||
|
||||
# An explicit list: index 5 exists only there.
|
||||
point, recorded = executor._resolve_index(5, "click", state, index_elements=ORIGINAL)
|
||||
assert point == [500, 750] and recorded["target_text"] == "Display"
|
||||
assert recorded["target_resource_id"] == "id/display"
|
||||
|
||||
# The snapshot bounds the valid range, not the state.
|
||||
with pytest.raises(_ArgError, match="Active index range is 1 to 2"):
|
||||
executor._resolve_index(5, "click", state, index_elements=REFRESHED)
|
||||
_, _, _, recorded = executor._translate(
|
||||
"long_press", {"target": 4}, state, index_elements=ORIGINAL
|
||||
)
|
||||
assert recorded["target_text"] == "Bluetooth"
|
||||
_, wire, _, recorded = executor._translate(
|
||||
"input_text", {"target": 2, "text": "hi"}, state, index_elements=ORIGINAL
|
||||
)
|
||||
assert wire["target"] == [500, 250] and recorded["target_text"] == "Search"
|
||||
|
||||
|
||||
# --- Runner: one snapshot per turn ------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_two_index_clicks_in_one_turn_resolve_against_the_original_list(mock_context):
|
||||
"""``click(3)`` then ``click(5)``: the first click's observation lists only
|
||||
two elements, yet the second index still lands on the original ``[5]``
|
||||
(Display), with its semantics recorded from the original element."""
|
||||
runner = _runner(mock_context, observations=[REFRESHED, [_el(1, "Display page", 1, 1)]])
|
||||
state = _state(ORIGINAL)
|
||||
messages: list = []
|
||||
turn = _TurnRecord()
|
||||
|
||||
final, _, xml_list, _ = await runner._process_tool_calls(
|
||||
[_click(3, "tc-1"), _click(5, "tc-2")],
|
||||
state,
|
||||
messages,
|
||||
"thought",
|
||||
{},
|
||||
b"pre",
|
||||
"xml",
|
||||
0,
|
||||
turn,
|
||||
)
|
||||
|
||||
assert final is None
|
||||
session = runner.executor._session
|
||||
assert [args["target"] for _, args in session.calls] == [[500, 417], [500, 750]]
|
||||
assert [status for _, status, _ in turn.actions] == ["success", "success"]
|
||||
tool_msgs = [m for m in messages if isinstance(m, ToolMessage)]
|
||||
assert [m.status for m in tool_msgs] == ["success", "success"]
|
||||
# The state kept being refreshed: the NEXT observation shows the newest list.
|
||||
assert [e["text"] for e in state.indexed_elements] == ["Display page"]
|
||||
assert xml_list == "[1] Display page"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_second_index_beyond_the_refreshed_list_is_still_valid(mock_context):
|
||||
"""Without the snapshot ``click(5)`` after the refresh was an
|
||||
'Active index range is 1 to 2' error; with it the turn's own list is
|
||||
authoritative and the recorded metadata comes from that list."""
|
||||
runner = _runner(mock_context, observations=[REFRESHED, REFRESHED])
|
||||
state = _state(ORIGINAL)
|
||||
captured = []
|
||||
real_execute = runner.executor.execute
|
||||
|
||||
async def spy(name, args, tc_id, st, **kwargs):
|
||||
result = await real_execute(name, args, tc_id, st, **kwargs)
|
||||
captured.append(result)
|
||||
return result
|
||||
|
||||
runner.executor.execute = spy
|
||||
|
||||
await runner._process_tool_calls(
|
||||
[_click(4, "tc-1"), _click(5, "tc-2")],
|
||||
state,
|
||||
[],
|
||||
"thought",
|
||||
{},
|
||||
b"pre",
|
||||
"xml",
|
||||
0,
|
||||
_TurnRecord(),
|
||||
)
|
||||
|
||||
assert [r.status for r in captured] == ["success", "success"]
|
||||
assert [r.metadata["target_coordinates"] for r in captured] == [[500, 583], [500, 750]]
|
||||
assert [r.metadata["target_semantics"]["target_text"] for r in captured] == [
|
||||
"Bluetooth",
|
||||
"Display",
|
||||
]
|
||||
assert captured[1].metadata["target_semantics"]["target_bounds"] == [407, 729, 593, 771]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explorer_candidates_registered_before_an_action_join_the_snapshot(mock_context):
|
||||
"""``ask_explorer`` appends to the state's list in place; before any
|
||||
action that list is the snapshot, so the new index resolves in the same
|
||||
turn even though the action's observation then replaces the state list."""
|
||||
runner = _runner(mock_context, observations=[REFRESHED])
|
||||
state = _state(ORIGINAL)
|
||||
|
||||
async def fake_explorer(query, context_feedback, st):
|
||||
st.indexed_points.append([540, 2200])
|
||||
st.indexed_elements.append(_el(6, "Hidden gear", 540, 2200))
|
||||
return "Found 'gear' at index 6.", True
|
||||
|
||||
runner.executor._ask_explorer = fake_explorer
|
||||
|
||||
await runner._process_tool_calls(
|
||||
[
|
||||
{"name": "ask_explorer", "args": {"query": "gear"}, "id": "tc-0"},
|
||||
_click(6, "tc-1"),
|
||||
],
|
||||
state,
|
||||
[],
|
||||
"thought",
|
||||
{},
|
||||
b"pre",
|
||||
"xml",
|
||||
0,
|
||||
_TurnRecord(),
|
||||
)
|
||||
|
||||
session = runner.executor._session
|
||||
assert [args["target"] for _, args in session.calls] == [[500, 917]]
|
||||
assert [e["text"] for e in state.indexed_elements] == ["Back", "Wi-Fi toggle"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explorer_after_an_action_moves_the_snapshot_to_the_list_it_numbered(mock_context):
|
||||
"""After an action replaced the state list, the explorer numbers its
|
||||
candidates against that replaced list; the snapshot follows so the index
|
||||
the explorer just handed out resolves correctly."""
|
||||
runner = _runner(mock_context, observations=[REFRESHED, REFRESHED])
|
||||
state = _state(ORIGINAL)
|
||||
|
||||
async def fake_explorer(query, context_feedback, st):
|
||||
st.indexed_points.append([200, 2000])
|
||||
st.indexed_elements.append(_el(len(st.indexed_elements) + 1, "Gear", 200, 2000))
|
||||
return "Found.", True
|
||||
|
||||
runner.executor._ask_explorer = fake_explorer
|
||||
|
||||
await runner._process_tool_calls(
|
||||
[
|
||||
_click(3, "tc-1"),
|
||||
{"name": "ask_explorer", "args": {"query": "gear"}, "id": "tc-2"},
|
||||
_click(3, "tc-3"), # the explorer's index on the refreshed list
|
||||
],
|
||||
state,
|
||||
[],
|
||||
"thought",
|
||||
{},
|
||||
b"pre",
|
||||
"xml",
|
||||
0,
|
||||
_TurnRecord(),
|
||||
)
|
||||
|
||||
session = runner.executor._session
|
||||
assert [args["target"] for _, args in session.calls] == [[500, 417], [185, 833]]
|
||||
@@ -13,8 +13,8 @@
|
||||
# limitations under the License.
|
||||
|
||||
"""§1.3 assembly discipline: prompts are functions of configuration x scenario.
|
||||
Both check gates off => zero context pollution; a plan carrying check lines
|
||||
gets the behavior explainer regardless of switches."""
|
||||
Both check gates off => zero context pollution; with a gate on, the check-line
|
||||
guidance lives once in the static system prompt, worded for that gate."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
@@ -23,10 +23,10 @@ from unittest.mock import MagicMock
|
||||
import pytest
|
||||
|
||||
from artemis.agents.operator.prompts import (
|
||||
CheckItemsExplainerPromptComponent,
|
||||
FeedbackPromptComponent,
|
||||
PromptBuilder,
|
||||
TemplatePromptComponent,
|
||||
render_transcript_static_system,
|
||||
)
|
||||
from artemis.agents.planner.planner import build_planner_system_blocks
|
||||
from artemis.context import ArtemisContext, ExecutionSetup
|
||||
@@ -95,35 +95,47 @@ def _state(**overrides):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explainer_renders_iff_plan_has_check_lines(tmp_path):
|
||||
component = CheckItemsExplainerPromptComponent()
|
||||
async def test_check_line_guidance_lives_in_the_static_prompt_once():
|
||||
"""The check-line guidance is a static-prompt section gated on the check
|
||||
gates (never a per-turn observation block), worded for the active gate."""
|
||||
off = await _render_main_template(disable_checker=True)
|
||||
assert "*Check Lines*" not in off
|
||||
assert "never declare a check passed" not in off
|
||||
|
||||
# Plan without check lines -> nothing rendered
|
||||
builder = PromptBuilder()
|
||||
ctx = _mock_ctx(tmp_path, plan="- [ ] G\n")
|
||||
await component(builder, _state(), ctx)
|
||||
assert builder.human_parts == []
|
||||
final_only = await _render_main_template(disable_midway_checks=True, disable_final_check=False)
|
||||
assert final_only.count("*Check Lines*") == 1
|
||||
assert "never declare a check passed" in final_only
|
||||
assert "restores deleted ones unless user guidance called for the change" in final_only
|
||||
# No midway repair loop -> no repair budget to recite, and the grammar says so.
|
||||
assert "reopens its milestone at most" not in final_only
|
||||
assert "there is no midway repair loop" in final_only
|
||||
|
||||
# Plan with check lines -> explainer rendered
|
||||
builder2 = PromptBuilder()
|
||||
ctx2 = _mock_ctx(tmp_path, plan="- [ ] G\n - verify: V\n")
|
||||
await component(builder2, _state(), ctx2)
|
||||
joined = "\n".join(p for p in builder2.human_parts if isinstance(p, str))
|
||||
assert "check lines" in joined
|
||||
assert "never declare a check passed yourself" in joined
|
||||
assert "automatically restored" in joined
|
||||
midway = await _render_main_template(disable_midway_checks=False, checkpoint_max_repairs=3)
|
||||
assert midway.count("*Check Lines*") == 1
|
||||
assert "reopens its milestone at most 3 times" in midway
|
||||
assert "- finding:" in midway
|
||||
|
||||
|
||||
def test_transcript_static_system_recites_the_repair_budget_from_setup():
|
||||
ctx = _mock_ctx(disable_midway_checks=False, checkpoint_max_repairs=5)
|
||||
ctx.actuator = None
|
||||
prompts = json.loads(OPERATOR_JSON.read_text(encoding="utf-8"))
|
||||
text = render_transcript_static_system(prompts, ctx, _state(initial_goal="G"))
|
||||
assert text.count("*Check Lines*") == 1
|
||||
assert "reopens its milestone at most 5 times" in text
|
||||
|
||||
quiet = _mock_ctx(disable_checker=True)
|
||||
quiet.actuator = None
|
||||
assert "*Check Lines*" not in render_transcript_static_system(
|
||||
prompts, quiet, _state(initial_goal="G")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explainer_is_content_driven_not_switch_driven(tmp_path):
|
||||
"""A resumed old plan with check lines still gets the explanation even when
|
||||
every check switch is off."""
|
||||
component = CheckItemsExplainerPromptComponent()
|
||||
builder = PromptBuilder()
|
||||
ctx = _mock_ctx(tmp_path, plan="- [ ] G\n - assert: A\n", disable_checker=True)
|
||||
await component(builder, _state(), ctx)
|
||||
joined = "\n".join(p for p in builder.human_parts if isinstance(p, str))
|
||||
assert "check lines" in joined
|
||||
async def test_static_prompt_declares_the_user_guidance_channel_once():
|
||||
text = await _render_main_template(disable_checker=True)
|
||||
assert text.count("--- User Guidance ---") == 1
|
||||
assert "outranks the task plan and its check lines" in text
|
||||
|
||||
|
||||
async def _render_main_template(**setup_kwargs) -> str:
|
||||
|
||||
@@ -400,7 +400,7 @@ async def test_replay_steps_result_is_a_plain_tool_message(tmp_path):
|
||||
assert tool_msg.name == "replay_steps"
|
||||
assert "- **Step 3 (Start: 9.0s)**" in tool_msg.content
|
||||
assert "[Screen]: Saved the alarm." in tool_msg.content
|
||||
assert "[Planned Action]: Tapped 'Save' at [0.5, 0.5]" in tool_msg.content
|
||||
assert "[Action]: Tapped 'Save' at [0.5, 0.5]" in tool_msg.content
|
||||
|
||||
|
||||
# --- Streamed-reasoning transcript (replay of the live Thought/Work text) -------------
|
||||
@@ -647,6 +647,89 @@ def test_format_history_marks_self_described_targets_only():
|
||||
assert "target_resource_id" not in text
|
||||
|
||||
|
||||
def test_format_history_uses_session_offsets_and_skips_empty_steps():
|
||||
"""Rows carry the session-relative ``T+mm:ss`` clock every agent shares
|
||||
(from the step timestamp, else from the engine's ``97.7s`` label), and a
|
||||
step with neither summary nor action renders no row."""
|
||||
from artemis.agents.checker.checker import _format_history
|
||||
|
||||
ctx = _mock_ctx()
|
||||
ctx.data_engine.session_start_time = 1000.0
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = [
|
||||
{"step_number": 1, "timestamp": 1097.7, "summary": "Opened the app"},
|
||||
{"step_number": 2, "relative_time": "12.4s", "summary": "", "action_taken": None},
|
||||
{
|
||||
"step_number": 3,
|
||||
"relative_time": "130.0s",
|
||||
"summary": None,
|
||||
"action_taken": {"action": "click", "coordinates": [1, 2], "target_text": "Login"},
|
||||
},
|
||||
]
|
||||
|
||||
lines = _format_history(ctx).splitlines()
|
||||
assert len(lines) == 2
|
||||
assert lines[0] == "- Step 1 (T+01:37): Opened the app"
|
||||
assert lines[1].startswith("- Step 3 (T+02:10): ")
|
||||
assert "'Login'" in lines[1]
|
||||
|
||||
|
||||
async def _final_check_human_text(ctx) -> str:
|
||||
report = CheckReport(verdicts=[])
|
||||
structured = MagicMock()
|
||||
structured.ainvoke = AsyncMock(return_value=report)
|
||||
llm = MagicMock()
|
||||
llm.with_structured_output.return_value = structured
|
||||
response = MagicMock()
|
||||
response.tool_calls = []
|
||||
acomplete_mock = AsyncMock(return_value=response)
|
||||
|
||||
with (
|
||||
patch("artemis.agents.checker.checker.get_llm", return_value=llm),
|
||||
patch("artemis.agents.checker.checker.acomplete", new=acomplete_mock),
|
||||
patch(
|
||||
"artemis.agents.checker.checker._capture_final_screen",
|
||||
new=AsyncMock(return_value=(None, "elements")),
|
||||
),
|
||||
):
|
||||
await run_final_check(ctx, goal="the goal", plan_text="- [x] G", ledger=[], check_items=[])
|
||||
|
||||
messages = acomplete_mock.await_args.args[1]
|
||||
human = messages[-1]
|
||||
return "\n".join(
|
||||
block["text"] for block in human.content if isinstance(block, dict) and "text" in block
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_review_lists_user_guidance_from_the_step_records():
|
||||
"""Every mid-run user instruction reaches the final review, read from the
|
||||
step it was stamped on, so the goal is audited as amended by it."""
|
||||
ctx = _mock_ctx()
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = [
|
||||
{"step_number": 1, "relative_time": "3.0s", "summary": "s1", "extra_metadata": {}},
|
||||
{
|
||||
"step_number": 2,
|
||||
"relative_time": "65.0s",
|
||||
"summary": "s2",
|
||||
"extra_metadata": {"injected_instruction": "Skip the login"},
|
||||
},
|
||||
]
|
||||
text = await _final_check_human_text(ctx)
|
||||
section = text.split("# User guidance received during the run\n", 1)[1].split("\n\n", 1)[0]
|
||||
assert section == '- Step 2 (T+01:05): "Skip the login"'
|
||||
# The system prompt audits the goal as amended by that guidance.
|
||||
from artemis.agents.checker.checker import _load_prompts
|
||||
|
||||
assert "as amended by the user guidance" in _load_prompts()["final_guide"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_review_reports_no_guidance_explicitly():
|
||||
ctx = _mock_ctx()
|
||||
text = await _final_check_human_text(ctx)
|
||||
assert "# User guidance received during the run\n(none)\n" in text
|
||||
|
||||
|
||||
# --- tool results: status is structural, never sniffed from the words -----------------
|
||||
|
||||
|
||||
|
||||
@@ -332,3 +332,39 @@ async def test_run_tool_message_status_is_structural(
|
||||
assert messages[0].tool_call_id == "c1"
|
||||
assert messages[0].status == expected_status
|
||||
assert messages[0].content == str(result)
|
||||
|
||||
|
||||
# --- submit_answer rendering ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_submit_answer_renders_numbered_steps(mock_context):
|
||||
agent = Diagnoser(mock_context)
|
||||
call = {
|
||||
"name": "submit_answer",
|
||||
"args": {
|
||||
"analysis": "Airplane mode blocks the toggle.",
|
||||
"actionable_steps": ["Turn off airplane mode", "Retry the toggle"],
|
||||
},
|
||||
}
|
||||
|
||||
outcome = agent._handle_submit_call(call, [])
|
||||
|
||||
assert outcome == (
|
||||
"Analysis: Airplane mode blocks the toggle.\n"
|
||||
"Actionable steps:\n"
|
||||
"1. Turn off airplane mode\n"
|
||||
"2. Retry the toggle"
|
||||
)
|
||||
|
||||
|
||||
def test_submit_answer_without_steps_omits_section(mock_context):
|
||||
agent = Diagnoser(mock_context)
|
||||
call = {
|
||||
"name": "submit_answer",
|
||||
"args": {"analysis": "Nothing to fix.", "actionable_steps": []},
|
||||
}
|
||||
|
||||
outcome = agent._handle_submit_call(call, [])
|
||||
|
||||
assert outcome == "Analysis: Nothing to fix."
|
||||
assert "Actionable" not in outcome
|
||||
@@ -154,7 +154,14 @@ def test_render_burst_abort_and_exec_error():
|
||||
|
||||
plain = _disappeared(kind=KIND_EXEC_ERROR, category="general")
|
||||
text = render_execution_incident(plain, STEPS)
|
||||
assert "was dispatched, but the device/executor reported" in text
|
||||
# A failed action is quoted as an intent, never as a done outcome, and
|
||||
# the sentence does not claim it was dispatched.
|
||||
assert (
|
||||
"your planned action `tap 'Skip' at [950, 288]` could not be executed; the"
|
||||
" device/executor reported:" in text
|
||||
)
|
||||
assert "was dispatched" not in text
|
||||
assert "`Tapped 'Skip'" not in text
|
||||
|
||||
|
||||
def test_render_without_history_omits_the_trigger_hint():
|
||||
@@ -179,15 +186,17 @@ async def test_component_renders_only_while_an_incident_is_open():
|
||||
builder, SimpleNamespace(open_incident=_disappeared()), ctx, steps=STEPS
|
||||
)
|
||||
assert len(builder.human_parts) == 1
|
||||
# Its own text part: the transcript scrub keeps it verbatim across turns.
|
||||
assert builder.human_parts[0].startswith(EXECUTION_INCIDENT_MARKER)
|
||||
# Re-rendered every turn while open, so older copies leave the history.
|
||||
assert builder.ephemeral_indices == [0]
|
||||
|
||||
|
||||
def test_render_closed_notice_settles_the_intent():
|
||||
closed = dict(_disappeared(), closed_at_step=15)
|
||||
text = render_closed_incident(closed)
|
||||
assert text.startswith("--- Execution Incident (CLOSED at Step 15) ---")
|
||||
assert "opened at Step 12 on `Tapped 'Skip' at [950, 288]`" in text
|
||||
# The blocked action is quoted in intent form: it never happened.
|
||||
assert "opened at Step 12 on `tap 'Skip' at [950, 288]`" in text
|
||||
assert "already served, still pending, or no longer needed" in text
|
||||
|
||||
|
||||
@@ -201,6 +210,7 @@ async def test_component_renders_closed_notice_once_no_incident_is_open():
|
||||
await ExecutionIncidentPromptComponent()(builder, state, ctx, steps=STEPS)
|
||||
assert len(builder.human_parts) == 1
|
||||
assert builder.human_parts[0].startswith("--- Execution Incident (CLOSED at Step 15) ---")
|
||||
assert builder.ephemeral_indices == [0]
|
||||
|
||||
# An open incident takes precedence over a stale closed notice.
|
||||
builder = PromptBuilder()
|
||||
|
||||
@@ -14,13 +14,18 @@
|
||||
|
||||
"""Unit tests for Universal FlashRunner."""
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
import base64
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
import pytest
|
||||
from langchain_core.messages import HumanMessage, ToolMessage
|
||||
|
||||
from artemis.agents.explorer.constants import ASK_EXPLORER_DESCRIPTION
|
||||
from artemis.agents.flash.runner import FlashRunner
|
||||
from artemis.agents.validator.tool_declarations import ASK_EXPLORER_TOOL
|
||||
from artemis.agents.validator.tool_declarations import (
|
||||
ASK_EXPLORER_TOOL,
|
||||
ToolExecutionResult,
|
||||
)
|
||||
from artemis.context import ArtemisContext
|
||||
|
||||
|
||||
@@ -57,6 +62,138 @@ def test_flash_runner_tools_initialization(mock_context):
|
||||
assert "report_task_status" in tool_names
|
||||
# Validator's report_failure_analysis should NOT be in FlashRunner
|
||||
assert "report_failure_analysis" not in tool_names
|
||||
# Flash has no note-writing tool and its prompt never teaches notes:
|
||||
# the Validator set's note readers are not bound.
|
||||
assert "read_note" not in tool_names
|
||||
assert "list_notes" not in tool_names
|
||||
assert tool_names[:2] == ["click", "click_sequence"]
|
||||
|
||||
|
||||
# --- Post-action screenshot fallback (failed action / unknown tool) ----------------
|
||||
|
||||
|
||||
def _screen(elements):
|
||||
return SimpleNamespace(
|
||||
base64=base64.b64encode(b"POST-IMG").decode("ascii"),
|
||||
elements=elements,
|
||||
width=1080,
|
||||
height=2400,
|
||||
)
|
||||
|
||||
|
||||
def _failed_click(tc_id="tc1"):
|
||||
return ToolExecutionResult(
|
||||
tool_call_id=tc_id,
|
||||
tool_name="click",
|
||||
status="error",
|
||||
text_summary="Error during click: Invalid target format: 3",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_observation_renders_the_indexed_ui_list(mock_context, tmp_path):
|
||||
"""After an action that produced no observation, the fallback capture must
|
||||
hand the next tail the same indexed minimal list the success path builds
|
||||
(never the raw element dict repr) and refresh the state like it."""
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
runner.controller = Mock()
|
||||
runner.controller.get_screen_data = AsyncMock(
|
||||
return_value=_screen(
|
||||
[
|
||||
{
|
||||
"text": "Settings",
|
||||
"content-desc": "",
|
||||
"bounds": "[40,120][1040,270]",
|
||||
"class": "android.widget.TextView",
|
||||
"resource-id": "android:id/title",
|
||||
}
|
||||
]
|
||||
)
|
||||
)
|
||||
state = Mock()
|
||||
state.latest_screenshot = "old.jpg"
|
||||
exec_result = _failed_click()
|
||||
|
||||
with patch("artemis.mcp.observation.get_temp_dir", return_value=tmp_path):
|
||||
post = await runner._capture_post_screenshot(
|
||||
exec_result, "click", frozenset({"click"}), state
|
||||
)
|
||||
|
||||
assert post == b"POST-IMG"
|
||||
assert isinstance(exec_result.ui_elements_text, str)
|
||||
assert exec_result.ui_elements_text == "[1] Text: 'Settings' | Bounds: [37,50][962,112]"
|
||||
assert "{'text'" not in exec_result.ui_elements_text
|
||||
# State refreshed exactly as the executor's success path does.
|
||||
assert state.indexed_elements[0]["index"] == 1
|
||||
assert state.indexed_elements[0]["text"] == "Settings"
|
||||
assert state.indexed_points == [state.indexed_elements[0]["center"]]
|
||||
assert state.latest_screenshot != "old.jpg"
|
||||
assert state.latest_screenshot.startswith(str(tmp_path))
|
||||
assert exec_result.screenshot_path == state.latest_screenshot
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_observation_is_skipped_for_non_action_tools(mock_context):
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
runner.controller = Mock()
|
||||
runner.controller.get_screen_data = AsyncMock(return_value=_screen([]))
|
||||
state = Mock()
|
||||
exec_result = ToolExecutionResult(
|
||||
tool_call_id="tc", tool_name="ask_explorer", status="success", text_summary="found"
|
||||
)
|
||||
|
||||
post = await runner._capture_post_screenshot(
|
||||
exec_result, "ask_explorer", frozenset({"click"}), state
|
||||
)
|
||||
|
||||
assert post is None
|
||||
assert exec_result.ui_elements_text is None
|
||||
runner.controller.get_screen_data.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_observation_keeps_the_executors_ui_list(mock_context, tmp_path):
|
||||
"""An action that reported a UI list but no screenshot keeps its own list."""
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
runner.controller = Mock()
|
||||
runner.controller.get_screen_data = AsyncMock(
|
||||
return_value=_screen([{"text": "Other", "bounds": "[0,0][100,100]"}])
|
||||
)
|
||||
exec_result = ToolExecutionResult(
|
||||
tool_call_id="tc",
|
||||
tool_name="click",
|
||||
status="success",
|
||||
text_summary="click executed",
|
||||
ui_elements_text="[1] Text: 'Mine' | Bounds: [0,0][10,10]",
|
||||
)
|
||||
|
||||
with patch("artemis.mcp.observation.get_temp_dir", return_value=tmp_path):
|
||||
post = await runner._capture_post_screenshot(
|
||||
exec_result, "click", frozenset({"click"}), None
|
||||
)
|
||||
|
||||
assert post == b"POST-IMG"
|
||||
assert exec_result.ui_elements_text == "[1] Text: 'Mine' | Bounds: [0,0][10,10]"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_observation_failure_is_contained(mock_context):
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
runner.controller = Mock()
|
||||
runner.controller.get_screen_data = AsyncMock(side_effect=RuntimeError("adb gone"))
|
||||
state = Mock()
|
||||
state.latest_screenshot = "old.jpg"
|
||||
exec_result = _failed_click()
|
||||
|
||||
post = await runner._capture_post_screenshot(exec_result, "click", frozenset({"click"}), state)
|
||||
|
||||
assert post is None
|
||||
assert exec_result.ui_elements_text is None
|
||||
assert state.latest_screenshot == "old.jpg"
|
||||
|
||||
|
||||
def test_flash_runner_executor_follows_the_flash_explorer_tier(mock_context):
|
||||
@@ -202,10 +339,12 @@ def test_flash_records_render_in_the_ledger_with_their_arguments(mock_context):
|
||||
typed = _record(runner, "input_text", {"target": [500, 600], "text": "hello"})
|
||||
assert format_actions_clean(typed) == "Inputted 'hello' into field at [500, 600]"
|
||||
|
||||
# A directional swipe names the direction first; the recorded canonical
|
||||
# path is the parenthesised detail.
|
||||
swipe_dir = _record(runner, "swipe", {"direction": "up"})
|
||||
assert format_actions_clean(swipe_dir) == "Swiped up"
|
||||
assert format_actions_clean(swipe_dir) == "Swiped up (from [600, 700] to [600, 300])"
|
||||
swipe_fa = _record(runner, "swipe", {"action": "down"})
|
||||
assert format_actions_clean(swipe_fa) == "Swiped down"
|
||||
assert format_actions_clean(swipe_fa) == "Swiped down (from [600, 300] to [600, 700])"
|
||||
swipe_coords = _record(runner, "swipe", {"coordinates": [556, 289, 556, 124]})
|
||||
assert format_actions_clean(swipe_coords) == "Swiped from [556, 289] to [556, 124]"
|
||||
|
||||
@@ -246,3 +385,174 @@ def test_flash_records_the_executors_target_semantics(mock_context):
|
||||
bare = _record(runner, "click", {"target": [320, 399]}, metadata={"target_semantics": {}})
|
||||
assert "target_description" not in bare
|
||||
assert "target_text" not in bare
|
||||
|
||||
|
||||
def test_flash_records_an_index_target_as_the_resolved_point_with_observed_semantics(
|
||||
mock_context,
|
||||
):
|
||||
"""An element index is not a coordinate: the record carries the normalized
|
||||
point the executor resolved it to, plus the element's observed fields in
|
||||
the shape Pro records (never a self-described label)."""
|
||||
from artemis.utils.task_tree import format_actions_clean
|
||||
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
|
||||
click = _record(
|
||||
runner,
|
||||
"click",
|
||||
{"target": 3},
|
||||
metadata={
|
||||
"target_coordinates": [500, 520],
|
||||
"target_semantics": {
|
||||
"target_text": "Wi-Fi",
|
||||
"target_bounds": [37, 480, 962, 560],
|
||||
"target_resource_id": "android:id/title",
|
||||
"target_class": "android.widget.TextView",
|
||||
},
|
||||
},
|
||||
)
|
||||
assert click["coordinates"] == [500, 520]
|
||||
assert click["normalized_coordinates"] == [500, 520]
|
||||
assert click["target_text"] == "Wi-Fi"
|
||||
assert click["target_bounds"] == [37, 480, 962, 560]
|
||||
assert "target_description" not in click
|
||||
assert click["args"] == {"target": 3} # the tool call itself, verbatim
|
||||
assert format_actions_clean(click) == "Tapped 'Wi-Fi' at [500, 520]"
|
||||
|
||||
typed = _record(
|
||||
runner,
|
||||
"input_text",
|
||||
{"target": 2, "text": "hello"},
|
||||
metadata={"target_coordinates": [500, 300], "target_semantics": {"target_text": "Search"}},
|
||||
)
|
||||
assert typed["coordinates"] == [500, 300]
|
||||
assert format_actions_clean(typed) == "Inputted 'hello' into 'Search' at [500, 300]"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_visual_lens_receives_the_recorded_action_shape(mock_context):
|
||||
"""The lens gets the recorded action minus its verb (what the Pro
|
||||
summarizer hands it), so ``manage_app``'s own ``action`` argument can no
|
||||
longer overwrite the action name in the rendered phrase."""
|
||||
from artemis.agents.flash.runner import _TurnRecord
|
||||
from artemis.agents.flash.summarizer import VisualStepSummarizer
|
||||
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
runner.summarizer = Mock()
|
||||
runner.executor.execute = AsyncMock(
|
||||
return_value=ToolExecutionResult(
|
||||
tool_call_id="tc",
|
||||
tool_name="manage_app",
|
||||
status="success",
|
||||
text_summary="Launched app 'Settings' (com.android.settings); foreground confirmed.",
|
||||
screenshot_bytes=b"post",
|
||||
metadata={"target_semantics": {}, "target_coordinates": None},
|
||||
)
|
||||
)
|
||||
args = {"action": "launch", "app_name": "Settings"}
|
||||
|
||||
await runner._execute_and_record_action(
|
||||
"manage_app", args, "tc", Mock(), [], "thought", {}, b"pre", "xml", 0, _TurnRecord()
|
||||
)
|
||||
|
||||
runner.summarizer.dispatch.assert_called_once()
|
||||
kwargs = runner.summarizer.dispatch.call_args.kwargs
|
||||
assert kwargs["action_name"] == "manage_app"
|
||||
assert "action" not in kwargs["action_args"]
|
||||
assert kwargs["action_args"]["args"] == args
|
||||
assert VisualStepSummarizer._action_phrase("manage_app", kwargs["action_args"]) == (
|
||||
"Launched app 'Settings'"
|
||||
)
|
||||
|
||||
|
||||
# --- Native thinking (thought summaries) reach the step record ----------------------
|
||||
|
||||
|
||||
def _thinking_response(content):
|
||||
from types import SimpleNamespace
|
||||
|
||||
return SimpleNamespace(content=content, tool_calls=[])
|
||||
|
||||
|
||||
def test_extract_response_thinking_reads_only_thinking_blocks(mock_context):
|
||||
"""``include_thoughts`` replies carry ``thinking`` blocks next to ``text``
|
||||
blocks; only the former are native thinking, and plain-string replies
|
||||
carry none."""
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
|
||||
blocks = [
|
||||
{"type": "thinking", "thinking": "**Verifying**\n\nMaps is open."},
|
||||
{"type": "text", "text": "Tapping the search bar."},
|
||||
{"type": "thinking", "thinking": " "},
|
||||
]
|
||||
assert runner._extract_response_thinking(_thinking_response(blocks)) == (
|
||||
"**Verifying**\n\nMaps is open."
|
||||
)
|
||||
assert runner._extract_response_text(_thinking_response(blocks)) == "Tapping the search bar."
|
||||
assert runner._extract_response_thinking(_thinking_response("plain reply")) == ""
|
||||
assert (
|
||||
runner._extract_response_thinking(_thinking_response([{"type": "text", "text": "t"}])) == ""
|
||||
)
|
||||
|
||||
|
||||
def test_record_action_step_persists_native_thinking(mock_context):
|
||||
from types import SimpleNamespace
|
||||
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
runner.ctx.data_engine = Mock()
|
||||
runner.ctx.data_engine.current_step_id = None
|
||||
runner.ctx.data_engine.record_step.return_value = "step-1"
|
||||
exec_result = SimpleNamespace(
|
||||
status="success", text_summary="click executed", metadata={}, ui_elements_text="ui"
|
||||
)
|
||||
|
||||
runner._record_action_step(
|
||||
"click",
|
||||
{"target": [1, 2]},
|
||||
exec_result,
|
||||
"thought",
|
||||
{},
|
||||
b"pre",
|
||||
["xml"],
|
||||
b"post",
|
||||
native_text="native summary",
|
||||
)
|
||||
kwargs = runner.ctx.data_engine.record_step.call_args.kwargs
|
||||
assert kwargs["operator_raw_thinking"] == "thought"
|
||||
assert kwargs["operator_native_thinking"] == "native summary"
|
||||
|
||||
runner._record_action_step(
|
||||
"click", {"target": [1, 2]}, exec_result, "thought", {}, b"pre", ["xml"], b"post"
|
||||
)
|
||||
assert runner.ctx.data_engine.record_step.call_args.kwargs["operator_native_thinking"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_report_persists_native_thinking(mock_context):
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
runner.ctx.data_engine = Mock()
|
||||
runner.ctx.data_engine.current_step_id = None
|
||||
runner.summarizer = None
|
||||
messages = []
|
||||
|
||||
report = await runner._finalize_task_report(
|
||||
"report_task_status",
|
||||
{"status": "completed"},
|
||||
"tc",
|
||||
"final text",
|
||||
{},
|
||||
b"pre",
|
||||
["xml"],
|
||||
messages,
|
||||
native_text="native summary",
|
||||
)
|
||||
assert report == {"status": "completed"}
|
||||
kwargs = runner.ctx.data_engine.record_step.call_args.kwargs
|
||||
assert kwargs["operator_raw_thinking"] == "final text"
|
||||
assert kwargs["operator_native_thinking"] == "native summary"
|
||||
assert isinstance(messages[-1], ToolMessage)
|
||||
@@ -17,33 +17,42 @@
|
||||
Covers: the ``video_analyzer`` availability gate (same as the Pro graph) and
|
||||
its prompt segment, the unbounded-by-default turn limit, the Pro-shaped
|
||||
observation tail (``# CURRENT OBSERVATION [T+mm:ss]`` header, screenshot,
|
||||
``--- Visible UI Elements ---`` list), and the end-to-end loop: committed
|
||||
turns carry text-only tool messages plus an ``--- Action Execution Result
|
||||
(T+mm:ss) ---`` message, every recorded step of a multi-action turn is
|
||||
registered on the committed turn, the final-turn tool restriction only
|
||||
applies to bounded loops, and a no-tool-call turn is nudged, not terminated.
|
||||
``--- Visible UI Elements ---`` list, ephemeral per-turn notices, the
|
||||
reasoning reminder after a silent turn), and the end-to-end loop: committed
|
||||
turns carry text-only tool messages and no execution-result message unless
|
||||
an action failed, every recorded step of a multi-action turn is registered
|
||||
on the committed turn, the final-turn tool restriction only applies to
|
||||
bounded loops, and a no-tool-call turn is nudged, not terminated.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import re
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
from unittest.mock import AsyncMock, Mock, PropertyMock, patch
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
|
||||
|
||||
from artemis.agents.flash.runner import FlashRunner, _TurnRecord
|
||||
from artemis.agents.flash.runner import _FINAL_TURN_WARNING, FlashRunner, _TurnRecord
|
||||
from artemis.agents.operator.prompts import (
|
||||
REASONING_REMINDER,
|
||||
USER_GUIDANCE_MARKER,
|
||||
UserGuidance,
|
||||
render_user_guidance,
|
||||
render_user_instruction,
|
||||
)
|
||||
from artemis.agents.validator.tool_declarations import ToolExecutionResult
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.memory.transcript import (
|
||||
EPHEMERAL_BLOCKS_KEY,
|
||||
EXECUTION_RESULT_MARKER,
|
||||
PRO_UI_LIST_MARKER,
|
||||
TranscriptLedger,
|
||||
)
|
||||
|
||||
OBSERVATION_HEADER_RE = re.compile(r"^# CURRENT OBSERVATION \[T\+\d{2,}:\d{2}\]$")
|
||||
RESULT_RE = re.compile(
|
||||
rf"^{re.escape(EXECUTION_RESULT_MARKER)} \(T\+\d{{2,}}:\d{{2}}\) ---\nStatus: dispatched$"
|
||||
FAILED_RESULT_RE = re.compile(
|
||||
rf"^{re.escape(EXECUTION_RESULT_MARKER)} \(T\+\d{{2,}}:\d{{2}}\) ---\nStatus: failed\n"
|
||||
)
|
||||
|
||||
|
||||
@@ -173,31 +182,139 @@ def test_turn_limit_semantics(mock_context):
|
||||
|
||||
|
||||
def test_observation_tail_has_pro_shape(mock_context):
|
||||
"""The objective lives in the system prompt only; the tail opens with the
|
||||
observation header on turn 1 exactly as on every later turn, and carries
|
||||
no reminder unless the previous turn was silent."""
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="Open Settings")
|
||||
ledger = TranscriptLedger()
|
||||
|
||||
tail = runner._build_tail(ledger, 1, b"IMG", "[1] Settings")
|
||||
texts = [b["text"] for b in tail.content if b["type"] == "text"]
|
||||
assert texts[0] == "Your objective is: Open Settings"
|
||||
assert OBSERVATION_HEADER_RE.match(texts[1])
|
||||
assert texts[2] == "--- Current Screenshot ---"
|
||||
assert texts[3] == f"{PRO_UI_LIST_MARKER}\n[1] Settings"
|
||||
assert OBSERVATION_HEADER_RE.match(texts[0])
|
||||
assert texts[1] == "--- Current Screenshot ---"
|
||||
assert texts[2] == f"{PRO_UI_LIST_MARKER}\n[1] Settings"
|
||||
assert len(texts) == 3
|
||||
assert sum(1 for b in tail.content if b["type"] == "image_url") == 1
|
||||
joined = "".join(texts)
|
||||
assert "Open Settings" not in joined and "objective" not in joined.lower()
|
||||
assert "CRITICAL RULE" not in joined and REASONING_REMINDER not in joined
|
||||
assert EPHEMERAL_BLOCKS_KEY not in tail.additional_kwargs
|
||||
|
||||
later = runner._build_tail(
|
||||
ledger, 5, None, None, injected="[INJECTED] stop", notices=["nudge"], is_final=True
|
||||
ledger,
|
||||
5,
|
||||
None,
|
||||
None,
|
||||
injected=UserGuidance(body='User instruction: "stop"', wrapper="[INJECTED]"),
|
||||
notices=["nudge"],
|
||||
is_final=True,
|
||||
)
|
||||
texts = [b["text"] for b in later.content]
|
||||
assert OBSERVATION_HEADER_RE.match(texts[0])
|
||||
assert "Your objective is" not in "".join(texts)
|
||||
assert "objective" not in "".join(texts).lower()
|
||||
assert texts[1:] == [
|
||||
"nudge",
|
||||
"[INJECTED] stop",
|
||||
"[WARNING] This is your final turn; only 'report_task_status' is available.",
|
||||
"[INJECTED]",
|
||||
'User instruction: "stop"',
|
||||
_FINAL_TURN_WARNING,
|
||||
]
|
||||
|
||||
|
||||
def test_per_turn_notices_are_marked_ephemeral(mock_context):
|
||||
"""Nudges, the user-guidance wrapper, the final-turn warning and the
|
||||
reasoning reminder are only meaningful for the turn they were built for:
|
||||
every one of them is flagged ephemeral (by block index) so the scrub edge
|
||||
drops them before the message freezes; the observation blocks never are."""
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="Open Settings")
|
||||
ledger = TranscriptLedger()
|
||||
guidance = render_user_guidance("stop", release_loop=False, has_plan=False)
|
||||
|
||||
tail = runner._build_tail(
|
||||
ledger,
|
||||
3,
|
||||
b"IMG",
|
||||
"[1] Settings",
|
||||
notices=["nudge"],
|
||||
injected=guidance,
|
||||
is_final=True,
|
||||
previous_turn_silent=True,
|
||||
)
|
||||
texts = [b.get("text") for b in tail.content]
|
||||
# header, screenshot label, image, UI list, then the per-turn notices with
|
||||
# the persistent instruction body right after its wrapper
|
||||
assert texts[4:] == [
|
||||
"nudge",
|
||||
guidance.wrapper,
|
||||
guidance.body,
|
||||
_FINAL_TURN_WARNING,
|
||||
REASONING_REMINDER,
|
||||
]
|
||||
assert tail.additional_kwargs[EPHEMERAL_BLOCKS_KEY] == [4, 5, 7, 8]
|
||||
|
||||
|
||||
def test_injected_instruction_body_outlives_its_wrapper(mock_context):
|
||||
"""Defect: a standing instruction ("don't send messages from now on")
|
||||
vanished after one turn because the whole guidance block was ephemeral.
|
||||
Now the wrapper is ephemeral and the verbatim body is a regular block, so
|
||||
it stays in the active window until the turn is chunked."""
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
ledger = TranscriptLedger()
|
||||
guidance = render_user_guidance(
|
||||
"don't send messages from now on", has_plan=False, offset_label="T+01:05"
|
||||
)
|
||||
|
||||
tail = runner._build_tail(ledger, 2, b"IMG", "[1] Send", injected=guidance)
|
||||
texts = [b.get("text") for b in tail.content]
|
||||
wrapper_index = texts.index(guidance.wrapper)
|
||||
body_index = texts.index(guidance.body)
|
||||
ephemeral = tail.additional_kwargs[EPHEMERAL_BLOCKS_KEY]
|
||||
assert wrapper_index in ephemeral
|
||||
assert body_index not in ephemeral
|
||||
assert body_index == wrapper_index + 1
|
||||
assert guidance.body == 'User instruction (T+01:05): "don\'t send messages from now on"'
|
||||
assert guidance.body == render_user_instruction(
|
||||
"don't send messages from now on", offset_label="T+01:05"
|
||||
)
|
||||
# The wrapper frames the quoted line but no longer carries the words.
|
||||
assert guidance.wrapper.startswith(USER_GUIDANCE_MARKER)
|
||||
assert "don't send messages" not in guidance.wrapper
|
||||
|
||||
# Scrub edge parity: once the turn has left the live position (a newer
|
||||
# committed observation exists) the wrapper is gone and the body is
|
||||
# still in the committed turn.
|
||||
ledger.stage_turn([tail, AIMessage(content="ok")])
|
||||
ledger.commit_staged(step_key="s1")
|
||||
ledger.stage_turn([runner._build_tail(ledger, 3, b"IMG", "[1] Send"), AIMessage(content="ok")])
|
||||
ledger.commit_staged(step_key="s2")
|
||||
rendered = ledger.render([runner._build_tail(ledger, 4, b"IMG", "[1] Send")])
|
||||
committed = next(
|
||||
m
|
||||
for m in rendered
|
||||
if isinstance(m, HumanMessage) and guidance.body in [b.get("text") for b in m.content]
|
||||
)
|
||||
committed_texts = [b.get("text", "") for b in committed.content if isinstance(b, dict)]
|
||||
assert guidance.wrapper not in committed_texts
|
||||
assert not any(t.startswith(USER_GUIDANCE_MARKER) for t in committed_texts)
|
||||
|
||||
|
||||
def test_reasoning_reminder_follows_a_silent_turn_only(mock_context):
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="g")
|
||||
ledger = TranscriptLedger()
|
||||
|
||||
quiet = runner._build_tail(ledger, 2, None, "[1] x", previous_turn_silent=False)
|
||||
assert REASONING_REMINDER not in [b["text"] for b in quiet.content]
|
||||
|
||||
reminded = runner._build_tail(ledger, 2, None, "[1] x", previous_turn_silent=True)
|
||||
texts = [b["text"] for b in reminded.content]
|
||||
assert texts[-1] == REASONING_REMINDER
|
||||
assert texts.count(REASONING_REMINDER) == 1
|
||||
assert reminded.additional_kwargs[EPHEMERAL_BLOCKS_KEY] == [len(texts) - 1]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Injected instruction: verbatim text for the record and the lens, wrapped
|
||||
# notice for the observation tail only
|
||||
@@ -209,8 +326,9 @@ async def test_injected_instruction_splits_verbatim_text_from_operator_notice(
|
||||
mock_context, tmp_path
|
||||
):
|
||||
"""The user's words are what the step record, the chunk ledger and the
|
||||
visual lens receive (Pro parity); the 'You MUST immediately ...' directive
|
||||
is operator-facing and rides on the observation tail alone."""
|
||||
visual lens receive (Pro parity); the shared ``--- User Guidance ---``
|
||||
block (the Pro rendering, without a plan to edit) rides on the
|
||||
observation tail alone."""
|
||||
import json
|
||||
|
||||
(tmp_path / "injected_instruction.json").write_text(
|
||||
@@ -225,11 +343,15 @@ async def test_injected_instruction_splits_verbatim_text_from_operator_notice(
|
||||
instruction, notice = await runner._read_injected_instruction()
|
||||
|
||||
assert instruction == "Skip the popup and log in"
|
||||
assert notice.startswith(
|
||||
"[REAL-TIME INJECTED INSTRUCTION from user]: Skip the popup and log in"
|
||||
assert notice == render_user_guidance(
|
||||
"Skip the popup and log in", release_loop=True, has_plan=False
|
||||
)
|
||||
assert "You MUST immediately follow this instruction" in notice
|
||||
assert "authorized stopping any ongoing monitoring loop" in notice
|
||||
assert notice.wrapper.startswith(USER_GUIDANCE_MARKER)
|
||||
assert notice.body == 'User instruction: "Skip the popup and log in"'
|
||||
assert "outranks your current milestones" in notice.wrapper
|
||||
assert "task plan" not in notice.wrapper
|
||||
assert "authorized stopping any ongoing monitoring loop" in notice.wrapper
|
||||
assert "REAL-TIME INJECTED" not in notice.wrapper and "You MUST" not in notice.wrapper
|
||||
assert not (tmp_path / "injected_instruction.json").exists()
|
||||
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
@@ -267,7 +389,7 @@ def _make_runner(mock_context, responses, max_turns=None):
|
||||
|
||||
runner._invoke_model = AsyncMock(side_effect=_invoke)
|
||||
runner.executor.execute = AsyncMock(
|
||||
side_effect=lambda name, args, tc_id, state: _exec_result(
|
||||
side_effect=lambda name, args, tc_id, state, **_: _exec_result(
|
||||
tc_id, name, f"IMG-{tc_id}".encode()
|
||||
)
|
||||
)
|
||||
@@ -289,9 +411,10 @@ _INITIAL_OBSERVATION = ("shot0.png", b"IMG0", "[1] Settings")
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_builds_prompt_from_ledger_with_session_offsets(mock_context):
|
||||
"""Turn 2 must see: the static system prefix, the committed turn 1 (its
|
||||
tail, the AI message, text-only tool messages and an execution-result
|
||||
message with a ``T+`` offset) and a fresh observation tail carrying the
|
||||
last action's post screenshot — the Pro transcript shape, with no cap."""
|
||||
tail, the AI message, text-only tool messages that already carry each
|
||||
action's outcome — no execution-result message repeats them) and a fresh
|
||||
observation tail carrying the last action's post screenshot — the Pro
|
||||
transcript shape, with no cap."""
|
||||
responses = [
|
||||
AIMessage(
|
||||
content="I will tap then wait.",
|
||||
@@ -327,33 +450,146 @@ async def test_run_builds_prompt_from_ledger_with_session_offsets(mock_context):
|
||||
tail1 = turn2_messages[1]
|
||||
assert isinstance(tail1, HumanMessage)
|
||||
tail1_texts = [b["text"] for b in tail1.content if b["type"] == "text"]
|
||||
assert tail1_texts[0] == "Your objective is: Open Settings"
|
||||
assert OBSERVATION_HEADER_RE.match(tail1_texts[1])
|
||||
assert OBSERVATION_HEADER_RE.match(tail1_texts[0])
|
||||
assert "objective" not in "".join(tail1_texts).lower()
|
||||
assert "Open Settings" in turn2_messages[0].content # stated once, in the prefix
|
||||
|
||||
assert isinstance(turn2_messages[2], AIMessage)
|
||||
tool_msgs = [m for m in turn2_messages if isinstance(m, ToolMessage)]
|
||||
assert [m.content for m in tool_msgs] == ["click executed", "wait_for_delay executed"]
|
||||
|
||||
result_msg = turn2_messages[5]
|
||||
assert isinstance(result_msg, HumanMessage)
|
||||
assert RESULT_RE.match(result_msg.content[0]["text"])
|
||||
# Every action succeeded: the tool messages are the outcome, nothing repeats them.
|
||||
assert not any(
|
||||
isinstance(m, HumanMessage)
|
||||
and any(EXECUTION_RESULT_MARKER in b.get("text", "") for b in m.content)
|
||||
for m in turn2_messages
|
||||
)
|
||||
|
||||
tail2 = turn2_messages[6]
|
||||
tail2 = turn2_messages[5]
|
||||
tail2_texts = [b["text"] for b in tail2.content if b["type"] == "text"]
|
||||
assert OBSERVATION_HEADER_RE.match(tail2_texts[0])
|
||||
assert "Your objective is" not in "".join(tail2_texts)
|
||||
assert "objective" not in "".join(tail2_texts).lower()
|
||||
images = [b for b in tail2.content if b["type"] == "image_url"]
|
||||
assert len(images) == 1
|
||||
assert images[0]["image_url"]["url"].endswith(base64.b64encode(b"IMG-tc2").decode())
|
||||
assert tail2_texts[1] == "--- Current Screenshot ---"
|
||||
assert tail2_texts[2] == f"{PRO_UI_LIST_MARKER}\n[1] next"
|
||||
assert len(turn2_messages) == 7
|
||||
assert len(turn2_messages) == 6
|
||||
|
||||
# Both actions of the multi-action turn were registered on the committed turn.
|
||||
ledger = mock_context.transcript_ledger
|
||||
assert ledger.unchunked_turns()[0]["step_keys"] == ["tc1", "tc2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_failed_action_turn_ends_with_the_error_result(mock_context):
|
||||
"""A failed action is the one case worth a result message: it carries the
|
||||
error the turn ended on, after the tool message."""
|
||||
responses = [
|
||||
AIMessage(
|
||||
content="Tapping.",
|
||||
tool_calls=[{"name": "click", "args": {"target": 99}, "id": "tc1"}],
|
||||
),
|
||||
_report(),
|
||||
]
|
||||
with (
|
||||
patch("artemis.controllers.unified_controller.get_driver"),
|
||||
patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(return_value=_INITIAL_OBSERVATION),
|
||||
),
|
||||
):
|
||||
runner = _make_runner(mock_context, responses, max_turns=0)
|
||||
runner.executor.execute = AsyncMock(
|
||||
return_value=ToolExecutionResult(
|
||||
tool_call_id="tc1",
|
||||
tool_name="click",
|
||||
status="error",
|
||||
text_summary="Error during click: Invalid target index 99.",
|
||||
)
|
||||
)
|
||||
runner._capture_post_screenshot = AsyncMock(return_value=None)
|
||||
await runner.run(State(initial_goal="Open Settings"))
|
||||
|
||||
turn2_messages = runner.calls[1]["messages"]
|
||||
result_msgs = [
|
||||
m
|
||||
for m in turn2_messages
|
||||
if isinstance(m, HumanMessage)
|
||||
and any(EXECUTION_RESULT_MARKER in b.get("text", "") for b in m.content)
|
||||
]
|
||||
assert len(result_msgs) == 1
|
||||
text = result_msgs[0].content[0]["text"]
|
||||
assert FAILED_RESULT_RE.match(text)
|
||||
assert "Invalid target index 99" in text
|
||||
assert isinstance(turn2_messages[turn2_messages.index(result_msgs[0]) - 1], ToolMessage)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_reminds_after_a_silent_turn_without_a_bounce(mock_context):
|
||||
"""The ledger judges the committed turn (sibling logic, authoritative
|
||||
here); the runner only reads ``last_turn_silent`` when it builds the
|
||||
next tail: one reminder block, marked ephemeral, one model call per turn."""
|
||||
responses = [
|
||||
AIMessage(
|
||||
content="", # bare tool call, no visible reasoning
|
||||
tool_calls=[
|
||||
{"name": "click", "args": {"target": 1}, "id": "tc1"},
|
||||
],
|
||||
),
|
||||
_report(),
|
||||
]
|
||||
with (
|
||||
patch("artemis.controllers.unified_controller.get_driver"),
|
||||
patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(return_value=_INITIAL_OBSERVATION),
|
||||
),
|
||||
patch.object(TranscriptLedger, "last_turn_silent", new_callable=PropertyMock) as silent,
|
||||
):
|
||||
silent.side_effect = lambda: silent.call_count > 1 # turn 1: False, later: True
|
||||
runner = _make_runner(mock_context, responses, max_turns=0)
|
||||
await runner.run(State(initial_goal="Open Settings"))
|
||||
|
||||
assert runner._invoke_model.await_count == 2
|
||||
tail1 = runner.calls[0]["messages"][-1]
|
||||
assert REASONING_REMINDER not in [b.get("text") for b in tail1.content]
|
||||
tail2 = runner.calls[1]["messages"][-1]
|
||||
texts = [b.get("text") for b in tail2.content]
|
||||
assert texts.count(REASONING_REMINDER) == 1
|
||||
assert tail2.additional_kwargs[EPHEMERAL_BLOCKS_KEY] == [texts.index(REASONING_REMINDER)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_calibrates_the_ledger_only_from_measured_usage(mock_context):
|
||||
"""A provider-reported prompt size calibrates the chars-per-token ratio
|
||||
with the sent messages; an estimated size (no usage metadata) records the
|
||||
base but never calibrates — the estimate is itself derived from character
|
||||
counts, so calibrating on it would be circular."""
|
||||
measured = AIMessage(
|
||||
content="Tapping.",
|
||||
tool_calls=[{"name": "click", "args": {"target": 1}, "id": "tc1"}],
|
||||
usage_metadata={"input_tokens": 1200, "output_tokens": 5, "total_tokens": 1205},
|
||||
)
|
||||
responses = [measured, _report()]
|
||||
with (
|
||||
patch("artemis.controllers.unified_controller.get_driver"),
|
||||
patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(return_value=_INITIAL_OBSERVATION),
|
||||
),
|
||||
patch.object(TranscriptLedger, "record_prompt_tokens") as record,
|
||||
):
|
||||
runner = _make_runner(mock_context, responses, max_turns=0)
|
||||
await runner.run(State(initial_goal="Open Settings"))
|
||||
|
||||
assert record.call_count == 2
|
||||
first_call, second_call = record.call_args_list
|
||||
assert first_call.args[0] == 1200 and first_call.kwargs["messages"] is not None
|
||||
# The report turn carries no usage metadata: estimated, recorded, not calibrated.
|
||||
assert second_call.args[0] >= 1 and second_call.kwargs["messages"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_final_turn_restricts_tools_and_fails_without_report(mock_context):
|
||||
responses = [AIMessage(content="I give up.", tool_calls=[])]
|
||||
@@ -411,11 +647,14 @@ def test_turn_record_result_is_none_for_helper_only_turns():
|
||||
assert _TurnRecord().result() is None
|
||||
|
||||
|
||||
def test_turn_record_result_is_dispatched_when_every_action_was_accepted():
|
||||
def test_turn_record_result_is_none_when_every_action_succeeded():
|
||||
"""The tool messages already say ``Swiped up.``; a ``Status: dispatched``
|
||||
line after them would only duplicate the outcome (and put two human
|
||||
messages back to back)."""
|
||||
record = _TurnRecord(
|
||||
actions=[("click", "success", "Tapped at [500, 500]"), ("swipe", "success", "Swiped up.")]
|
||||
)
|
||||
assert record.result() == {"status": "dispatched"}
|
||||
assert record.result() is None
|
||||
|
||||
|
||||
def test_turn_record_failed_result_keeps_an_executor_message_that_names_the_action():
|
||||
|
||||
@@ -12,13 +12,11 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for the Flash scrub-edge compressor (history redesign §3.2, M1).
|
||||
"""Tests for screenshot summaries, text cleanup, and frozen history.
|
||||
|
||||
Covers the M1 acceptance items: the legacy-equivalent success-path product
|
||||
(asserted directly since the legacy ``compress_flash_messages`` reference was
|
||||
removed in M5), the three race regressions (grace recovery, grace exhaustion
|
||||
without backfill, failed-summary placeholder), the freeze invariant (bytes
|
||||
behind the scrub edge never change), and depth-1 XML stripping equivalence.
|
||||
Covers pending-summary recovery and expiry, failed-summary placeholders,
|
||||
independent text and image depths, ephemeral-block remapping, and stable
|
||||
content after both cleanup passes.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -83,9 +81,10 @@ def test_success_path_product_behind_the_scrub_edge(mock_context):
|
||||
by the ``--- Historical Visual Transition ---`` summary block.
|
||||
|
||||
(Asserted directly; the legacy ``compress_flash_messages`` reference this
|
||||
was originally diffed against was removed in M5.) Intended M1 timing
|
||||
change: the swap happens at depth K, so the newest K-1 historical steps
|
||||
still carry their screenshots.
|
||||
was originally diffed against was removed in M5.) The swap happens at
|
||||
depth K, so the newest K-1 historical steps still carry their screenshots;
|
||||
their UI lists leave earlier, at the shallow text edge (depth 1), so only
|
||||
the live observation ever carries an indexed element list.
|
||||
"""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
for i in range(1, 7):
|
||||
@@ -120,8 +119,8 @@ def test_success_path_product_behind_the_scrub_edge(mock_context):
|
||||
assert _has_image(new_messages[6])
|
||||
assert "Tree 6" in str(new_messages[6].content)
|
||||
|
||||
# Intended timing change: step 5 (depth 2 < K) keeps its screenshot under
|
||||
# the scrub edge; its XML was still stripped at depth 1.
|
||||
# Step 5 (depth 2 < K) is past the text edge only: screenshot intact,
|
||||
# UI list already gone, summary not yet swapped in.
|
||||
assert _has_image(new_messages[5])
|
||||
assert "Visual summary 5." not in str(new_messages[5].content)
|
||||
assert "Tree 5" not in str(new_messages[5].content)
|
||||
@@ -249,10 +248,10 @@ def test_freeze_invariant_bytes_never_change(mock_context):
|
||||
assert "Late summary 2." not in " ".join(str(m.content) for m in messages)
|
||||
|
||||
|
||||
def test_xml_depth1_strip_matches_legacy_semantics(mock_context):
|
||||
"""Depth-1 XML stripping keeps combined-block prefixes and the live list."""
|
||||
def test_strip_at_the_edge_keeps_combined_block_prefix_and_live_list(mock_context):
|
||||
"""The XML strip at the edge keeps combined-block prefixes and the live list."""
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(summarizer=None, prune_history_xml=True)
|
||||
compressor = ScrubEdgeCompressor(summarizer=None, prune_history_xml=True, image_scrub_depth=2)
|
||||
|
||||
combined = ToolMessage(
|
||||
tool_call_id="tc1",
|
||||
@@ -267,13 +266,44 @@ def test_xml_depth1_strip_matches_legacy_semantics(mock_context):
|
||||
)
|
||||
_simulate_turns(compressor, messages, [combined, _step(2)])
|
||||
|
||||
# Historical combined block: prefix retained, tree stripped (the exact
|
||||
# Combined block at the edge: prefix retained, tree stripped (the exact
|
||||
# legacy ``compress_flash_messages`` product for this input).
|
||||
assert messages[0].content[0] == {"type": "text", "text": "Tapped Settings."}
|
||||
# Live observation keeps its UI list untouched.
|
||||
assert "Tree 2" in str(messages[1].content)
|
||||
|
||||
|
||||
def test_imageless_ui_list_carrier_is_stripped_as_soon_as_a_newer_observation_exists(
|
||||
mock_context,
|
||||
):
|
||||
"""A step whose screenshot failed still carries a UI list; it follows the
|
||||
text edge like any observation: while it is the newest observation it keeps
|
||||
the list, and it is stripped as soon as a newer observation exists — even
|
||||
when that newer one has no image either — so only the live observation ever
|
||||
carries an indexed element list."""
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(summarizer=None, image_scrub_depth=2)
|
||||
|
||||
def imageless(n: int) -> ToolMessage:
|
||||
return ToolMessage(
|
||||
tool_call_id=f"tc{n}",
|
||||
name="click",
|
||||
content=[
|
||||
{"type": "text", "text": f"Tapped {n}, screenshot failed."},
|
||||
{"type": "text", "text": f"--- UI Element List ---\nTree {n}"},
|
||||
],
|
||||
)
|
||||
|
||||
_simulate_turns(compressor, messages, [imageless(1)])
|
||||
assert "Tree 1" in str(messages[0].content) # newest observation: kept
|
||||
_simulate_turns(compressor, messages, [imageless(2)]) # screenshots keep failing
|
||||
assert messages[0].content == [{"type": "text", "text": "Tapped 1, screenshot failed."}]
|
||||
assert "Tree 2" in str(messages[1].content)
|
||||
_simulate_turns(compressor, messages, [_step(3)])
|
||||
assert messages[1].content == [{"type": "text", "text": "Tapped 2, screenshot failed."}]
|
||||
assert "Tree 3" in str(messages[2].content) # live observation untouched
|
||||
|
||||
|
||||
def test_prune_history_xml_disabled_keeps_ui_lists(mock_context):
|
||||
"""With the switch off, UI lists survive both depth-1 and freeze scrubs."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
@@ -317,3 +347,348 @@ def test_untracked_messages_are_never_touched(mock_context):
|
||||
assert messages[2].content == "thinking"
|
||||
# Image-only tool message froze with the defensive fallback text.
|
||||
assert messages[1].content == [{"type": "text", "text": "Action dispatched."}]
|
||||
|
||||
|
||||
def test_depth_override_tightens_once_and_deeper_pass_never_backfills(mock_context):
|
||||
"""A per-call depth override: a shallower pass scrubs the extra depths in
|
||||
one go; a deeper pass afterwards touches nothing already frozen."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
for i in range(1, 9):
|
||||
summarizer._step_inputs[f"tc{i}"] = {"step_number": i}
|
||||
summarizer._summaries[f"tc{i}"] = f"Summary {i}."
|
||||
messages: list = [_initial_observation()]
|
||||
compressor = ScrubEdgeCompressor(summarizer=summarizer, image_scrub_depth=3)
|
||||
|
||||
for i in range(1, 9):
|
||||
compressor.compress(messages, image_scrub_depth=6)
|
||||
messages.append(_step(i))
|
||||
compressor.compress(messages, image_scrub_depth=6)
|
||||
with_images = [m for m in messages if _has_image(m)]
|
||||
# Depth 6 with the live step at depth 1: tc4..tc7 (depths 5..2) keep
|
||||
# their screenshots, tc3 (depth 6) is scrubbed.
|
||||
assert [m.tool_call_id for m in with_images] == ["tc4", "tc5", "tc6", "tc7", "tc8"]
|
||||
|
||||
compressor.compress(messages, image_scrub_depth=3) # tighten: three more scrubbed at once
|
||||
with_images = [m for m in messages if _has_image(m)]
|
||||
assert [m.tool_call_id for m in with_images] == ["tc7", "tc8"]
|
||||
frozen = {idx: _content_fingerprint(messages[idx]) for idx in compressor._frozen}
|
||||
|
||||
compressor.compress(messages, image_scrub_depth=6) # relax: nothing restored
|
||||
assert [m.tool_call_id for m in messages if _has_image(m)] == ["tc7", "tc8"]
|
||||
assert {idx: _content_fingerprint(messages[idx]) for idx in frozen} == frozen
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Screenshot replacement, ephemeral blocks, and frozen content
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _labelled_step(i: int, *, footer: str | None = None) -> ToolMessage:
|
||||
"""A step in the Flash observation shape with the screenshot label block."""
|
||||
blocks = [
|
||||
{"type": "text", "text": f"Action {i} completed."},
|
||||
{"type": "text", "text": "--- Current Screenshot ---"},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG_{i}"}},
|
||||
{"type": "text", "text": f"--- UI Element List ---\nTree {i}"},
|
||||
]
|
||||
if footer is not None:
|
||||
blocks.append({"type": "text", "text": footer})
|
||||
return ToolMessage(tool_call_id=f"tc{i}", name="click", content=blocks)
|
||||
|
||||
|
||||
def test_summary_takes_the_images_place_and_the_label_leaves_with_it(mock_context):
|
||||
"""(i) The visual summary block sits exactly where the image block was —
|
||||
not appended after whatever followed — and no ``--- Current Screenshot ---``
|
||||
label is left dangling above it."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._summaries["tc1"] = "Summary 1."
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(summarizer=summarizer, image_scrub_depth=2)
|
||||
_simulate_turns(
|
||||
compressor, messages, [_labelled_step(1, footer="Footer note 1"), _labelled_step(2)]
|
||||
)
|
||||
|
||||
assert messages[0].content == [
|
||||
{"type": "text", "text": "Action 1 completed."},
|
||||
{"type": "text", "text": "--- Historical Visual Transition ---\nSummary 1."},
|
||||
{"type": "text", "text": "Footer note 1"},
|
||||
]
|
||||
assert "--- Current Screenshot ---" not in str(messages[0].content)
|
||||
# The live step keeps its label above its screenshot.
|
||||
assert messages[1].content[1] == {"type": "text", "text": "--- Current Screenshot ---"}
|
||||
|
||||
|
||||
def test_ephemeral_blocks_vanish_at_the_text_edge_and_not_before(mock_context):
|
||||
"""(ii) Blocks listed under ``EPHEMERAL_BLOCKS_KEY`` survive only while
|
||||
the message is the live observation; at the text edge (a newer observation
|
||||
exists) they are deleted together with the UI list, the screenshot stays
|
||||
until depth K; the consumed indices are cleared at the text edge."""
|
||||
from artemis.memory.transcript import EPHEMERAL_BLOCKS_KEY, mark_ephemeral
|
||||
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._summaries["tc1"] = "Summary 1."
|
||||
step1 = ToolMessage(
|
||||
tool_call_id="tc1",
|
||||
name="click",
|
||||
content=[
|
||||
{"type": "text", "text": "Action 1 completed."},
|
||||
{"type": "text", "text": "[reminder: state your reasoning]"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
|
||||
{"type": "text", "text": "--- UI Element List ---\nTree 1"},
|
||||
{"type": "text", "text": "[hint: valid this turn only]"},
|
||||
],
|
||||
)
|
||||
mark_ephemeral(step1, [1, 4])
|
||||
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(summarizer=summarizer, image_scrub_depth=3)
|
||||
_simulate_turns(compressor, messages, [step1])
|
||||
# Live observation: everything, ephemeral blocks included, still there.
|
||||
assert "[reminder: state your reasoning]" in str(messages[0].content)
|
||||
assert "[hint: valid this turn only]" in str(messages[0].content)
|
||||
assert messages[0].additional_kwargs[EPHEMERAL_BLOCKS_KEY] == [1, 4]
|
||||
|
||||
_simulate_turns(compressor, messages, [_step(2)]) # step 1 reaches depth 2
|
||||
# Text edge: notices and UI list gone, the screenshot stays (depth 2 < K).
|
||||
assert messages[0].content == [
|
||||
{"type": "text", "text": "Action 1 completed."},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
|
||||
]
|
||||
assert EPHEMERAL_BLOCKS_KEY not in messages[0].additional_kwargs
|
||||
|
||||
_simulate_turns(compressor, messages, [_step(3)]) # step 1 reaches depth 3
|
||||
assert messages[0].content == [
|
||||
{"type": "text", "text": "Action 1 completed."},
|
||||
{"type": "text", "text": "--- Historical Visual Transition ---\nSummary 1."},
|
||||
]
|
||||
assert EPHEMERAL_BLOCKS_KEY not in messages[0].additional_kwargs
|
||||
|
||||
|
||||
def test_grace_does_the_text_edits_at_k_once_and_only_the_image_swap_waits(mock_context):
|
||||
"""A pending summary keeps the screenshot (and its label) inside the grace
|
||||
window, but the strip and the ephemeral deletion happen at K exactly once;
|
||||
the later swap puts the summary where the image was and freezes."""
|
||||
from artemis.memory.transcript import mark_ephemeral
|
||||
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._step_inputs["tc1"] = {"step_number": 1} # pending
|
||||
step1 = _labelled_step(1, footer="[per-turn notice]")
|
||||
mark_ephemeral(step1, [4])
|
||||
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer, image_scrub_depth=2, pending_grace_steps=3
|
||||
)
|
||||
_simulate_turns(compressor, messages, [step1, _labelled_step(2)])
|
||||
assert messages[0].content == [
|
||||
{"type": "text", "text": "Action 1 completed."},
|
||||
{"type": "text", "text": "--- Current Screenshot ---"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
|
||||
]
|
||||
after_text_edits = _content_fingerprint(messages[0])
|
||||
_simulate_turns(compressor, messages, [_labelled_step(3)]) # still pending, in grace
|
||||
assert _content_fingerprint(messages[0]) == after_text_edits
|
||||
|
||||
summarizer._summaries["tc1"] = "Late but in time."
|
||||
_simulate_turns(compressor, messages, [_labelled_step(4)])
|
||||
assert messages[0].content == [
|
||||
{"type": "text", "text": "Action 1 completed."},
|
||||
{"type": "text", "text": "--- Historical Visual Transition ---\nLate but in time."},
|
||||
]
|
||||
assert 0 in compressor._frozen
|
||||
|
||||
|
||||
def test_messages_past_the_edge_are_byte_identical_on_every_later_pass(mock_context):
|
||||
"""(iii) Once a message is past the edge, compressing again — with new
|
||||
turns appended, depths relaxed or tightened, summaries arriving late —
|
||||
leaves it byte-identical. Each pass rewrites exactly the two messages
|
||||
that reached an edge (the text edge at depth 2, the screenshot edge at
|
||||
K) and nothing older."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
for i in range(1, 13):
|
||||
summarizer._step_inputs[f"tc{i}"] = {"step_number": i}
|
||||
summarizer._summaries[f"tc{i}"] = f"Summary {i}."
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(summarizer=summarizer, image_scrub_depth=3)
|
||||
|
||||
previous: list[str] = []
|
||||
for i in range(1, 13):
|
||||
messages.append(_labelled_step(i))
|
||||
compressor.compress(messages)
|
||||
current = [_content_fingerprint(m) for m in messages]
|
||||
# Everything before the messages that reached an edge is unchanged.
|
||||
changed = [idx for idx, fp in enumerate(previous) if current[idx] != fp]
|
||||
# The message reaching depth 2 (text edge) is index len-1 of the previous
|
||||
# pass's list, the one reaching depth 3 (screenshot edge) is len-2; the
|
||||
# new live message is not in that list yet.
|
||||
expected = [idx for idx in (len(previous) - 2, len(previous) - 1) if idx >= 0]
|
||||
assert changed == expected, changed
|
||||
previous = current
|
||||
|
||||
# No new messages: a repeated pass, relaxed or not, is a no-op byte for byte.
|
||||
compressor.compress(messages)
|
||||
compressor.compress(messages, image_scrub_depth=6)
|
||||
assert [_content_fingerprint(m) for m in messages] == previous
|
||||
# Tightening rewrites exactly the message at the new depth, nothing older.
|
||||
compressor.compress(messages, image_scrub_depth=2)
|
||||
changed = [i for i, fp in enumerate(previous) if _content_fingerprint(messages[i]) != fp]
|
||||
assert changed == [len(messages) - 2]
|
||||
|
||||
|
||||
def test_text_edge_deeper_than_screenshot_edge_keeps_the_edges_independent(mock_context):
|
||||
"""xml_scrub_depth=4 with K=3: the screenshot is resolved at depth 3 while
|
||||
the UI list stays until the message is deeper than 4; the message is only
|
||||
frozen once both edits have happened."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
for i in range(1, 8):
|
||||
summarizer._summaries[f"tc{i}"] = f"Summary {i}."
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(summarizer=summarizer, image_scrub_depth=3, xml_scrub_depth=4)
|
||||
_simulate_turns(compressor, messages, [_labelled_step(i) for i in range(1, 4)])
|
||||
# Step 1 at depth 3: image swapped, label gone, UI list still there.
|
||||
assert "Summary 1." in str(messages[0].content)
|
||||
assert not _has_image(messages[0])
|
||||
assert "--- Current Screenshot ---" not in str(messages[0].content)
|
||||
assert "Tree 1" in str(messages[0].content)
|
||||
|
||||
_simulate_turns(compressor, messages, [_labelled_step(4), _labelled_step(5)])
|
||||
# Step 1 at depth 5 (> 4): UI list gone now; the summary is untouched.
|
||||
assert "Tree 1" not in str(messages[0].content)
|
||||
assert messages[0].content == [
|
||||
{"type": "text", "text": "Action 1 completed."},
|
||||
{"type": "text", "text": "--- Historical Visual Transition ---\nSummary 1."},
|
||||
]
|
||||
frozen = _content_fingerprint(messages[0])
|
||||
_simulate_turns(compressor, messages, [_labelled_step(6), _labelled_step(7)])
|
||||
assert _content_fingerprint(messages[0]) == frozen
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ephemeral indices stay aligned when the image edge is shallower than the
|
||||
# text edge (a rewrite before the text edit shifts the later blocks)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _step_with_notices(i: int) -> ToolMessage:
|
||||
"""Flash observation shape with one ephemeral notice on each side of the
|
||||
screenshot and a body block *after* the notice that follows the image, so
|
||||
a stale index would delete the body instead of the notice."""
|
||||
from artemis.memory.transcript import mark_ephemeral
|
||||
|
||||
msg = ToolMessage(
|
||||
tool_call_id=f"tc{i}",
|
||||
name="click",
|
||||
content=[
|
||||
{"type": "text", "text": f"Action {i} completed."},
|
||||
{"type": "text", "text": f"[hint {i}: before the image]"},
|
||||
{"type": "text", "text": "--- Current Screenshot ---"},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG_{i}"}},
|
||||
{"type": "text", "text": f"--- UI Element List ---\nTree {i}"},
|
||||
{"type": "text", "text": f"[reminder {i}: after the image]"},
|
||||
{"type": "text", "text": f"Body note {i}"},
|
||||
],
|
||||
)
|
||||
mark_ephemeral(msg, [1, 5])
|
||||
return msg
|
||||
|
||||
|
||||
def test_image_edge_before_text_edge_remaps_ephemeral_indices_to_the_summary_layout(
|
||||
mock_context,
|
||||
):
|
||||
"""K=3 with xml_scrub_depth=4: the summary swap at depth 3 removes the
|
||||
label block, so the notice after the image moves up by one. The stored
|
||||
indices must follow it; the later text pass then deletes both notices and
|
||||
leaves the body block intact."""
|
||||
from artemis.memory.transcript import EPHEMERAL_BLOCKS_KEY
|
||||
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
for i in range(1, 6):
|
||||
summarizer._summaries[f"tc{i}"] = f"Summary {i}."
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(summarizer=summarizer, image_scrub_depth=3, xml_scrub_depth=4)
|
||||
_simulate_turns(compressor, messages, [_step_with_notices(i) for i in range(1, 4)])
|
||||
|
||||
# Image pass (depth 3): label + image -> summary; notices still present.
|
||||
assert messages[0].content == [
|
||||
{"type": "text", "text": "Action 1 completed."},
|
||||
{"type": "text", "text": "[hint 1: before the image]"},
|
||||
{"type": "text", "text": "--- Historical Visual Transition ---\nSummary 1."},
|
||||
{"type": "text", "text": "--- UI Element List ---\nTree 1"},
|
||||
{"type": "text", "text": "[reminder 1: after the image]"},
|
||||
{"type": "text", "text": "Body note 1"},
|
||||
]
|
||||
assert messages[0].additional_kwargs[EPHEMERAL_BLOCKS_KEY] == [1, 4]
|
||||
|
||||
_simulate_turns(compressor, messages, [_step_with_notices(4), _step_with_notices(5)])
|
||||
# Text pass (depth 5 > 4): notices and UI list gone, body block intact.
|
||||
assert messages[0].content == [
|
||||
{"type": "text", "text": "Action 1 completed."},
|
||||
{"type": "text", "text": "--- Historical Visual Transition ---\nSummary 1."},
|
||||
{"type": "text", "text": "Body note 1"},
|
||||
]
|
||||
assert EPHEMERAL_BLOCKS_KEY not in messages[0].additional_kwargs
|
||||
assert 0 in compressor._frozen
|
||||
|
||||
|
||||
def test_image_edge_before_text_edge_remaps_when_the_image_is_dropped(mock_context):
|
||||
"""Same ordering with no summary job at all: label and image both leave
|
||||
(two blocks removed), the notice after the image moves up by two."""
|
||||
from artemis.memory.transcript import EPHEMERAL_BLOCKS_KEY
|
||||
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(summarizer=None, image_scrub_depth=3, xml_scrub_depth=4)
|
||||
_simulate_turns(compressor, messages, [_step_with_notices(i) for i in range(1, 4)])
|
||||
|
||||
assert messages[0].content == [
|
||||
{"type": "text", "text": "Action 1 completed."},
|
||||
{"type": "text", "text": "[hint 1: before the image]"},
|
||||
{"type": "text", "text": "--- UI Element List ---\nTree 1"},
|
||||
{"type": "text", "text": "[reminder 1: after the image]"},
|
||||
{"type": "text", "text": "Body note 1"},
|
||||
]
|
||||
assert messages[0].additional_kwargs[EPHEMERAL_BLOCKS_KEY] == [1, 3]
|
||||
|
||||
_simulate_turns(compressor, messages, [_step_with_notices(4), _step_with_notices(5)])
|
||||
assert messages[0].content == [
|
||||
{"type": "text", "text": "Action 1 completed."},
|
||||
{"type": "text", "text": "Body note 1"},
|
||||
]
|
||||
assert EPHEMERAL_BLOCKS_KEY not in messages[0].additional_kwargs
|
||||
assert 0 in compressor._frozen
|
||||
|
||||
|
||||
def test_grace_swap_before_the_text_edge_remaps_ephemeral_indices(mock_context):
|
||||
"""Pending summary inside the grace window with the text edge still ahead:
|
||||
the keep-image revisit changes nothing (indices untouched); the late swap
|
||||
removes the label and remaps; the text pass then deletes the right blocks."""
|
||||
from artemis.memory.transcript import EPHEMERAL_BLOCKS_KEY
|
||||
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
summarizer._step_inputs["tc1"] = {"step_number": 1} # pending
|
||||
messages: list = []
|
||||
compressor = ScrubEdgeCompressor(
|
||||
summarizer=summarizer, image_scrub_depth=3, xml_scrub_depth=5, pending_grace_steps=3
|
||||
)
|
||||
_simulate_turns(compressor, messages, [_step_with_notices(i) for i in range(1, 4)])
|
||||
# Depth 3, pending, in grace: byte-identical, indices untouched.
|
||||
assert _has_image(messages[0])
|
||||
assert "--- Current Screenshot ---" in str(messages[0].content)
|
||||
assert messages[0].additional_kwargs[EPHEMERAL_BLOCKS_KEY] == [1, 5]
|
||||
|
||||
summarizer._summaries["tc1"] = "Late but in time."
|
||||
_simulate_turns(compressor, messages, [_step_with_notices(4)]) # depth 4: swap
|
||||
assert messages[0].content[2] == {
|
||||
"type": "text",
|
||||
"text": "--- Historical Visual Transition ---\nLate but in time.",
|
||||
}
|
||||
assert messages[0].additional_kwargs[EPHEMERAL_BLOCKS_KEY] == [1, 4]
|
||||
|
||||
_simulate_turns(compressor, messages, [_step_with_notices(5), _step_with_notices(6)])
|
||||
# Depth 6 > 5: text pass on the remapped layout.
|
||||
assert messages[0].content == [
|
||||
{"type": "text", "text": "Action 1 completed."},
|
||||
{"type": "text", "text": "--- Historical Visual Transition ---\nLate but in time."},
|
||||
{"type": "text", "text": "Body note 1"},
|
||||
]
|
||||
assert EPHEMERAL_BLOCKS_KEY not in messages[0].additional_kwargs
|
||||
assert 0 in compressor._frozen
|
||||
@@ -569,8 +569,9 @@ async def test_raw_lens_call_meters_llm_usage_without_touching_context_base(mock
|
||||
@pytest.mark.asyncio
|
||||
async def test_focus_block_and_provenance_marked_action_phrase_reach_the_lens(mock_context):
|
||||
"""The lens input opens with the action as every history reader sees it
|
||||
(self-described target marked), the verbatim arguments, the outcome, and
|
||||
the operator focus block; the system prompt teaches how to use it."""
|
||||
(self-described target marked), the outcome, and the operator focus
|
||||
block; the phrase already carries every argument, so no verbatim
|
||||
argument line repeats them. The system prompt teaches how to use it."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
captured: list = []
|
||||
|
||||
@@ -609,9 +610,9 @@ async def test_focus_block_and_provenance_marked_action_phrase_reach_the_lens(mo
|
||||
assert "Absence is a fact" in system_text
|
||||
assert lead_text.startswith(
|
||||
"Step 3 Physical Action: Tapped 'Wi-Fi toggle' (self-described) at [880, 410]\n"
|
||||
"Controller Outcome: Tapped at [880, 410] (normalized)."
|
||||
)
|
||||
assert "Action arguments (verbatim): click({'target': [880, 410]" in lead_text
|
||||
assert "Controller Outcome: Tapped at [880, 410] (normalized)." in lead_text
|
||||
assert "Action arguments (verbatim)" not in lead_text
|
||||
assert FOCUS_BLOCK_HEADER in lead_text
|
||||
assert "Task goal: Turn on Wi-Fi" in lead_text
|
||||
assert "Active sub-goal: Enable Wi-Fi > Tap the Wi-Fi switch" in lead_text
|
||||
@@ -688,6 +689,102 @@ async def test_single_frame_focus_label_forbids_describing_the_outcome(mock_cont
|
||||
assert summarizer.get_job_payload(2)["focus"] is None
|
||||
|
||||
|
||||
def test_action_phrase_action_name_wins_over_an_action_argument():
|
||||
"""``manage_app``'s own ``action="launch"`` argument is the intent the
|
||||
renderer reads, never the verb: the phrase is ``Launched app 'X'`` for
|
||||
the Flash record shape, the Pro record shape and a raw argument dict."""
|
||||
flash_record = {
|
||||
"coordinates": None,
|
||||
"coordinate_space": "normalized",
|
||||
"args": {"action": "launch", "app_name": "Settings"},
|
||||
}
|
||||
assert VisualStepSummarizer._action_phrase("manage_app", flash_record) == (
|
||||
"Launched app 'Settings'"
|
||||
)
|
||||
pro_record = {"intent": "stop", "app_name": "com.android.settings"}
|
||||
assert VisualStepSummarizer._action_phrase("manage_app", pro_record) == (
|
||||
"Stopped app 'com.android.settings'"
|
||||
)
|
||||
raw = {"action": "launch", "app_name": "Settings"}
|
||||
assert VisualStepSummarizer._action_phrase("manage_app", raw) == "Launched app 'Settings'"
|
||||
assert "Action: launch" not in VisualStepSummarizer._action_phrase("manage_app", raw)
|
||||
|
||||
# The Flash record shape renders every other action through the same path.
|
||||
key = {"coordinates": None, "args": {"key": "BACK"}}
|
||||
assert VisualStepSummarizer._action_phrase("press_key", key) == "Pressed key 'BACK'"
|
||||
swipe = {
|
||||
"coordinates": [600, 700, 600, 300],
|
||||
"normalized_start_coordinates": [600, 700],
|
||||
"normalized_end_coordinates": [600, 300],
|
||||
"args": {"direction": "up"},
|
||||
}
|
||||
assert VisualStepSummarizer._action_phrase("swipe", swipe) == (
|
||||
"Swiped up (from [600, 700] to [600, 300])"
|
||||
)
|
||||
index_click = {
|
||||
"coordinates": [500, 520],
|
||||
"normalized_coordinates": [500, 520],
|
||||
"target_text": "Wi-Fi",
|
||||
"args": {"target": 3},
|
||||
}
|
||||
assert VisualStepSummarizer._action_phrase("click", index_click) == (
|
||||
"Tapped 'Wi-Fi' at [500, 520]"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verbatim_argument_line_appears_only_when_the_phrase_folds_arguments(
|
||||
mock_context,
|
||||
):
|
||||
"""``Launched app 'Settings'`` carries its arguments: no verbatim line. An
|
||||
index target (``target=3``) is folded into the resolved point, so the
|
||||
verbatim line keeps the index the model actually named."""
|
||||
summarizer = VisualStepSummarizer(mock_context)
|
||||
captured: list = []
|
||||
|
||||
async def mock_ainvoke(messages):
|
||||
captured.append(messages)
|
||||
return AIMessage(content="In this step, I acted and the screen showed the result.")
|
||||
|
||||
summarizer._llm = Mock()
|
||||
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
|
||||
|
||||
summarizer.dispatch(
|
||||
step_number=1,
|
||||
action_name="manage_app",
|
||||
action_args={"coordinates": None, "args": {"action": "launch", "app_name": "Settings"}},
|
||||
pre_img_bytes=b"pre",
|
||||
post_img_bytes=b"post",
|
||||
exec_outcome="Launched app 'Settings' (com.android.settings); foreground confirmed.",
|
||||
)
|
||||
summarizer.dispatch(
|
||||
step_number=2,
|
||||
action_name="click",
|
||||
action_args={
|
||||
"coordinates": [500, 520],
|
||||
"normalized_coordinates": [500, 520],
|
||||
"target_text": "Wi-Fi",
|
||||
"args": {"target": 3},
|
||||
},
|
||||
pre_img_bytes=b"pre",
|
||||
post_img_bytes=b"post",
|
||||
exec_outcome="Tapped at [500, 520] (normalized).",
|
||||
)
|
||||
await summarizer.flush()
|
||||
|
||||
lead_by_step = {}
|
||||
for messages in captured:
|
||||
text = messages[1].content[0]["text"]
|
||||
lead_by_step[text.split(" ")[1]] = text
|
||||
launch = lead_by_step["1"]
|
||||
assert launch.startswith("Step 1 Physical Action: Launched app 'Settings'\nController Outcome:")
|
||||
assert "Action arguments (verbatim)" not in launch
|
||||
assert "Action: launch" not in launch
|
||||
click = lead_by_step["2"]
|
||||
assert click.startswith("Step 2 Physical Action: Tapped 'Wi-Fi' at [500, 520]\n")
|
||||
assert "Action arguments (verbatim): click({'target': 3})" in click
|
||||
|
||||
|
||||
def test_action_phrase_renders_flash_swipe_points_and_bursts():
|
||||
"""Flash's ``start``/``end`` swipe points and a Pro burst's additional
|
||||
actions render through the shared history phrase, never as None."""
|
||||
|
||||
@@ -263,9 +263,9 @@ async def test_history_analyzer_read_note_tool_call():
|
||||
] # Index 5 contains the ToolMessage for read_note
|
||||
assert isinstance(read_msg, ToolMessage)
|
||||
assert read_msg.tool_call_id == "call_read"
|
||||
assert (
|
||||
read_msg.content == "Successfully read note 'tactical_plan'. 'tactical_plan' note"
|
||||
f" content:\n{fake_plan_content}"
|
||||
assert read_msg.content == (
|
||||
f"Note 'tactical_plan' ({len(fake_plan_content.splitlines())} lines):"
|
||||
f"\n{fake_plan_content}"
|
||||
)
|
||||
|
||||
|
||||
@@ -306,10 +306,7 @@ def test_history_analyzer_robust_tools_behavior():
|
||||
patch("pathlib.Path.read_text", return_value="Tactical plan content"),
|
||||
):
|
||||
result_read = read_tool.invoke({"key": "tactical_plan.md"})
|
||||
assert (
|
||||
result_read == "Successfully read note 'tactical_plan.md'. 'tactical_plan.md'"
|
||||
" note content:\nTactical plan content"
|
||||
)
|
||||
assert result_read == "Note 'tactical_plan.md' (1 lines):\nTactical plan content"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -399,6 +399,14 @@ async def test_operator_node_multiple_actions():
|
||||
assert decisions[1]["action"] == "tap"
|
||||
assert decisions[1]["coordinates"] == [54, 120]
|
||||
|
||||
# Each action's ToolMessage names its tool, like helper-tool results do
|
||||
# (the loop appends them to the same list it sent to the model).
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
sent = mock_llm.ainvoke.call_args[0][0]
|
||||
recorded = [(m.name, m.content) for m in sent if isinstance(m, ToolMessage)]
|
||||
assert recorded == [("input_text", "Action Recorded"), ("click", "Action Recorded")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_node_no_record_step():
|
||||
@@ -1467,6 +1475,40 @@ async def test_operator_swipe_translation():
|
||||
assert actions[0]["coordinates"] == [648, 720, 648, 1680]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_press_key_translation_is_case_insensitive():
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.context import ArtemisContext
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_state = MagicMock()
|
||||
mock_state.operator_raw_data = {"width": 1080, "height": 2400}
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
|
||||
for spelling in ("back", "Back", "BACK", "KEYCODE_BACK", "keycode_back", " back "):
|
||||
actions, err = node._translate_and_validate_tool(
|
||||
{"name": "press_key", "args": {"key": spelling}}, mock_state
|
||||
)
|
||||
assert err is None, spelling
|
||||
assert actions == [{"action": "press_key", "keycode": "KEYCODE_BACK"}], spelling
|
||||
|
||||
actions, err = node._translate_and_validate_tool(
|
||||
{"name": "press_key", "args": {"key": "app_switch"}}, mock_state
|
||||
)
|
||||
assert err is None
|
||||
assert actions == [{"action": "press_key", "keycode": "KEYCODE_APP_SWITCH"}]
|
||||
|
||||
# Unsupported keys get a self-explanatory error that lists the accepted names.
|
||||
actions, err = node._translate_and_validate_tool(
|
||||
{"name": "press_key", "args": {"key": "volume_up"}}, mock_state
|
||||
)
|
||||
assert actions == []
|
||||
assert err == (
|
||||
"Error: Unsupported key 'volume_up'. Supported keys: ENTER, BACK, HOME, APP_SWITCH."
|
||||
)
|
||||
|
||||
|
||||
# --- helper tool results: status is structural, never sniffed from the words ----------
|
||||
|
||||
|
||||
@@ -1537,5 +1579,79 @@ async def test_operator_helper_tool_message_status_is_structural(result, expecte
|
||||
tool_msgs = {m.tool_call_id: m for m in second_turn_messages if isinstance(m, ToolMessage)}
|
||||
assert tool_msgs["call_read"].status == expected_status
|
||||
assert tool_msgs["call_read"].content == str(result)
|
||||
# The accompanying click is rejected only when the helper actually failed.
|
||||
assert tool_msgs["call_click"].status == ("error" if expected_status == "error" else "success")
|
||||
# The accompanying click never ran: rejected when the helper failed, deferred
|
||||
# otherwise. Both are reported as errors and never claim success.
|
||||
assert tool_msgs["call_click"].status == "error"
|
||||
assert tool_msgs["call_click"].content.startswith("Not executed:")
|
||||
assert "success" not in tool_msgs["call_click"].content.lower()
|
||||
if expected_status == "error":
|
||||
assert str(result) in tool_msgs["call_click"].content
|
||||
else:
|
||||
assert "deferred" in tool_msgs["call_click"].content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_burst_limit_error_text_is_not_repeated_per_call():
|
||||
"""An over-long burst answers every tool_call id, but the full burst-limit
|
||||
paragraph travels once; the remaining ToolMessages just point at it."""
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.execution_setup = None
|
||||
mock_ctx.data_engine = None
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.initial_goal = "Test goal"
|
||||
mock_state.open_incident = None
|
||||
|
||||
turn_1 = MagicMock()
|
||||
turn_1.tool_calls = [
|
||||
{
|
||||
"name": "click",
|
||||
"args": {"target": [50, 50], "target_description": f"button {i}"},
|
||||
"id": f"call_click_{i}",
|
||||
}
|
||||
for i in range(6)
|
||||
]
|
||||
turn_2 = MagicMock()
|
||||
turn_2.tool_calls = [
|
||||
{
|
||||
"name": "click",
|
||||
"args": {"target": [50, 50], "target_description": "button"},
|
||||
"id": "call_click_final",
|
||||
}
|
||||
]
|
||||
mock_llm = MagicMock()
|
||||
mock_llm.ainvoke = AsyncMock(side_effect=[turn_1, turn_2])
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
with (
|
||||
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
|
||||
patch.object(OperatorNode, "_max_burst_actions", return_value=4),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
assert mock_llm.ainvoke.await_count == 2
|
||||
second_turn_messages = mock_llm.ainvoke.call_args_list[1].args[0]
|
||||
burst_ids = {f"call_click_{i}" for i in range(6)}
|
||||
tool_msgs = {
|
||||
m.tool_call_id: m
|
||||
for m in second_turn_messages
|
||||
if isinstance(m, ToolMessage) and m.tool_call_id in burst_ids
|
||||
}
|
||||
assert set(tool_msgs) == burst_ids
|
||||
assert all(m.status == "error" for m in tool_msgs.values())
|
||||
|
||||
first = tool_msgs["call_click_0"].content
|
||||
assert first.startswith("Error: 6 Turn-Ending Actions in one turn exceed")
|
||||
assert "burst limit of 4" in first
|
||||
for i in range(1, 6):
|
||||
assert tool_msgs[f"call_click_{i}"].content == (
|
||||
"Not executed: see the burst-limit error above."
|
||||
)
|
||||
|
||||
# Nothing from the rejected burst ran; turn 2's single click did.
|
||||
decisions = json.loads(node_update["structured_decisions"])
|
||||
assert len(decisions) == 1 and decisions[0]["action"] == "tap"
|
||||
@@ -0,0 +1,122 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""The Pro operator's context base on a provider without usage metadata.
|
||||
|
||||
The transcript ledger's start gate and soft/hard thresholds read the
|
||||
operator's own measured prompt size; when the provider reports no usage the
|
||||
operator records a provider-less estimate instead (Flash already does), so
|
||||
the thresholds still see a base. A measured call also hands the ledger the
|
||||
messages it was measured on, for the chars-per-token calibration.
|
||||
"""
|
||||
|
||||
import base64
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.config.agent import MemoryTranscriptConfig
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.memory.step_memory import StepMemoryService
|
||||
from artemis.memory.transcript import TranscriptLedger, estimate_prompt_tokens
|
||||
|
||||
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
|
||||
|
||||
|
||||
def _ctx():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.execution_setup = None
|
||||
ctx.actuator = None
|
||||
ctx.data_engine = None
|
||||
ctx.step_memory = StepMemoryService(ctx=None)
|
||||
ctx.transcript_ledger = None
|
||||
return ctx
|
||||
|
||||
|
||||
def _state():
|
||||
state = MagicMock()
|
||||
state.subagent_calls = []
|
||||
state.initial_goal = "Estimate goal"
|
||||
state.operator_feedback = None
|
||||
state.injected_instruction = None
|
||||
state.operator_tool_limit_exceeded = False
|
||||
state.structured_decisions = None
|
||||
state.last_execution_result = None
|
||||
state.current_step_id = None
|
||||
state.latest_ui_hierarchy = []
|
||||
state.operator_raw_data = {
|
||||
"screenshot_b64": SCREENSHOT_B64,
|
||||
"xml_hierarchy": [],
|
||||
"ocr_results": None,
|
||||
"width": 1080,
|
||||
"height": 2400,
|
||||
}
|
||||
return state
|
||||
|
||||
|
||||
def _llm(captured: list, usage_metadata=None):
|
||||
mock_llm = MagicMock()
|
||||
response = MagicMock()
|
||||
response.tool_calls = []
|
||||
response.content = "no action this turn"
|
||||
# usage_from_message only trusts a non-empty dict; a MagicMock attribute
|
||||
# (the default) reads as "no usage reported".
|
||||
if usage_metadata is not None:
|
||||
response.usage_metadata = usage_metadata
|
||||
|
||||
async def ainvoke(*args, **kwargs):
|
||||
captured.append(list(args[0]))
|
||||
return response
|
||||
|
||||
mock_llm.ainvoke = AsyncMock(side_effect=ainvoke)
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
return mock_llm
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_records_an_estimate_when_the_provider_reports_no_usage():
|
||||
ctx = _ctx()
|
||||
captured: list = []
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=_llm(captured)):
|
||||
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
|
||||
await node(_state())
|
||||
|
||||
ledger = ctx.transcript_ledger
|
||||
assert isinstance(ledger, TranscriptLedger)
|
||||
assert captured, "the operator made no model call"
|
||||
assert ledger.last_prompt_tokens == estimate_prompt_tokens(captured[-1])
|
||||
assert ledger.last_prompt_tokens > 0
|
||||
# No measurement, no calibration.
|
||||
assert ledger.chars_per_token == 4.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_measured_usage_calibrates_the_ledger_ratio():
|
||||
ctx = _ctx()
|
||||
captured: list = []
|
||||
llm = _llm(
|
||||
captured,
|
||||
usage_metadata={"input_tokens": 50_000, "output_tokens": 10, "total_tokens": 50_010},
|
||||
)
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=llm):
|
||||
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
|
||||
await node(_state())
|
||||
|
||||
ledger = ctx.transcript_ledger
|
||||
assert ledger.last_prompt_tokens == 50_000
|
||||
# The prompt is a few thousand chars of English plus one image; measured
|
||||
# at 50k tokens the ratio clamps to the lower bound — proof the messages
|
||||
# reached the calibration, not just the token figure.
|
||||
assert ledger.chars_per_token == 1.0
|
||||
@@ -0,0 +1,147 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Tests for the Operator reasoning reminder.
|
||||
|
||||
Thinking-mode models satisfy "reason before acting" inside their thought channel
|
||||
and emit bare tool calls. After such a silent turn the next observation tail
|
||||
carries a one-line reminder; nothing is bounced and no extra model call is made.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.agents.operator.prompts import (
|
||||
REASONING_REMINDER,
|
||||
apply_operator_prompt_contract,
|
||||
load_operator_prompts,
|
||||
)
|
||||
from artemis.config.agent import MemoryTranscriptConfig
|
||||
from artemis.context import ArtemisContext
|
||||
|
||||
LEGACY_TRANSCRIPT = MemoryTranscriptConfig(enabled=False)
|
||||
|
||||
PLAN_WITH_LEAF = """- [x] Open Google Maps and search for SFO
|
||||
- [/] Read the commute duration and record it into note `commute_eta_info`
|
||||
- [/] Read the driving duration
|
||||
- [/] Tap the "Driving" tab so the fastest route is highlighted
|
||||
- [ ] Draft the ETA message
|
||||
"""
|
||||
|
||||
|
||||
def _make_node(tmp_path):
|
||||
notes = tmp_path / "notes"
|
||||
notes.mkdir(parents=True)
|
||||
(notes / "task_plan.md").write_text(PLAN_WITH_LEAF, encoding="utf-8")
|
||||
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.execution_setup = None
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
ctx.data_engine.current_session_id = "s"
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
|
||||
state = MagicMock()
|
||||
state.subagent_calls = []
|
||||
state.initial_goal = "goal"
|
||||
state.open_incident = None
|
||||
return OperatorNode(ctx, transcript_config=LEGACY_TRANSCRIPT), state
|
||||
|
||||
|
||||
def _click_response(content):
|
||||
response = MagicMock()
|
||||
response.content = content
|
||||
response.tool_calls = [
|
||||
{
|
||||
"name": "click",
|
||||
"args": {"target": [50, 50], "target_description": "button"},
|
||||
"id": "call_0",
|
||||
}
|
||||
]
|
||||
return response
|
||||
|
||||
|
||||
def _texts(messages):
|
||||
out = []
|
||||
for m in messages:
|
||||
content = getattr(m, "content", None)
|
||||
if isinstance(content, str):
|
||||
out.append(content)
|
||||
elif isinstance(content, list):
|
||||
out.extend(b.get("text", "") for b in content if isinstance(b, dict))
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reminder_follows_a_silent_turn_and_costs_no_extra_call(tmp_path):
|
||||
node, state = _make_node(tmp_path)
|
||||
seen = []
|
||||
replies = iter(
|
||||
[
|
||||
_click_response(""), # turn 1: silent
|
||||
_click_response("Tapping the Driving tab."), # turn 2: reminded, speaks
|
||||
_click_response([{"type": "thinking", "thinking": "internal"}]), # turn 3: silent
|
||||
_click_response("Done."), # turn 4
|
||||
]
|
||||
)
|
||||
|
||||
async def ainvoke(messages, *args, **kwargs):
|
||||
seen.append(list(messages))
|
||||
return next(replies)
|
||||
|
||||
llm = MagicMock()
|
||||
llm.ainvoke = AsyncMock(side_effect=ainvoke)
|
||||
llm.bind_tools.return_value = llm
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=llm):
|
||||
for _ in range(4):
|
||||
await node(state)
|
||||
|
||||
assert llm.ainvoke.await_count == 4, "one model call per turn, never a bounce"
|
||||
assert REASONING_REMINDER not in _texts(seen[0]) # nothing to remind on turn 1
|
||||
assert REASONING_REMINDER in _texts(seen[1]) # turn 1 was silent
|
||||
assert REASONING_REMINDER not in _texts(seen[2]) # turn 2 spoke
|
||||
assert REASONING_REMINDER in _texts(seen[3]) # thinking-only turn 3 counts as silent
|
||||
assert _texts(seen[3]).count(REASONING_REMINDER) == 1
|
||||
|
||||
|
||||
def test_ledger_silence_verdict_outranks_the_local_flag(tmp_path):
|
||||
"""With a transcript ledger, the ledger's own judgment of the previous
|
||||
turn is authoritative; the local flag only serves the legacy path."""
|
||||
from artemis.memory import TranscriptLedger
|
||||
|
||||
node, _ = _make_node(tmp_path)
|
||||
node._previous_turn_silent = True
|
||||
assert node._previous_turn_was_silent() is True # no ledger: local flag
|
||||
|
||||
ledger = MagicMock(spec=TranscriptLedger)
|
||||
ledger.last_turn_silent = False
|
||||
node.ctx.transcript_ledger = ledger
|
||||
assert node._previous_turn_was_silent() is False
|
||||
|
||||
ledger.last_turn_silent = True
|
||||
node._previous_turn_silent = False
|
||||
assert node._previous_turn_was_silent() is True
|
||||
|
||||
|
||||
def test_prompt_asks_for_one_paragraph_not_phase_sections():
|
||||
prompt = apply_operator_prompt_contract(load_operator_prompts()["main_template"])
|
||||
assert prompt.count("one short paragraph of connected prose reflecting these three phases") == 1
|
||||
assert "in your raw thinking" not in prompt
|
||||
assert "in your written reply" not in prompt
|
||||
assert (
|
||||
"bounced"
|
||||
not in prompt.split("# THE THREE-PHASE COGNITIVE PROTOCOL")[1].split("## Phase 1")[0]
|
||||
)
|
||||
@@ -45,6 +45,7 @@ from artemis.config.agent import MemoryTranscriptConfig
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.memory.step_memory import StepMemoryService
|
||||
from artemis.memory.transcript import (
|
||||
EPHEMERAL_BLOCKS_KEY,
|
||||
EXECUTION_RESULT_MARKER,
|
||||
PLAN_RECITATION_MARKER,
|
||||
RESTORED_HISTORY_HEADER,
|
||||
@@ -53,8 +54,8 @@ from artemis.memory.transcript import (
|
||||
|
||||
# SHA-256 snapshots of the legacy system message with the fixed inputs below.
|
||||
# Update these when an intentional template change alters the rendered prompt.
|
||||
GOLDEN_EMPTY_PLAN = "d8cf83565c766d32893b124c974060bbd98de2d862c3b9925e546ea0c946c324"
|
||||
GOLDEN_SENTINEL_PLAN = "8f871b5e5700ba5ee07ed03b80a8f82664abe68b0c170d3185bf1ee35f7821ed"
|
||||
GOLDEN_EMPTY_PLAN = "513e0341014b9a2342f608ad4dd173b7dcf5704ce53ee028ba302a9a1b696f96"
|
||||
GOLDEN_SENTINEL_PLAN = "012fe1b08645266d5a7e518023e20fc6fbfa65d4f2d985847ebb06e0184c67ee"
|
||||
|
||||
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
|
||||
|
||||
@@ -103,6 +104,8 @@ def test_transcript_static_system_is_stable_and_carries_no_history():
|
||||
# ...and no unrendered slot or trailing observation header survives.
|
||||
assert "{{" not in first
|
||||
assert not first.endswith("# CURRENT OBSERVATION\n")
|
||||
# The history guide names the marker a resolved screenshot renders as.
|
||||
assert "--- Historical Visual Transition ---" in first
|
||||
|
||||
|
||||
def _transcript_ctx():
|
||||
@@ -214,7 +217,9 @@ async def test_transcript_mode_two_turns_build_four_regions(tmp_path):
|
||||
# Committed turn-1 observation followed by its validator result.
|
||||
assert any(EXECUTION_RESULT_MARKER in str(m.content) for m in active)
|
||||
committed_obs = active[0]
|
||||
# Depth-1 scrub removed the old plan recitation copy.
|
||||
# Text edge at depth 1: the committed observation has lost its plan
|
||||
# recitation as soon as the next tail exists (its screenshot stays
|
||||
# until the screenshot edge).
|
||||
assert PLAN_RECITATION_MARKER not in str(committed_obs.content)
|
||||
# The live tail still recites the plan.
|
||||
assert PLAN_RECITATION_MARKER in str(turn2_messages[-1].content)
|
||||
@@ -223,6 +228,52 @@ async def test_transcript_mode_two_turns_build_four_regions(tmp_path):
|
||||
assert re.search(r"T\+\d{2,}:\d{2}", str(turn2_messages[-1].content))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_tail_marks_per_turn_notices_ephemeral():
|
||||
"""The hand-built tail carries the builder's ephemeral indices: per-turn
|
||||
notices are deleted by the scrub edge; header, plan recitation and the
|
||||
observation itself are not."""
|
||||
from artemis.agents.operator.prompts import USER_GUIDANCE_MARKER
|
||||
|
||||
ctx = _transcript_ctx()
|
||||
captured: list = []
|
||||
mock_llm = _no_action_llm(captured)
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
|
||||
await node(
|
||||
_transcript_state(
|
||||
injected_instruction="Skip the login step",
|
||||
user_stop_requested=False,
|
||||
operator_tool_limit_exceeded=True,
|
||||
)
|
||||
)
|
||||
|
||||
tail = captured[0][-1]
|
||||
assert isinstance(tail, HumanMessage)
|
||||
texts = {
|
||||
i: block.get("text", "")
|
||||
for i, block in enumerate(tail.content)
|
||||
if isinstance(block, dict) and block.get("type") == "text"
|
||||
}
|
||||
guidance = next(i for i, t in texts.items() if t.startswith(USER_GUIDANCE_MARKER))
|
||||
warning = next(i for i, t in texts.items() if "tool-call budget" in t)
|
||||
assert set(tail.additional_kwargs[EPHEMERAL_BLOCKS_KEY]) == {guidance, warning}
|
||||
# The user's stop bit is relayed on the guidance block itself.
|
||||
assert "authorized stopping" not in texts[guidance]
|
||||
# The verbatim instruction is its own persistent block right after the
|
||||
# wrapper (stamped with the ledger's session offset), never ephemeral:
|
||||
# a standing instruction must outlive the turn it arrived on.
|
||||
body = guidance + 1
|
||||
assert re.fullmatch(r'User instruction \(T\+\d{2,}:\d{2}\): "Skip the login step"', texts[body])
|
||||
assert body not in tail.additional_kwargs[EPHEMERAL_BLOCKS_KEY]
|
||||
assert "Skip the login step" not in texts[guidance]
|
||||
# Header, plan recitation and the observation stay in the history.
|
||||
assert texts[0].startswith("# CURRENT OBSERVATION")
|
||||
plan = next(i for i, t in texts.items() if t.startswith(PLAN_RECITATION_MARKER))
|
||||
screenshot = next(i for i, t in texts.items() if t == "--- Current Screenshot ---")
|
||||
assert not {0, plan, screenshot} & set(tail.additional_kwargs[EPHEMERAL_BLOCKS_KEY])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcript_actionless_turn_commits_without_validator_message():
|
||||
ctx = _transcript_ctx()
|
||||
|
||||
@@ -287,7 +287,7 @@ async def test_outputter_builds_concise_history(mock_get_llm):
|
||||
}
|
||||
]
|
||||
mock_data_engine.get_agent_friendly_steps.return_value = steps
|
||||
# M5: the transcript flag defaults on, so the outputter's compiled view
|
||||
# The transcript flag defaults on, so the outputter's compiled view
|
||||
# consults the chunk store; no persisted chunks keeps the classic path.
|
||||
mock_data_engine.get_history_chunks.return_value = []
|
||||
mock_data_engine.base_dir = "/tmp/mock_session"
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""SummarizerNode tests (M2: visual-transition lens dispatch).
|
||||
"""Tests for SummarizerNode visual-transition dispatch.
|
||||
|
||||
Per the history redesign §5 the Pro SummarizerNode no longer runs its own LLM
|
||||
call to produce a text capsule: it dispatches the step to the shared
|
||||
|
||||
@@ -0,0 +1,364 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for the end-to-end device observation smoke test (no real device)."""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.core.diagnostics import smoke_test_device
|
||||
from artemis.core.diagnostics import device_smoke
|
||||
from artemis.runtime.device_lock import DeviceExecutionLock, DeviceLockOwner
|
||||
|
||||
SERIAL = "59100DLCR0033X"
|
||||
FAKE_JPEG = b"\xff\xd8\xff" + bytes(range(256)) * 8 + b"\xff\xd9"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def idle_devices(monkeypatch):
|
||||
"""No live lock owners unless a test installs some."""
|
||||
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", classmethod(lambda cls: {}))
|
||||
monkeypatch.delenv("ARTEMIS_DEVICE_ID", raising=False)
|
||||
monkeypatch.delenv("ADB_DEVICE_SERIAL", raising=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patch_controller(monkeypatch):
|
||||
"""Install a fake ``_get_controller`` and return the controller it hands out."""
|
||||
|
||||
def _install(get_screen_data=None, factory=None):
|
||||
controller = MagicMock()
|
||||
controller.ctx = SimpleNamespace(device=SimpleNamespace(device_id=SERIAL))
|
||||
if get_screen_data is not None:
|
||||
controller.get_screen_data = get_screen_data
|
||||
calls: list[str | None] = []
|
||||
|
||||
def _fake_get_controller(device_serial=None):
|
||||
calls.append(device_serial)
|
||||
if factory is not None:
|
||||
return factory(device_serial)
|
||||
return controller
|
||||
|
||||
monkeypatch.setattr("artemis.mcp.adb_server._get_controller", _fake_get_controller)
|
||||
controller.calls = calls
|
||||
return controller
|
||||
|
||||
return _install
|
||||
|
||||
|
||||
def _screen_data(elements, jpeg=FAKE_JPEG):
|
||||
return SimpleNamespace(
|
||||
base64=base64.b64encode(jpeg).decode("ascii"),
|
||||
elements=elements,
|
||||
width=1080,
|
||||
height=2400,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_success_reports_bytes_elements_serial_and_elapsed(patch_controller):
|
||||
elements = [{"class": "android.widget.FrameLayout"}, {"text": "Hello"}, {"text": "World"}]
|
||||
controller = patch_controller(AsyncMock(return_value=_screen_data(elements)))
|
||||
|
||||
result = await smoke_test_device(device_serial=SERIAL, timeout_seconds=5)
|
||||
|
||||
assert result["ok"] is True
|
||||
assert result["error"] is None
|
||||
assert result["fix"] == []
|
||||
assert result["serial"] == SERIAL
|
||||
assert result["screenshot_bytes"] == len(FAKE_JPEG)
|
||||
assert result["element_count"] == 3
|
||||
assert isinstance(result["elapsed_seconds"], float)
|
||||
assert result["elapsed_seconds"] >= 0.0
|
||||
assert controller.calls == [SERIAL]
|
||||
controller.get_screen_data.assert_awaited_once()
|
||||
assert set(result) == {
|
||||
"ok",
|
||||
"serial",
|
||||
"elapsed_seconds",
|
||||
"screenshot_bytes",
|
||||
"element_count",
|
||||
"error",
|
||||
"fix",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serial_comes_from_controller_when_not_requested(patch_controller):
|
||||
patch_controller(AsyncMock(return_value=_screen_data([{"text": "x"}])))
|
||||
|
||||
result = await smoke_test_device()
|
||||
|
||||
assert result["ok"] is True
|
||||
assert result["serial"] == SERIAL
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_xml_string_elements_are_counted_by_node(patch_controller):
|
||||
xml = '<hierarchy><node class="a"><node class="b"/><node class="c"/></node></hierarchy>'
|
||||
patch_controller(AsyncMock(return_value=_screen_data(xml)))
|
||||
|
||||
result = await smoke_test_device(device_serial=SERIAL)
|
||||
|
||||
assert result["ok"] is True
|
||||
assert result["element_count"] == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_init_failure_no_devices_points_to_launch_avd(patch_controller):
|
||||
def _boom(device_serial):
|
||||
raise Exception("No Android devices found at localhost:5037")
|
||||
|
||||
patch_controller(factory=_boom)
|
||||
|
||||
result = await smoke_test_device(device_serial=SERIAL, timeout_seconds=5)
|
||||
|
||||
assert result["ok"] is False
|
||||
assert result["serial"] == SERIAL
|
||||
assert result["screenshot_bytes"] is None
|
||||
assert result["element_count"] is None
|
||||
assert "No Android devices found" in result["error"]
|
||||
assert any("launch_avd" in step for step in result["fix"])
|
||||
assert any("mobile_diagnose" in step for step in result["fix"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_init_failure_target_not_found_points_to_launch_avd(patch_controller):
|
||||
def _boom(device_serial):
|
||||
raise Exception(
|
||||
f"Target Android device '{device_serial}' not found at localhost:5037 among available devices."
|
||||
)
|
||||
|
||||
patch_controller(factory=_boom)
|
||||
|
||||
result = await smoke_test_device(device_serial="emulator-5554")
|
||||
|
||||
assert result["ok"] is False
|
||||
assert result["serial"] == "emulator-5554"
|
||||
assert any("launch_avd" in step for step in result["fix"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_screen_data_timeout_reports_timeout_and_force_stop(patch_controller):
|
||||
async def _hang():
|
||||
await asyncio.sleep(3)
|
||||
return _screen_data([{"text": "late"}])
|
||||
|
||||
patch_controller(_hang)
|
||||
|
||||
result = await smoke_test_device(device_serial=SERIAL, timeout_seconds=0.3)
|
||||
|
||||
assert result["ok"] is False
|
||||
assert result["serial"] == SERIAL
|
||||
assert result["error"] == "UIAutomator/screen capture did not respond within 0.3s"
|
||||
assert result["elapsed_seconds"] < 2.5
|
||||
joined = "\n".join(result["fix"])
|
||||
assert f"adb -s {SERIAL} shell am force-stop com.github.uiautomator" in joined
|
||||
assert "uiautomator2 init" in joined
|
||||
assert "screen on" in joined
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_init_timeout_is_reported_without_blocking(patch_controller):
|
||||
import time
|
||||
|
||||
def _slow(device_serial):
|
||||
time.sleep(2)
|
||||
return MagicMock()
|
||||
|
||||
patch_controller(factory=_slow)
|
||||
|
||||
result = await smoke_test_device(device_serial=SERIAL, timeout_seconds=0.3)
|
||||
|
||||
assert result["ok"] is False
|
||||
assert "did not respond within 0.3s" in result["error"]
|
||||
assert result["elapsed_seconds"] < 1.5
|
||||
assert any("force-stop" in step for step in result["fix"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generic_screen_data_exception_falls_back_to_manual_dump(patch_controller):
|
||||
patch_controller(AsyncMock(side_effect=RuntimeError("weird failure 42")))
|
||||
|
||||
result = await smoke_test_device(device_serial=SERIAL)
|
||||
|
||||
assert result["ok"] is False
|
||||
assert result["error"] == "Screen capture failed: weird failure 42"
|
||||
assert any("uiautomator dump" in step for step in result["fix"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_base_exception_from_uiautomator2_is_captured_not_raised(patch_controller):
|
||||
class HTTPTimeoutError(BaseException):
|
||||
"""uiautomator2 errors derive from BaseException."""
|
||||
|
||||
patch_controller(
|
||||
AsyncMock(side_effect=HTTPTimeoutError("HTTP request timeout: read timed out"))
|
||||
)
|
||||
|
||||
result = await smoke_test_device(device_serial=SERIAL)
|
||||
|
||||
assert result["ok"] is False
|
||||
assert "HTTPTimeoutError" in result["error"]
|
||||
assert any("force-stop" in step for step in result["fix"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unauthorized_and_offline_guidance(patch_controller):
|
||||
def _unauth(device_serial):
|
||||
raise Exception("device unauthorized. This adb server's $ADB_VENDOR_KEYS is not set")
|
||||
|
||||
patch_controller(factory=_unauth)
|
||||
result = await smoke_test_device(device_serial=SERIAL)
|
||||
assert result["ok"] is False
|
||||
assert any("USB debugging" in step for step in result["fix"])
|
||||
|
||||
def _offline(device_serial):
|
||||
raise Exception("device offline")
|
||||
|
||||
patch_controller(factory=_offline)
|
||||
result = await smoke_test_device(device_serial=SERIAL)
|
||||
assert result["ok"] is False
|
||||
assert any("adb reconnect" in step for step in result["fix"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_hierarchy_is_not_ok(patch_controller):
|
||||
patch_controller(AsyncMock(return_value=_screen_data([])))
|
||||
|
||||
result = await smoke_test_device(device_serial=SERIAL)
|
||||
|
||||
assert result["ok"] is False
|
||||
assert result["screenshot_bytes"] == len(FAKE_JPEG)
|
||||
assert result["element_count"] == 0
|
||||
assert "no UI elements" in result["error"]
|
||||
assert any("force-stop" in step for step in result["fix"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_placeholder_screenshot_is_not_ok(patch_controller):
|
||||
one_pixel_png = base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
|
||||
)
|
||||
patch_controller(AsyncMock(return_value=_screen_data([{"text": "x"}], jpeg=one_pixel_png)))
|
||||
|
||||
result = await smoke_test_device(device_serial=SERIAL)
|
||||
|
||||
assert result["ok"] is False
|
||||
assert result["screenshot_bytes"] == len(one_pixel_png)
|
||||
assert "placeholder" in result["error"]
|
||||
|
||||
|
||||
def _owner(device_id: str, description: str = "task-1") -> DeviceLockOwner:
|
||||
return DeviceLockOwner(
|
||||
pid=4242,
|
||||
process_created_at=1.0,
|
||||
token="tok",
|
||||
device_id=device_id,
|
||||
description=description,
|
||||
acquired_at="2026-09-09T00:00:00Z",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_busy_device_is_reported_without_touching_the_controller(
|
||||
patch_controller, monkeypatch
|
||||
):
|
||||
controller = patch_controller(AsyncMock(return_value=_screen_data([{"text": "x"}])))
|
||||
monkeypatch.setattr(
|
||||
DeviceExecutionLock,
|
||||
"get_active_owners",
|
||||
classmethod(lambda cls: {SERIAL: _owner(SERIAL, "youtube smoke")}),
|
||||
)
|
||||
|
||||
result = await smoke_test_device(device_serial=SERIAL)
|
||||
|
||||
assert result["ok"] is False
|
||||
assert result["serial"] == SERIAL
|
||||
assert "busy" in result["error"]
|
||||
assert "youtube smoke" in result["error"]
|
||||
assert "4242" in result["error"]
|
||||
assert any("mobile_manage_task" in step for step in result["fix"])
|
||||
assert controller.calls == []
|
||||
controller.get_screen_data.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_busy_guard_matches_scoped_lock_keys(patch_controller, monkeypatch):
|
||||
controller = patch_controller(AsyncMock(return_value=_screen_data([{"text": "x"}])))
|
||||
monkeypatch.setattr(
|
||||
DeviceExecutionLock,
|
||||
"get_active_owners",
|
||||
classmethod(lambda cls: {f"endpoint1__{SERIAL}": _owner(SERIAL)}),
|
||||
)
|
||||
|
||||
result = await smoke_test_device(device_serial=SERIAL)
|
||||
|
||||
assert result["ok"] is False
|
||||
assert "busy" in result["error"]
|
||||
assert controller.calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_busy_guard_ignores_other_devices(patch_controller, monkeypatch):
|
||||
controller = patch_controller(AsyncMock(return_value=_screen_data([{"text": "x"}])))
|
||||
monkeypatch.setattr(
|
||||
DeviceExecutionLock,
|
||||
"get_active_owners",
|
||||
classmethod(lambda cls: {"emulator-5554": _owner("emulator-5554")}),
|
||||
)
|
||||
|
||||
result = await smoke_test_device(device_serial=SERIAL)
|
||||
|
||||
assert result["ok"] is True
|
||||
assert controller.calls == [SERIAL]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_busy_guard_without_target_serial_treats_any_owner_as_busy(
|
||||
patch_controller, monkeypatch
|
||||
):
|
||||
controller = patch_controller(AsyncMock(return_value=_screen_data([{"text": "x"}])))
|
||||
monkeypatch.setattr(
|
||||
DeviceExecutionLock,
|
||||
"get_active_owners",
|
||||
classmethod(lambda cls: {"emulator-5554": _owner("emulator-5554")}),
|
||||
)
|
||||
|
||||
result = await smoke_test_device()
|
||||
|
||||
assert result["ok"] is False
|
||||
assert result["serial"] == "emulator-5554"
|
||||
assert "busy" in result["error"]
|
||||
assert controller.calls == []
|
||||
|
||||
|
||||
def test_fix_for_error_generic_fallback_uses_serial():
|
||||
fix = device_smoke.fix_for_error("something odd", SERIAL)
|
||||
assert fix
|
||||
assert all(SERIAL in step for step in fix)
|
||||
assert any("uiautomator dump" in step for step in fix)
|
||||
assert device_smoke.fix_for_error(None, SERIAL) == []
|
||||
assert "<serial>" in device_smoke.fix_for_error("timeout", None)[1]
|
||||
|
||||
|
||||
def test_smoke_test_is_exported_from_diagnostics_package():
|
||||
import artemis.core.diagnostics as diagnostics
|
||||
|
||||
assert "smoke_test_device" in diagnostics.__all__
|
||||
assert diagnostics.smoke_test_device is device_smoke.smoke_test_device
|
||||
@@ -429,3 +429,87 @@ async def test_emulator_manager_lifecycle():
|
||||
dismiss_res = manager.dismiss()
|
||||
assert dismiss_res["success"] is True
|
||||
assert manager.get_status().status == EmulatorLaunchStage.IDLE
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_report_turns_crashing_probe_into_fail_result():
|
||||
engine = ReadinessEngine()
|
||||
healthy = ProbeResult(
|
||||
id="healthy",
|
||||
category=ProbeCategory.RUNTIME,
|
||||
title="Healthy",
|
||||
status=ProbeStatus.PASS,
|
||||
is_blocker=True,
|
||||
summary="Ready",
|
||||
description="Ready",
|
||||
)
|
||||
good = Mock()
|
||||
good.probe_id = "healthy"
|
||||
good.category = ProbeCategory.RUNTIME
|
||||
good.is_blocker = True
|
||||
good.probe = AsyncMock(return_value=healthy)
|
||||
|
||||
bad = Mock()
|
||||
bad.probe_id = "integration_host"
|
||||
bad.category = ProbeCategory.RUNTIME
|
||||
bad.is_blocker = True
|
||||
bad.probe = AsyncMock(side_effect=PermissionError(13, "Permission denied", "/ro/traces"))
|
||||
engine._probes = {"healthy": good, "integration_host": bad}
|
||||
|
||||
report = await engine._build_report()
|
||||
|
||||
assert report.overall_ready is False
|
||||
assert report.blocker_count == 2
|
||||
assert report.passed_blocker_count == 1
|
||||
by_id = {r.id: r for r in report.probes}
|
||||
assert by_id["healthy"].status is ProbeStatus.PASS
|
||||
crashed = by_id["integration_host"]
|
||||
assert crashed.status is ProbeStatus.FAIL
|
||||
assert crashed.is_blocker is True
|
||||
assert crashed.category is ProbeCategory.RUNTIME
|
||||
assert crashed.summary == "Probe crashed"
|
||||
assert "PermissionError" in crashed.description
|
||||
assert "Permission denied" in crashed.description
|
||||
assert crashed.metadata["exception_type"] == "PermissionError"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_report_turns_hung_probe_into_fail_result(monkeypatch):
|
||||
"""A probe that never returns (wedged ADB server) is cut off at the
|
||||
engine's per-probe deadline and reported as a FAIL with the restart step,
|
||||
so no surface running the report can hang."""
|
||||
monkeypatch.setattr(ReadinessEngine, "PROBE_TIMEOUT_SECONDS", 0.05)
|
||||
engine = ReadinessEngine()
|
||||
|
||||
async def never_returns():
|
||||
await asyncio.sleep(10)
|
||||
|
||||
hung = Mock()
|
||||
hung.probe_id = "android_adb"
|
||||
hung.category = ProbeCategory.DEVICE
|
||||
hung.is_blocker = True
|
||||
hung.probe = never_returns
|
||||
engine._probes = {"android_adb": hung}
|
||||
|
||||
report = await engine._build_report()
|
||||
|
||||
result = report.probes[0]
|
||||
assert result.status is ProbeStatus.FAIL
|
||||
assert result.summary == "Probe timed out"
|
||||
assert result.metadata["exception_type"] == "TimeoutError"
|
||||
assert any("adb kill-server" in a.payload for a in result.actions)
|
||||
assert report.overall_ready is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_report_does_not_swallow_cancellation():
|
||||
engine = ReadinessEngine()
|
||||
probe = Mock()
|
||||
probe.probe_id = "cancelled"
|
||||
probe.category = ProbeCategory.RUNTIME
|
||||
probe.is_blocker = True
|
||||
probe.probe = AsyncMock(side_effect=asyncio.CancelledError())
|
||||
engine._probes = {"cancelled": probe}
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await engine._build_report()
|
||||
@@ -0,0 +1,471 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for the MCP / IDE integration-host readiness probe."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.core.diagnostics import IntegrationHostProbe, ReadinessEngine
|
||||
from artemis.core.diagnostics.probes import host_probe
|
||||
from artemis.core.diagnostics.schema import ProbeCategory, ProbeStatus
|
||||
|
||||
|
||||
#: Environment keys / prefixes any IDE might export; scrubbed so the developer's
|
||||
#: own host (this test suite is often run from inside Claude Code or Cursor) does
|
||||
#: not leak into the detection tests.
|
||||
_SIGNAL_PREFIXES = (
|
||||
"ANTIGRAVITY_",
|
||||
"CLAUDE_CODE_",
|
||||
"CURSOR_",
|
||||
"WINDSURF_",
|
||||
"CODEX_",
|
||||
"OPENCLAW_",
|
||||
"CLINE_",
|
||||
"ROO_",
|
||||
"VSCODE_",
|
||||
)
|
||||
_SIGNAL_KEYS = ("CLAUDECODE", "TERM_PROGRAM")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clean_ide_env(monkeypatch):
|
||||
for key in list(os.environ):
|
||||
if key in _SIGNAL_KEYS or key.startswith(_SIGNAL_PREFIXES):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
return monkeypatch
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def host_env(tmp_path, monkeypatch, clean_ide_env):
|
||||
"""A consistent, healthy host layout the individual tests then break."""
|
||||
venv_python = host_probe.project_venv_python(tmp_path)
|
||||
venv_python.parent.mkdir(parents=True)
|
||||
venv_python.write_bytes(b"")
|
||||
traces = tmp_path / "traces"
|
||||
|
||||
monkeypatch.setattr("artemis.config.paths.ROOT_DIR", tmp_path)
|
||||
monkeypatch.setattr("artemis.config.paths.get_env_file", lambda: tmp_path / ".env")
|
||||
monkeypatch.setattr("artemis.config.paths.get_default_traces_path", lambda: traces)
|
||||
monkeypatch.setattr("artemis.config.paths.is_source_checkout", lambda: True)
|
||||
monkeypatch.setattr(sys, "executable", str(venv_python))
|
||||
|
||||
monkeypatch.setattr("artemis.runtime.daemon_client.is_standalone_forced", lambda: False)
|
||||
monkeypatch.setattr("artemis.runtime.daemon_client.is_artemis_daemon", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
"artemis.runtime.daemon_client.daemon_log_path", lambda: tmp_path / "daemon.log"
|
||||
)
|
||||
monkeypatch.setattr("artemis.runtime.server_lifecycle.is_port_in_use", lambda *a, **k: True)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_probe_is_not_part_of_the_default_console_report():
|
||||
assert "integration_host" not in ReadinessEngine()._probes
|
||||
probe = IntegrationHostProbe()
|
||||
assert probe.probe_id == "integration_host"
|
||||
assert probe.category is ProbeCategory.RUNTIME
|
||||
assert probe.is_blocker is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_consistent_host_passes(host_env):
|
||||
result = await IntegrationHostProbe().probe()
|
||||
|
||||
assert result.status is ProbeStatus.PASS
|
||||
assert result.is_blocker is True
|
||||
assert result.summary == "Host Ready"
|
||||
meta = result.metadata
|
||||
assert meta["venv_exists"] is True
|
||||
assert meta["interpreter_matches_venv"] is True
|
||||
assert Path(meta["runner_python"]) == host_probe.project_venv_python(host_env)
|
||||
assert meta["env_file_exists"] is False
|
||||
assert meta["traces_dir_writable"] is True
|
||||
assert (host_env / "traces").is_dir()
|
||||
assert meta["daemon"] == {
|
||||
"host": "127.0.0.1",
|
||||
"port": 8000,
|
||||
"standalone_forced": False,
|
||||
"reachable": True,
|
||||
"port_in_use": True,
|
||||
"port_held_by_other_process": False,
|
||||
"log_path": str(host_env / "daemon.log"),
|
||||
}
|
||||
labels = [a.label for a in result.actions]
|
||||
assert labels[0] == "Host OK"
|
||||
assert "Env File Location" in labels # informational even when passing
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_daemon_port_held_by_other_process_warns(host_env, monkeypatch):
|
||||
monkeypatch.setattr("artemis.runtime.daemon_client.is_artemis_daemon", lambda: False)
|
||||
|
||||
result = await IntegrationHostProbe().probe()
|
||||
|
||||
assert result.status is ProbeStatus.WARN
|
||||
assert result.is_blocker is False # tasks still run standalone
|
||||
assert result.metadata["daemon"]["port_held_by_other_process"] is True
|
||||
assert "port 8000 is held" in result.description.lower()
|
||||
port_fix = next(a for a in result.actions if a.label == "Free the Daemon Port")
|
||||
assert "ARTEMIS_DAEMON_PORT" in port_fix.payload
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_daemon_not_running_and_port_free_is_informational(host_env, monkeypatch):
|
||||
monkeypatch.setattr("artemis.runtime.daemon_client.is_artemis_daemon", lambda: False)
|
||||
monkeypatch.setattr("artemis.runtime.server_lifecycle.is_port_in_use", lambda *a, **k: False)
|
||||
|
||||
result = await IntegrationHostProbe().probe()
|
||||
|
||||
assert result.status is ProbeStatus.PASS
|
||||
assert "auto-start" in result.description
|
||||
auto = next(a for a in result.actions if a.label == "Daemon Auto-Start")
|
||||
assert str(host_env / "daemon.log") in auto.payload
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_standalone_mode_skips_daemon_probe(host_env, monkeypatch):
|
||||
monkeypatch.setattr("artemis.runtime.daemon_client.is_standalone_forced", lambda: True)
|
||||
monkeypatch.setattr("artemis.runtime.daemon_client.is_artemis_daemon", lambda: False)
|
||||
|
||||
result = await IntegrationHostProbe().probe()
|
||||
|
||||
assert result.status is ProbeStatus.PASS
|
||||
assert result.metadata["daemon"]["standalone_forced"] is True
|
||||
assert result.metadata["daemon"]["port_held_by_other_process"] is False
|
||||
auto = next(a for a in result.actions if a.label == "Daemon Auto-Start")
|
||||
assert "ARTEMIS_STANDALONE" in auto.payload
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interpreter_mismatch_warns_and_points_at_installer(host_env, monkeypatch):
|
||||
other = host_env / "other-python.exe"
|
||||
other.write_bytes(b"")
|
||||
monkeypatch.setattr(sys, "executable", str(other))
|
||||
|
||||
result = await IntegrationHostProbe().probe()
|
||||
|
||||
assert result.status is ProbeStatus.WARN
|
||||
assert result.metadata["interpreter_matches_venv"] is False
|
||||
assert Path(result.metadata["runner_python"]) == host_probe.project_venv_python(host_env)
|
||||
assert result.metadata["mcp_client"] is None
|
||||
assert not [a for a in result.actions if a.action_type == "command"]
|
||||
regen = next(a for a in result.actions if a.label == "Regenerate MCP Config")
|
||||
assert regen.action_type == "hint"
|
||||
assert "ask the user which ide" in regen.payload.lower()
|
||||
assert "uv run artemis mcp --install <client>" in regen.payload
|
||||
for name in host_probe.MCP_CLIENT_NAMES:
|
||||
assert name in regen.payload
|
||||
assert next(a for a in result.actions if a.label == "Interpreter Mismatch")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interpreter_mismatch_emits_runnable_install_for_detected_client(
|
||||
host_env, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("CLAUDECODE", "1")
|
||||
other = host_env / "other-python.exe"
|
||||
other.write_bytes(b"")
|
||||
monkeypatch.setattr(sys, "executable", str(other))
|
||||
|
||||
result = await IntegrationHostProbe().probe()
|
||||
|
||||
assert result.status is ProbeStatus.WARN
|
||||
assert result.metadata["mcp_client"] == "claude"
|
||||
regen = next(a for a in result.actions if a.action_type == "command")
|
||||
assert regen.label == "Regenerate MCP Config"
|
||||
assert regen.payload == "uv run artemis mcp --install claude"
|
||||
assert "<client>" not in regen.payload
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_reports_client_even_when_host_is_healthy(host_env, monkeypatch):
|
||||
monkeypatch.setenv("CURSOR_TRACE_ID", "abc")
|
||||
|
||||
result = await IntegrationHostProbe().probe()
|
||||
|
||||
assert result.status is ProbeStatus.PASS
|
||||
assert result.metadata["mcp_client"] == "cursor"
|
||||
|
||||
|
||||
def test_detect_mcp_client_unknown_in_clean_environment(clean_ide_env):
|
||||
assert host_probe.detect_mcp_client() is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("variables", "expected"),
|
||||
[
|
||||
({"ANTIGRAVITY_LS_ADDRESS": "127.0.0.1:1"}, "antigravity"),
|
||||
({"ANTIGRAVITY_CSRF_TOKEN": "t"}, "antigravity"),
|
||||
({"CLAUDECODE": "1"}, "claude"),
|
||||
({"CLAUDE_CODE_ENTRYPOINT": "cli"}, "claude"),
|
||||
({"CURSOR_TRACE_ID": "abc"}, "cursor"),
|
||||
({"CURSOR_AGENT": "1"}, "cursor"),
|
||||
({"WINDSURF_SESSION": "x"}, "windsurf"),
|
||||
({"CODEX_SANDBOX": "seatbelt"}, "codex"),
|
||||
({"CODEX_HOME": "/tmp/codex", "CODEX_SANDBOX_NETWORK_DISABLED": "1"}, "codex"),
|
||||
({"OPENCLAW_SESSION_ID": "s"}, "openclaw"),
|
||||
({"CLINE_TASK_ID": "1"}, "cline"),
|
||||
({"ROO_CODE_TASK": "1"}, "roo"),
|
||||
({"VSCODE_PID": "42"}, "vscode"),
|
||||
({"TERM_PROGRAM": "vscode"}, "vscode"),
|
||||
],
|
||||
)
|
||||
def test_detect_mcp_client_signals(clean_ide_env, variables, expected):
|
||||
for key, value in variables.items():
|
||||
clean_ide_env.setenv(key, value)
|
||||
assert host_probe.detect_mcp_client() == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"variables",
|
||||
[
|
||||
{"CODEX_HOME": "/home/me/.codex"}, # user relocated the Codex config dir
|
||||
{"OPENCLAW_WEBHOOK_URL": "https://hooks.example/x"}, # notifier setting, any host
|
||||
{"TERM_PROGRAM": "iTerm.app"},
|
||||
],
|
||||
)
|
||||
def test_detect_mcp_client_ignores_weak_or_user_set_variables(clean_ide_env, variables):
|
||||
for key, value in variables.items():
|
||||
clean_ide_env.setenv(key, value)
|
||||
assert host_probe.detect_mcp_client() is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("variables", "expected"),
|
||||
[
|
||||
# Claude Code or Codex running inside a VS Code / Cursor terminal: the
|
||||
# agent that spawned the server wins over the editor around it.
|
||||
({"CLAUDECODE": "1", "VSCODE_PID": "1", "TERM_PROGRAM": "vscode"}, "claude"),
|
||||
({"CLAUDECODE": "1", "CURSOR_TRACE_ID": "x"}, "claude"),
|
||||
({"CODEX_SANDBOX": "seatbelt", "VSCODE_PID": "1"}, "codex"),
|
||||
# Cursor / Windsurf / Cline are VS Code forks or extensions and inherit
|
||||
# the generic VS Code markers.
|
||||
({"CURSOR_TRACE_ID": "x", "VSCODE_PID": "1", "TERM_PROGRAM": "vscode"}, "cursor"),
|
||||
({"WINDSURF_SESSION": "x", "TERM_PROGRAM": "vscode"}, "windsurf"),
|
||||
({"CLINE_TASK_ID": "1", "VSCODE_PID": "1"}, "cline"),
|
||||
],
|
||||
)
|
||||
def test_detect_mcp_client_specific_host_beats_generic_vscode(clean_ide_env, variables, expected):
|
||||
for key, value in variables.items():
|
||||
clean_ide_env.setenv(key, value)
|
||||
assert host_probe.detect_mcp_client() == expected
|
||||
|
||||
|
||||
def test_mcp_install_actions_shapes():
|
||||
(cmd,) = host_probe.mcp_install_actions("antigravity")
|
||||
assert cmd.action_type == "command"
|
||||
assert cmd.payload == "uv run artemis mcp --install antigravity"
|
||||
|
||||
(hint,) = host_probe.mcp_install_actions(None)
|
||||
assert hint.action_type == "hint"
|
||||
assert "antigravity, claude, cursor, windsurf, vscode, cline, roo, openclaw, codex" in (
|
||||
hint.payload
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_venv_warns_with_uv_sync(host_env, monkeypatch):
|
||||
host_probe.project_venv_python(host_env).unlink()
|
||||
monkeypatch.setattr(sys, "executable", str(host_env / "system-python.exe"))
|
||||
|
||||
result = await IntegrationHostProbe().probe()
|
||||
|
||||
assert result.status is ProbeStatus.WARN
|
||||
assert result.metadata["venv_exists"] is False
|
||||
assert result.metadata["runner_python"] == str(host_env / "system-python.exe")
|
||||
assert any(a.payload == "uv sync" for a in result.actions)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unwritable_traces_dir_fails_even_with_other_warnings(host_env, monkeypatch):
|
||||
monkeypatch.setattr(host_probe, "directory_is_writable", lambda _p: (False, "denied"))
|
||||
monkeypatch.setattr("artemis.runtime.daemon_client.is_artemis_daemon", lambda: False)
|
||||
|
||||
result = await IntegrationHostProbe().probe()
|
||||
|
||||
assert result.status is ProbeStatus.FAIL
|
||||
assert result.is_blocker is True
|
||||
assert result.summary == "Host Misconfigured"
|
||||
assert result.metadata["traces_dir_writable"] is False
|
||||
assert result.metadata["traces_dir_error"] == "denied"
|
||||
assert result.actions[0].label == "Fix Traces Directory"
|
||||
assert "ARTEMIS_TRACES_DIR" in result.actions[0].payload
|
||||
# The port problem is still reported alongside the blocking failure.
|
||||
assert "port 8000 is held" in result.description.lower()
|
||||
|
||||
|
||||
def test_directory_is_writable_reports_real_errors(tmp_path):
|
||||
ok, error = host_probe.directory_is_writable(tmp_path / "new" / "nested")
|
||||
assert ok is True and error is None
|
||||
|
||||
blocker = tmp_path / "file"
|
||||
blocker.write_text("x")
|
||||
ok, error = host_probe.directory_is_writable(blocker / "child")
|
||||
assert ok is False and error
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wheel_install_without_venv_is_not_a_venv_warning(host_env, monkeypatch):
|
||||
"""A wheel install has no .venv under site-packages; that is normal, not a warning."""
|
||||
host_probe.project_venv_python(host_env).unlink()
|
||||
system_python = host_env / "system-python.exe"
|
||||
monkeypatch.setattr(sys, "executable", str(system_python))
|
||||
monkeypatch.setattr("artemis.config.paths.is_source_checkout", lambda: False)
|
||||
|
||||
result = await IntegrationHostProbe().probe()
|
||||
|
||||
assert result.status is ProbeStatus.PASS
|
||||
assert result.summary == "Host Ready"
|
||||
meta = result.metadata
|
||||
assert meta["is_source_checkout"] is False
|
||||
assert meta["venv_python"] is None
|
||||
assert meta["venv_exists"] is False
|
||||
assert meta["interpreter_matches_venv"] is False
|
||||
assert meta["runner_python"] == str(system_python)
|
||||
assert "venv" not in result.description.lower()
|
||||
assert not [a for a in result.actions if a.payload == "uv sync"]
|
||||
assert not [a for a in result.actions if a.label == "Interpreter Mismatch"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wheel_install_ignores_stray_venv_next_to_site_packages(host_env, monkeypatch):
|
||||
"""Even if a .venv happens to exist under ROOT_DIR, a non-source install never compares against it."""
|
||||
other = host_env / "other-python.exe"
|
||||
other.write_bytes(b"")
|
||||
monkeypatch.setattr(sys, "executable", str(other))
|
||||
monkeypatch.setattr("artemis.config.paths.is_source_checkout", lambda: False)
|
||||
|
||||
result = await IntegrationHostProbe().probe()
|
||||
|
||||
assert result.status is ProbeStatus.PASS
|
||||
assert result.metadata["venv_python"] is None
|
||||
assert result.metadata["runner_python"] == str(other)
|
||||
assert not [a for a in result.actions if a.label == "Interpreter Mismatch"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_source_checkout_without_venv_still_warns_with_uv_sync(host_env, monkeypatch):
|
||||
host_probe.project_venv_python(host_env).unlink()
|
||||
monkeypatch.setattr(sys, "executable", str(host_env / "system-python.exe"))
|
||||
monkeypatch.setattr("artemis.config.paths.is_source_checkout", lambda: True)
|
||||
|
||||
result = await IntegrationHostProbe().probe()
|
||||
|
||||
assert result.status is ProbeStatus.WARN
|
||||
assert result.metadata["is_source_checkout"] is True
|
||||
assert result.metadata["venv_python"] == str(host_probe.project_venv_python(host_env))
|
||||
assert result.metadata["venv_exists"] is False
|
||||
assert "no project virtualenv" in result.description.lower()
|
||||
assert any(a.payload == "uv sync" for a in result.actions)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_traces_path_creation_error_becomes_fail_not_crash(host_env, monkeypatch):
|
||||
"""get_default_traces_path mkdirs as a side effect; a read-only parent must not crash the probe."""
|
||||
|
||||
def denied():
|
||||
raise PermissionError(13, "Permission denied", str(host_env / "traces"))
|
||||
|
||||
monkeypatch.setattr("artemis.config.paths.get_default_traces_path", denied)
|
||||
monkeypatch.setenv("ARTEMIS_TRACES_DIR", str(host_env / "ro" / "traces"))
|
||||
|
||||
result = await IntegrationHostProbe().probe()
|
||||
|
||||
assert result.status is ProbeStatus.FAIL
|
||||
assert result.is_blocker is True
|
||||
assert result.summary == "Host Misconfigured"
|
||||
meta = result.metadata
|
||||
assert meta["traces_dir_writable"] is False
|
||||
assert "Permission denied" in meta["traces_dir_error"]
|
||||
# The intended path is still named, resolved without touching the disk.
|
||||
assert Path(meta["traces_dir"]) == host_env / "ro" / "traces"
|
||||
assert not (host_env / "ro").exists()
|
||||
assert result.actions[0].label == "Fix Traces Directory"
|
||||
assert "ARTEMIS_TRACES_DIR" in result.actions[0].payload
|
||||
assert "not writable" in result.description
|
||||
|
||||
|
||||
def test_intended_traces_path_mirrors_selection_without_mkdir(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr("artemis.config.paths.ROOT_DIR", tmp_path)
|
||||
monkeypatch.delenv("ARTEMIS_TRACES_DIR", raising=False)
|
||||
|
||||
monkeypatch.setattr("artemis.config.paths._use_user_app_dir", lambda: False)
|
||||
assert host_probe.intended_traces_path() == tmp_path / "traces"
|
||||
|
||||
monkeypatch.setattr("artemis.config.paths._use_user_app_dir", lambda: True)
|
||||
monkeypatch.setattr("artemis.config.paths.get_app_dir", lambda: tmp_path / "app")
|
||||
assert host_probe.intended_traces_path() == tmp_path / "app" / "traces"
|
||||
|
||||
monkeypatch.setenv("ARTEMIS_TRACES_DIR", str(tmp_path / "env-traces"))
|
||||
assert host_probe.intended_traces_path() == tmp_path / "env-traces"
|
||||
|
||||
assert not (tmp_path / "traces").exists()
|
||||
assert not (tmp_path / "app").exists()
|
||||
assert not (tmp_path / "env-traces").exists()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Daemon identity (a dev server on port 8000 is not the daemon)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, status: int, body: bytes):
|
||||
self.status = status
|
||||
self._body = body
|
||||
|
||||
def read(self) -> bytes:
|
||||
return self._body
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"status, body, expected",
|
||||
[
|
||||
(200, b'{"running": true, "port": 8000, "current_pid": 42}', True),
|
||||
(200, b"<!doctype html><title>Vite App</title>", False), # dev server on /
|
||||
(200, b'{"hello": "world"}', False),
|
||||
(404, b"", False),
|
||||
],
|
||||
)
|
||||
def test_is_artemis_daemon_requires_the_server_status_shape(monkeypatch, status, body, expected):
|
||||
from artemis.runtime import daemon_client
|
||||
|
||||
seen: dict[str, str] = {}
|
||||
|
||||
def fake_urlopen(req, timeout):
|
||||
seen["url"] = req.full_url
|
||||
return _FakeResponse(status, body)
|
||||
|
||||
monkeypatch.setattr(daemon_client.urllib.request, "urlopen", fake_urlopen)
|
||||
assert daemon_client.is_artemis_daemon("127.0.0.1", 8000) is expected
|
||||
assert seen["url"] == "http://127.0.0.1:8000/api/system/server-status"
|
||||
|
||||
|
||||
def test_is_artemis_daemon_is_false_when_nothing_listens(monkeypatch):
|
||||
from artemis.runtime import daemon_client
|
||||
|
||||
def refuse(req, timeout):
|
||||
raise ConnectionRefusedError()
|
||||
|
||||
monkeypatch.setattr(daemon_client.urllib.request, "urlopen", refuse)
|
||||
assert daemon_client.is_artemis_daemon("127.0.0.1", 8000) is False
|
||||
@@ -0,0 +1,157 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for the readiness core shared by ``artemis doctor`` and ``mobile_diagnose``."""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from artemis.core.diagnostics import readiness
|
||||
from artemis.core.diagnostics.probes.host_probe import IntegrationHostProbe
|
||||
from artemis.core.diagnostics.readiness import (
|
||||
CHECK_ORDER,
|
||||
adb_keys_corrupted,
|
||||
base_verdict,
|
||||
collect_readiness,
|
||||
sort_by_fix_order,
|
||||
)
|
||||
from artemis.core.diagnostics.schema import (
|
||||
ProbeCategory,
|
||||
ProbeResult,
|
||||
ProbeStatus,
|
||||
SystemReadinessReport,
|
||||
)
|
||||
|
||||
|
||||
def _probe(
|
||||
probe_id: str,
|
||||
status: ProbeStatus = ProbeStatus.PASS,
|
||||
*,
|
||||
blocker: bool = True,
|
||||
metadata: dict | None = None,
|
||||
) -> ProbeResult:
|
||||
return ProbeResult(
|
||||
id=probe_id,
|
||||
title=probe_id,
|
||||
category=ProbeCategory.RUNTIME,
|
||||
status=status,
|
||||
summary=status.value,
|
||||
description=f"{probe_id} is {status.value}",
|
||||
is_blocker=blocker,
|
||||
metadata=metadata or {},
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# base_verdict
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_base_verdict_without_any_blocker_is_blocked():
|
||||
assert base_verdict([]) == "blocked"
|
||||
assert base_verdict([_probe("toolchain", blocker=False)]) == "blocked"
|
||||
|
||||
|
||||
def test_base_verdict_failing_blocker_is_blocked():
|
||||
results = [
|
||||
_probe("python_runtime"),
|
||||
_probe("android_adb", ProbeStatus.FAIL),
|
||||
_probe("toolchain", blocker=False),
|
||||
]
|
||||
assert base_verdict(results) == "blocked"
|
||||
|
||||
|
||||
def test_base_verdict_passing_blockers_with_warning_optional_is_degraded():
|
||||
results = [
|
||||
_probe("python_runtime"),
|
||||
_probe("android_adb"),
|
||||
_probe("toolchain", ProbeStatus.WARN, blocker=False),
|
||||
]
|
||||
assert base_verdict(results) == "degraded"
|
||||
|
||||
|
||||
def test_base_verdict_all_pass_is_ready():
|
||||
results = [_probe("python_runtime"), _probe("android_adb"), _probe("toolchain", blocker=False)]
|
||||
assert base_verdict(results) == "ready"
|
||||
|
||||
|
||||
def test_base_verdict_warning_blocker_is_blocked_not_degraded():
|
||||
results = [_probe("python_runtime"), _probe("integration_host", ProbeStatus.WARN)]
|
||||
assert base_verdict(results) == "blocked"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# adb_keys_corrupted
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_adb_keys_corrupted_reads_the_adb_probe_flag():
|
||||
corrupted = _probe("android_adb", metadata={"adb_keys": {"is_corrupted": True}})
|
||||
healthy = _probe("android_adb", metadata={"adb_keys": {"is_corrupted": False}})
|
||||
assert adb_keys_corrupted([_probe("python_runtime"), corrupted]) is True
|
||||
assert adb_keys_corrupted([_probe("python_runtime"), healthy]) is False
|
||||
|
||||
|
||||
def test_adb_keys_corrupted_is_false_without_probe_or_metadata():
|
||||
assert adb_keys_corrupted([]) is False
|
||||
assert adb_keys_corrupted([_probe("python_runtime")]) is False
|
||||
assert adb_keys_corrupted([_probe("android_adb")]) is False
|
||||
assert adb_keys_corrupted([_probe("android_adb", metadata={"adb_keys": None})]) is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# ordering / collection
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_sort_by_fix_order_follows_check_order_and_appends_unknown_ids():
|
||||
results = [
|
||||
_probe("custom_probe"),
|
||||
_probe("toolchain", blocker=False),
|
||||
_probe("android_adb"),
|
||||
_probe("python_runtime"),
|
||||
]
|
||||
ordered = [r.id for r in sort_by_fix_order(results)]
|
||||
assert ordered == ["python_runtime", "android_adb", "toolchain", "custom_probe"]
|
||||
assert list(CHECK_ORDER) == [
|
||||
"python_runtime",
|
||||
"system_config",
|
||||
"integration_host",
|
||||
"gemini_api_key",
|
||||
"android_adb",
|
||||
"toolchain",
|
||||
"vision_ocr_key",
|
||||
]
|
||||
|
||||
|
||||
def test_collect_readiness_gathers_engine_report_and_host_probe(monkeypatch):
|
||||
report = SystemReadinessReport(
|
||||
overall_ready=True,
|
||||
blocker_count=1,
|
||||
passed_blocker_count=1,
|
||||
probes=[_probe("python_runtime")],
|
||||
os_type="windows",
|
||||
timestamp=time.time(),
|
||||
)
|
||||
host = _probe("integration_host")
|
||||
run_all = AsyncMock(return_value=report)
|
||||
monkeypatch.setattr(readiness.readiness_engine, "run_all", run_all)
|
||||
monkeypatch.setattr(IntegrationHostProbe, "probe", AsyncMock(return_value=host))
|
||||
|
||||
got_report, got_host = asyncio.run(collect_readiness())
|
||||
|
||||
assert got_report is report
|
||||
assert got_host is host
|
||||
run_all.assert_awaited_once_with(force_refresh=True)
|
||||
@@ -392,7 +392,7 @@ def _foreground_engine(tmp_path):
|
||||
|
||||
|
||||
def test_record_step_persists_explicit_foreground_app(tmp_path):
|
||||
"""M5: the historically dropped foreground_app parameter is persisted."""
|
||||
"""The foreground_app parameter is persisted."""
|
||||
engine = _foreground_engine(tmp_path)
|
||||
engine.record_step(
|
||||
foreground_app="com.android.settings",
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""A detached lens call (segment capsule, visual summary) must never be
|
||||
recorded as the current step's reasoning: the callback handler attaches
|
||||
``thinking``/``raw_thinking`` traces by the DataEngine's global
|
||||
``current_step_id``, which is independent of the trace context, so the
|
||||
handler itself has to recognise the detached lens context."""
|
||||
|
||||
import uuid
|
||||
from unittest.mock import Mock
|
||||
|
||||
from langchain_core.messages import AIMessage
|
||||
from langchain_core.outputs import ChatGeneration, LLMResult
|
||||
|
||||
from artemis.data_engine.context_vars import CURRENT_NODE_NAME, CURRENT_TRACE_ID
|
||||
from artemis.data_engine.trace import DataEngineCallbackHandler, detached_trace
|
||||
|
||||
CAPSULE = '{"doing": "scrolling", "did": "swiped", "entry_state": "home"}'
|
||||
|
||||
|
||||
def _handler():
|
||||
ctx = Mock()
|
||||
ctx.data_engine = Mock()
|
||||
ctx.data_engine.current_step_id = uuid.uuid4()
|
||||
ctx.data_engine.record_trace = Mock()
|
||||
return DataEngineCallbackHandler(ctx), ctx
|
||||
|
||||
|
||||
def _result(text: str) -> LLMResult:
|
||||
return LLMResult(generations=[[ChatGeneration(message=AIMessage(content=text))]])
|
||||
|
||||
|
||||
def _recorded(ctx) -> dict[str, dict]:
|
||||
return {c.kwargs["type"]: c.kwargs for c in ctx.data_engine.record_trace.call_args_list}
|
||||
|
||||
|
||||
def test_agent_call_reply_is_recorded_as_raw_thinking_under_the_step():
|
||||
handler, ctx = _handler()
|
||||
span = uuid.uuid4()
|
||||
token = CURRENT_TRACE_ID.set(span)
|
||||
token_name = CURRENT_NODE_NAME.set("operator")
|
||||
try:
|
||||
handler.on_llm_end(_result("I see the settings list."), run_id=uuid.uuid4())
|
||||
finally:
|
||||
CURRENT_TRACE_ID.reset(token)
|
||||
CURRENT_NODE_NAME.reset(token_name)
|
||||
recorded = _recorded(ctx)
|
||||
assert recorded["llm_call"]["step_id"] == ctx.data_engine.current_step_id
|
||||
assert recorded["raw_thinking"]["step_id"] == ctx.data_engine.current_step_id
|
||||
assert recorded["raw_thinking"]["payload"]["thought"] == "I see the settings list."
|
||||
|
||||
|
||||
def test_detached_lens_reply_is_neither_thinking_nor_attached_to_the_step():
|
||||
handler, ctx = _handler()
|
||||
span = uuid.uuid4()
|
||||
token = CURRENT_TRACE_ID.set(span)
|
||||
token_name = CURRENT_NODE_NAME.set("operator")
|
||||
try:
|
||||
with detached_trace("lens:step_capsule"):
|
||||
handler.on_llm_end(_result(CAPSULE), run_id=uuid.uuid4())
|
||||
finally:
|
||||
CURRENT_TRACE_ID.reset(token)
|
||||
CURRENT_NODE_NAME.reset(token_name)
|
||||
recorded = _recorded(ctx)
|
||||
assert "raw_thinking" not in recorded
|
||||
assert "thinking" not in recorded
|
||||
assert recorded["llm_call"]["step_id"] is None
|
||||
@@ -208,6 +208,219 @@ async def test_process_plan_write_only_queues(tmp_path):
|
||||
assert ctx.checkpoint_tasks == {}
|
||||
|
||||
|
||||
VERIFY_LINE = " - verify: the alarm list shows 7:30 AM\n"
|
||||
ASSERT_LINE = " - assert: a toast appeared\n"
|
||||
|
||||
|
||||
def _guidance_arrives(ctx, plan_text):
|
||||
"""What perception_node does when a non-empty user instruction lands: the
|
||||
check lines that exist at that moment lose their machine restoration."""
|
||||
from artemis.utils.plan_grammar import parse_plan
|
||||
|
||||
current = getattr(ctx, "guidance_unprotected_checks", None)
|
||||
if not isinstance(current, set):
|
||||
current = set()
|
||||
ctx.guidance_unprotected_checks = current | {
|
||||
(ci.kind, ci.text) for ci in parse_plan(plan_text).check_items
|
||||
}
|
||||
|
||||
|
||||
async def _plan_write(tmp_path, ctx, before, after):
|
||||
"""One accepted task_plan write through the shared post-write pipeline;
|
||||
returns the plan text as it stands on disk afterwards."""
|
||||
from artemis.graph.graph import _process_plan_write
|
||||
|
||||
task_plan_path = _write_plan(tmp_path, after)
|
||||
await _process_plan_write(ctx, _make_state(), task_plan_path, before, after, "ok", True)
|
||||
return task_plan_path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
NEW_VERIFY_LINE = " - verify: the build number is recorded\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_guidance_check_lines_may_be_dropped_at_any_later_write(tmp_path):
|
||||
"""Without guidance a deleted check line grows back. Once user guidance has
|
||||
arrived, the lines that existed at that moment stay editable for the rest
|
||||
of the run: the Operator re-plans over several writes, turns later, and
|
||||
none of them is undone."""
|
||||
without_verify = PLAN_WITH_CHECKS.replace(VERIFY_LINE, "")
|
||||
without_both = without_verify.replace(ASSERT_LINE, "")
|
||||
ctx = _make_ctx(tmp_path, disable_planner_validation=True)
|
||||
ctx.guidance_unprotected_checks = set()
|
||||
|
||||
# Ordinary write (no guidance): the deleted line grows back.
|
||||
assert VERIFY_LINE in await _plan_write(tmp_path, ctx, PLAN_WITH_CHECKS, without_verify)
|
||||
|
||||
# Turn N: guidance arrives; the Operator only advances a milestone.
|
||||
_guidance_arrives(ctx, PLAN_WITH_CHECKS)
|
||||
started_next = PLAN_WITH_CHECKS.replace("- [ ] Next milestone", "- [/] Next milestone")
|
||||
assert await _plan_write(tmp_path, ctx, PLAN_WITH_CHECKS, started_next) == started_next
|
||||
|
||||
# Turn N+2: the first check line goes, and stands.
|
||||
dropped_verify = started_next.replace(VERIFY_LINE, "")
|
||||
assert await _plan_write(tmp_path, ctx, started_next, dropped_verify) == dropped_verify
|
||||
# Turn N+3: the second one goes too, and also stands (no single-write window).
|
||||
dropped_both = dropped_verify.replace(ASSERT_LINE, "")
|
||||
assert await _plan_write(tmp_path, ctx, dropped_verify, dropped_both) == dropped_both
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_lines_added_after_guidance_stay_protected(tmp_path):
|
||||
"""A check line the Operator adds after the instruction is a new standard:
|
||||
deleting it later is restored, while the pre-guidance line still may go."""
|
||||
ctx = _make_ctx(tmp_path, disable_planner_validation=True)
|
||||
ctx.guidance_unprotected_checks = set()
|
||||
_guidance_arrives(ctx, PLAN_WITH_CHECKS)
|
||||
|
||||
with_new = PLAN_WITH_CHECKS.replace(ASSERT_LINE, ASSERT_LINE + NEW_VERIFY_LINE)
|
||||
assert await _plan_write(tmp_path, ctx, PLAN_WITH_CHECKS, with_new) == with_new
|
||||
|
||||
dropped_new = with_new.replace(NEW_VERIFY_LINE, "")
|
||||
on_disk = await _plan_write(tmp_path, ctx, with_new, dropped_new)
|
||||
assert NEW_VERIFY_LINE in on_disk # restored: it postdates the guidance
|
||||
|
||||
dropped_old = with_new.replace(VERIFY_LINE, "")
|
||||
on_disk = await _plan_write(tmp_path, ctx, with_new, dropped_old)
|
||||
assert VERIFY_LINE not in on_disk # stands: it predates the guidance
|
||||
assert NEW_VERIFY_LINE in on_disk
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_second_guidance_waives_the_lines_that_exist_by_then(tmp_path):
|
||||
ctx = _make_ctx(tmp_path, disable_planner_validation=True)
|
||||
ctx.guidance_unprotected_checks = set()
|
||||
_guidance_arrives(ctx, PLAN_WITH_CHECKS)
|
||||
with_new = PLAN_WITH_CHECKS.replace(ASSERT_LINE, ASSERT_LINE + NEW_VERIFY_LINE)
|
||||
assert await _plan_write(tmp_path, ctx, PLAN_WITH_CHECKS, with_new) == with_new
|
||||
dropped_new = with_new.replace(NEW_VERIFY_LINE, "")
|
||||
assert NEW_VERIFY_LINE in await _plan_write(tmp_path, ctx, with_new, dropped_new)
|
||||
|
||||
_guidance_arrives(ctx, with_new) # the new line exists now: it is waived too
|
||||
assert NEW_VERIFY_LINE not in await _plan_write(tmp_path, ctx, with_new, dropped_new)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mixed_write_restores_only_the_protected_lines(tmp_path):
|
||||
"""One write drops a pre-guidance line and a post-guidance line: only the
|
||||
protected one is merged back."""
|
||||
ctx = _make_ctx(tmp_path, disable_planner_validation=True)
|
||||
ctx.guidance_unprotected_checks = set()
|
||||
_guidance_arrives(ctx, PLAN_WITH_CHECKS)
|
||||
with_new = PLAN_WITH_CHECKS.replace(ASSERT_LINE, ASSERT_LINE + NEW_VERIFY_LINE)
|
||||
assert await _plan_write(tmp_path, ctx, PLAN_WITH_CHECKS, with_new) == with_new
|
||||
|
||||
dropped_both = with_new.replace(VERIFY_LINE, "").replace(NEW_VERIFY_LINE, "")
|
||||
on_disk = await _plan_write(tmp_path, ctx, with_new, dropped_both)
|
||||
assert VERIFY_LINE not in on_disk
|
||||
assert NEW_VERIFY_LINE in on_disk
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guided_drop_records_the_retired_lines_and_the_outcome_excludes_them(tmp_path):
|
||||
"""A line dropped under guidance is remembered as retired; a verdict the
|
||||
ledger recorded for it earlier is reported as retired, not as a failure,
|
||||
while a line still declared keeps its verdict."""
|
||||
from artemis.graph.checkpoints import compute_test_summary
|
||||
from artemis.utils.plan_grammar import parse_plan
|
||||
|
||||
ctx = _make_ctx(tmp_path, disable_planner_validation=True)
|
||||
ctx.guidance_unprotected_checks = set()
|
||||
ctx.guidance_retired_checks = set()
|
||||
_guidance_arrives(ctx, PLAN_WITH_CHECKS)
|
||||
without_verify = PLAN_WITH_CHECKS.replace(VERIFY_LINE, "")
|
||||
assert await _plan_write(tmp_path, ctx, PLAN_WITH_CHECKS, without_verify) == without_verify
|
||||
assert ctx.guidance_retired_checks == {("verify", "the alarm list shows 7:30 AM")}
|
||||
|
||||
records = [
|
||||
{"kind": "verify", "item_text": "the alarm list shows 7:30 AM", "status": "failed"},
|
||||
{"kind": "assert", "item_text": "a toast appeared", "status": "passed"},
|
||||
]
|
||||
summary = compute_test_summary(
|
||||
list(parse_plan(without_verify).all_check_items),
|
||||
records,
|
||||
retired=ctx.guidance_retired_checks,
|
||||
)
|
||||
assert (summary.passed, summary.failed, summary.retired) == (1, 0, 1)
|
||||
assert summary.retired_items == [
|
||||
{"item_text": "the alarm list shows 7:30 AM", "kind": "verify", "last_status": "failed"}
|
||||
]
|
||||
# A retired line that is declared again (the Operator re-added it) counts normally.
|
||||
summary = compute_test_summary(
|
||||
list(parse_plan(PLAN_WITH_CHECKS).all_check_items),
|
||||
records,
|
||||
retired=ctx.guidance_retired_checks,
|
||||
)
|
||||
assert (summary.passed, summary.failed, summary.retired) == (1, 1, 0)
|
||||
|
||||
|
||||
REWORDED_MILESTONE = PLAN_WITH_CHECKS.replace(f"- [x] {GOAL_TEXT}", "- [x] Create the 7:30 alarm")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rewording_a_milestone_keeps_its_check_lines_in_place(tmp_path):
|
||||
"""The check lines under a reworded milestone change signature (parent
|
||||
hash) but stay declared: they are neither duplicated as task-level @end
|
||||
orphans nor, under guidance, retired."""
|
||||
ctx = _make_ctx(tmp_path, disable_planner_validation=True)
|
||||
ctx.guidance_unprotected_checks = set()
|
||||
ctx.guidance_retired_checks = set()
|
||||
|
||||
# No guidance: nothing to restore, the plan stands as written.
|
||||
on_disk = await _plan_write(tmp_path, ctx, PLAN_WITH_CHECKS, REWORDED_MILESTONE)
|
||||
assert on_disk == REWORDED_MILESTONE
|
||||
assert "@end" not in on_disk
|
||||
|
||||
# Guidance armed: the same edit retires nothing.
|
||||
_guidance_arrives(ctx, PLAN_WITH_CHECKS)
|
||||
on_disk = await _plan_write(tmp_path, ctx, PLAN_WITH_CHECKS, REWORDED_MILESTONE)
|
||||
assert on_disk == REWORDED_MILESTONE
|
||||
assert ctx.guidance_retired_checks == set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_redeclaring_a_retired_line_reinstates_it(tmp_path):
|
||||
"""Dropped under guidance, then added back: the harvest gate must see the
|
||||
line as live again, matching what compute_test_summary reports."""
|
||||
from artemis.graph.checkpoints import reinstate_check_items
|
||||
|
||||
ctx = _make_ctx(tmp_path, disable_planner_validation=True)
|
||||
ctx.guidance_unprotected_checks = set()
|
||||
ctx.guidance_retired_checks = set()
|
||||
_guidance_arrives(ctx, PLAN_WITH_CHECKS)
|
||||
without_verify = PLAN_WITH_CHECKS.replace(VERIFY_LINE, "")
|
||||
await _plan_write(tmp_path, ctx, PLAN_WITH_CHECKS, without_verify)
|
||||
assert ctx.guidance_retired_checks == {RETIRED_VERIFY}
|
||||
|
||||
await _plan_write(tmp_path, ctx, without_verify, PLAN_WITH_CHECKS)
|
||||
assert ctx.guidance_retired_checks == set()
|
||||
|
||||
# Direct helper: unrelated items leave the set untouched.
|
||||
ctx.guidance_retired_checks = {RETIRED_VERIFY}
|
||||
reinstate_check_items(ctx, [RETIRED_ASSERT])
|
||||
assert ctx.guidance_retired_checks == {RETIRED_VERIFY}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejected_plan_write_leaves_the_waiver_untouched(tmp_path):
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
from artemis.graph.graph import _process_plan_write
|
||||
|
||||
ctx = _make_ctx(tmp_path, disable_planner_validation=True)
|
||||
ctx.guidance_unprotected_checks = set()
|
||||
_guidance_arrives(ctx, PLAN_WITH_CHECKS)
|
||||
waived = set(ctx.guidance_unprotected_checks)
|
||||
reworded = PLAN_WITH_CHECKS.replace("- [ ] Next milestone", "- [ ] The next milestone")
|
||||
task_plan_path = _write_plan(tmp_path, reworded)
|
||||
result = await _process_plan_write(
|
||||
ctx, _make_state(), task_plan_path, PLAN_WITH_CHECKS, reworded, "ok", False
|
||||
)
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert task_plan_path.read_text(encoding="utf-8") == PLAN_WITH_CHECKS
|
||||
assert ctx.guidance_unprotected_checks == waived
|
||||
|
||||
|
||||
# --- §8.1 / §8.10: spawn after record_step; record_step unconditional ----------------
|
||||
|
||||
|
||||
@@ -515,6 +728,146 @@ async def test_assert_fail_halt_policy_latches_halt(tmp_path):
|
||||
assert ctx.assert_halt is True
|
||||
|
||||
|
||||
# --- Retired check items: late verdicts are ledger-only ------------------------------
|
||||
|
||||
|
||||
RETIRED_VERIFY = ("verify", "the alarm list shows 7:30 AM")
|
||||
RETIRED_ASSERT = ("assert", "a toast appeared")
|
||||
|
||||
|
||||
def _last_done_event(ctx):
|
||||
calls = [
|
||||
c for c in ctx.data_engine._publish.call_args_list if c.args[1].get("status") == "done"
|
||||
]
|
||||
return calls[-1].args[1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_late_failing_verify_for_retired_line_is_ledger_only(tmp_path):
|
||||
"""The user retired the verify line but kept the milestone: a late failing
|
||||
verdict for it neither reopens the subgoal nor produces findings/headline,
|
||||
yet the ledger still books it and the done event reports it as retired."""
|
||||
plan_path = _write_plan(tmp_path, PLAN_WITH_CHECKS.replace(VERIFY_LINE, ""))
|
||||
ctx = _make_ctx(tmp_path)
|
||||
ctx.guidance_retired_checks = {RETIRED_VERIFY}
|
||||
run = _done_run(ctx, CheckReport(verdicts=[_verdict("verify", "failed")]))
|
||||
|
||||
findings = harvest_run(ctx, _make_state(), run, allow_side_effects=True)
|
||||
|
||||
assert findings == []
|
||||
assert f"- [x] {GOAL_TEXT}" in plan_path.read_text(encoding="utf-8")
|
||||
assert "- finding:" not in plan_path.read_text(encoding="utf-8")
|
||||
assert ctx.checkpoint_repairs == {}
|
||||
assert getattr(ctx, "checker_findings", {}) == {}
|
||||
records = read_ledger(tmp_path)
|
||||
assert [(r["kind"], r["status"]) for r in records] == [("verify", "failed")]
|
||||
event = _last_done_event(ctx)
|
||||
assert event["applicable"] is False
|
||||
assert event["reverted"] is False
|
||||
assert event["retired"] == [{"kind": "verify", "item_text": RETIRED_VERIFY[1]}]
|
||||
assert [v["status"] for v in event["verdicts"]] == ["failed"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_assert_for_retired_line_never_latches_halt(tmp_path):
|
||||
_write_plan(tmp_path, PLAN_WITH_CHECKS.replace(ASSERT_LINE, ""))
|
||||
ctx = _make_ctx(tmp_path, assert_failure_policy="halt")
|
||||
ctx.guidance_retired_checks = {RETIRED_ASSERT}
|
||||
run = _done_run(ctx, CheckReport(verdicts=[_verdict("assert", "failed")]))
|
||||
|
||||
harvest_run(ctx, _make_state(), run, allow_side_effects=True)
|
||||
|
||||
assert ctx.assert_halt is False
|
||||
assert read_ledger(tmp_path)[0]["status"] == "failed" # still booked
|
||||
assert _last_done_event(ctx)["applicable"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_retired_sibling_verdict_still_takes_effect(tmp_path):
|
||||
"""Same report: the retired verify is dropped, the live assert still halts
|
||||
and the live verify sibling still repairs."""
|
||||
plan_path = _write_plan(tmp_path, PLAN_WITH_CHECKS.replace(VERIFY_LINE, ""))
|
||||
ctx = _make_ctx(tmp_path, assert_failure_policy="halt")
|
||||
ctx.guidance_retired_checks = {RETIRED_VERIFY}
|
||||
run = _done_run(
|
||||
ctx,
|
||||
CheckReport(verdicts=[_verdict("verify", "failed"), _verdict("assert", "failed")]),
|
||||
)
|
||||
|
||||
findings = harvest_run(ctx, _make_state(), run, allow_side_effects=True)
|
||||
|
||||
assert findings == [] # the retired verify produces no repair finding
|
||||
assert f"- [x] {GOAL_TEXT}" in plan_path.read_text(encoding="utf-8")
|
||||
assert ctx.assert_halt is True # the live assert sibling still counts
|
||||
event = _last_done_event(ctx)
|
||||
assert event["applicable"] is True
|
||||
assert event["retired"] == [{"kind": "verify", "item_text": RETIRED_VERIFY[1]}]
|
||||
|
||||
# And a live verify sibling next to a retired one still drives the repair,
|
||||
# with the headline naming only the live criterion.
|
||||
live_line = " - verify: the alarm is enabled\n"
|
||||
plan_path = _write_plan(tmp_path, PLAN_WITH_CHECKS.replace(VERIFY_LINE, live_line))
|
||||
ctx = _make_ctx(tmp_path)
|
||||
ctx.guidance_retired_checks = {RETIRED_VERIFY}
|
||||
run = _done_run(
|
||||
ctx,
|
||||
CheckReport(
|
||||
verdicts=[
|
||||
_verdict("verify", "failed"),
|
||||
_verdict("verify", "failed", text="the alarm is enabled", evidence="toggle off"),
|
||||
]
|
||||
),
|
||||
)
|
||||
findings = harvest_run(ctx, _make_state(), run, allow_side_effects=True)
|
||||
content = plan_path.read_text(encoding="utf-8")
|
||||
assert f"- [/] {GOAL_TEXT}" in content
|
||||
assert "- finding: verify failed — 'the alarm is enabled'" in content
|
||||
assert RETIRED_VERIFY[1] not in " ".join(findings)
|
||||
assert any("the alarm is enabled" in f for f in findings)
|
||||
assert ctx.checker_finding_items[GOAL_KEY] == {("verify", "the alarm is enabled")}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retiring_an_item_withdraws_its_registered_headline(tmp_path):
|
||||
"""A headline registered by an earlier failing attempt disappears from the
|
||||
plan when the user guidance retires the item that produced it; a headline
|
||||
that still names a live item stays."""
|
||||
from artemis.graph.checkpoints import retire_check_items
|
||||
|
||||
plan_path = _write_plan(tmp_path)
|
||||
ctx = _make_ctx(tmp_path)
|
||||
run = _done_run(ctx, CheckReport(verdicts=[_verdict("verify", "failed")]))
|
||||
harvest_run(ctx, _make_state(), run, allow_side_effects=True)
|
||||
assert "- finding: verify failed" in plan_path.read_text(encoding="utf-8")
|
||||
|
||||
retire_check_items(ctx, [RETIRED_ASSERT]) # unrelated item: headline stays
|
||||
assert "- finding: verify failed" in plan_path.read_text(encoding="utf-8")
|
||||
assert ctx.guidance_retired_checks == {RETIRED_ASSERT}
|
||||
|
||||
retire_check_items(ctx, [RETIRED_VERIFY])
|
||||
assert "- finding:" not in plan_path.read_text(encoding="utf-8")
|
||||
assert ctx.checker_findings == {}
|
||||
assert ctx.checker_finding_items == {}
|
||||
assert ctx.guidance_retired_checks == {RETIRED_ASSERT, RETIRED_VERIFY}
|
||||
|
||||
# The plan-write path reaches the same helper: dropping a waived line
|
||||
# after a headline was registered removes the headline on that write.
|
||||
plan_path = _write_plan(tmp_path)
|
||||
ctx = _make_ctx(tmp_path, disable_planner_validation=True)
|
||||
ctx.guidance_unprotected_checks = set()
|
||||
ctx.guidance_retired_checks = set()
|
||||
run = _done_run(ctx, CheckReport(verdicts=[_verdict("verify", "failed")]))
|
||||
harvest_run(ctx, _make_state(), run, allow_side_effects=True)
|
||||
with_finding = plan_path.read_text(encoding="utf-8")
|
||||
assert "- finding: verify failed" in with_finding
|
||||
_guidance_arrives(ctx, with_finding)
|
||||
without_verify = with_finding.replace(VERIFY_LINE, "")
|
||||
on_disk = await _plan_write(tmp_path, ctx, with_finding, without_verify)
|
||||
assert VERIFY_LINE not in on_disk
|
||||
assert "- finding:" not in on_disk
|
||||
assert ctx.guidance_retired_checks == {RETIRED_VERIFY}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ledger_is_append_only_fail_then_pass_both_present(tmp_path):
|
||||
_write_plan(tmp_path)
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Perception side of the guidance waiver: a non-empty user instruction records
|
||||
the plan's current check lines in ``ctx.guidance_unprotected_checks`` on the run
|
||||
context (which outlives the per-turn State rebuild); stop-only or empty
|
||||
payloads do not, and an ordinary turn never touches the set."""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.graph.perception import perception_node
|
||||
from artemis.graph.state import State
|
||||
|
||||
|
||||
async def _run_perception(ctx, payload):
|
||||
with (
|
||||
patch("artemis.graph.perception.is_ocr_configured", return_value=False),
|
||||
patch("artemis.graph.perception.UnifiedMobileController") as controller_cls,
|
||||
patch("artemis.graph.perception._check_injected_instruction_file", return_value=payload),
|
||||
):
|
||||
device_data = MagicMock()
|
||||
device_data.width = 1080
|
||||
device_data.height = 2400
|
||||
device_data.base64 = "ZHVtbXk="
|
||||
device_data.elements = []
|
||||
controller = MagicMock()
|
||||
controller.get_screen_data = AsyncMock(return_value=device_data)
|
||||
controller_cls.return_value = controller
|
||||
|
||||
state = MagicMock(spec=State)
|
||||
state.structured_decisions = []
|
||||
return await perception_node(state, ctx)
|
||||
|
||||
|
||||
PLAN = "- [/] Set the alarm\n - verify: the alarm list shows 7:30 AM\n- [ ] Next milestone\n"
|
||||
|
||||
|
||||
def _ctx(tmp_path):
|
||||
from artemis.utils.notes import get_note_file_path
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = tmp_path
|
||||
ctx.data_engine.current_step_id = "step_1"
|
||||
ctx.device = MagicMock()
|
||||
ctx.guidance_unprotected_checks = set()
|
||||
plan_path = get_note_file_path(tmp_path, "task_plan")
|
||||
plan_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
plan_path.write_text(PLAN, encoding="utf-8")
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_instruction_waives_the_current_check_lines_and_still_reaches_state(tmp_path):
|
||||
ctx = _ctx(tmp_path)
|
||||
update = await _run_perception(ctx, {"instruction": "Skip the toast check"})
|
||||
assert ctx.guidance_unprotected_checks == {("verify", "the alarm list shows 7:30 AM")}
|
||||
assert update["injected_instruction"] == "Skip the toast check"
|
||||
assert update["user_stop_requested"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_only_or_blank_payload_does_not_arm(tmp_path):
|
||||
ctx = _ctx(tmp_path)
|
||||
update = await _run_perception(ctx, {"release_loop": True})
|
||||
assert update["user_stop_requested"] is True
|
||||
assert ctx.guidance_unprotected_checks == set()
|
||||
|
||||
update = await _run_perception(ctx, {"instruction": " ", "release_loop": True})
|
||||
assert update["user_stop_requested"] is True
|
||||
assert ctx.guidance_unprotected_checks == set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ordinary_turn_leaves_the_waiver_as_it_was(tmp_path):
|
||||
"""The per-turn reset applies to State (``injected_instruction`` goes back
|
||||
to None); the waiver on the context lives for the rest of the run."""
|
||||
ctx = _ctx(tmp_path)
|
||||
ctx.guidance_unprotected_checks = {("verify", "earlier line")}
|
||||
update = await _run_perception(ctx, None)
|
||||
assert update["injected_instruction"] is None
|
||||
assert ctx.guidance_unprotected_checks == {("verify", "earlier line")}
|
||||
@@ -77,6 +77,11 @@ async def test_ask_explorer_runs_the_pipeline_for_the_executor_agent():
|
||||
assert result.status == "success"
|
||||
assert "Explorer located 1 candidate(s) for 'blue Send button'" in result.text_summary
|
||||
assert "[1] 'Send' at normalized [500, 500]" in result.text_summary
|
||||
# This executor's click resolves an index against the element list the
|
||||
# candidates just joined, so the guidance teaches the index syntax (Pro
|
||||
# parity) next to the coordinates.
|
||||
assert "target=3, a bare integer" in result.text_summary
|
||||
assert "normalized coordinate" in result.text_summary
|
||||
# Candidates were registered on the state for index-based actions.
|
||||
assert state.indexed_points == [[540, 1200]]
|
||||
assert state.indexed_elements[0]["index"] == 1
|
||||
|
||||
@@ -12,12 +12,14 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Coordinate targets must be described by the model, never inferred.
|
||||
"""Targets are either observed (an element index) or described (coordinates).
|
||||
|
||||
The Flash/Validator dialect addresses every target by coordinates, so the
|
||||
executor refuses a click, long press, focused input, coordinate swipe or click
|
||||
sequence that does not say what it is aiming at. The description is recorded
|
||||
by the runner and never reaches the wire.
|
||||
Flash binds the Pro Operator's agent dialect: a click, long press or focused
|
||||
input names an element index from the indexed UI list -- resolved here to the
|
||||
element's center with its observed text/bounds/id recorded -- or a coordinate
|
||||
pair, which the model must describe itself. A coordinate swipe or click
|
||||
sequence must be described too. Nothing is inferred, the description never
|
||||
reaches the wire, and the wire only ever sees coordinates.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
@@ -44,6 +46,123 @@ def _state():
|
||||
return state
|
||||
|
||||
|
||||
def _indexed_state():
|
||||
"""Two observed elements (pixel centers, the shape ``observe()`` writes)."""
|
||||
state = _state()
|
||||
state.indexed_elements = [
|
||||
{
|
||||
"index": 1,
|
||||
"center": [540, 1248],
|
||||
"text": "Wi-Fi",
|
||||
"bounds": [40, 1152, 1040, 1344],
|
||||
"class": "android.widget.TextView",
|
||||
"resource_id": "android:id/title",
|
||||
"is_ocr": False,
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"center": [540, 720],
|
||||
"text": "Search settings",
|
||||
"bounds": [80, 660, 1000, 780],
|
||||
"class": "android.widget.EditText",
|
||||
"resource_id": "com.android.settings:id/search",
|
||||
"is_ocr": False,
|
||||
},
|
||||
]
|
||||
state.indexed_points = [el["center"] for el in state.indexed_elements]
|
||||
return state
|
||||
|
||||
|
||||
# --- Index targets: resolved against the element list, observed semantics recorded ---
|
||||
|
||||
|
||||
@pytest.mark.parametrize("index", [1, "1", 1.0, [1]])
|
||||
def test_click_index_resolves_to_the_elements_center_and_records_observed_fields(index):
|
||||
executor = _make_executor()
|
||||
wire_name, wire_args, _, recorded = executor._translate(
|
||||
"click", {"target": index}, _indexed_state()
|
||||
)
|
||||
assert wire_name == "click"
|
||||
# 540/1080 -> 500, 1248/2400 -> 520 (normalized 0-1000)
|
||||
assert wire_args == {"target": [500, 520], "times": 1, "delay_ms": 100}
|
||||
assert recorded == {
|
||||
"target_text": "Wi-Fi",
|
||||
"target_bounds": [37, 480, 963, 560],
|
||||
"target_resource_id": "android:id/title",
|
||||
"target_class": "android.widget.TextView",
|
||||
}
|
||||
assert "target_description" not in recorded
|
||||
|
||||
|
||||
def test_index_target_needs_no_description_and_ignores_a_stray_one():
|
||||
"""An index already names its element; the model's belief never mixes with
|
||||
the observation on the record."""
|
||||
executor = _make_executor()
|
||||
_, wire_args, _, recorded = executor._translate(
|
||||
"click", {"target": 2, "target_description": "the search box"}, _indexed_state()
|
||||
)
|
||||
assert wire_args["target"] == [500, 300]
|
||||
assert recorded["target_text"] == "Search settings"
|
||||
assert "target_description" not in recorded
|
||||
|
||||
|
||||
def test_long_press_and_input_text_resolve_indices_too():
|
||||
executor = _make_executor()
|
||||
_, press_args, _, recorded = executor._translate(
|
||||
"long_press", {"target": 1, "duration": 1500}, _indexed_state()
|
||||
)
|
||||
assert press_args == {"target": [500, 520], "duration_ms": 1500}
|
||||
assert recorded["target_text"] == "Wi-Fi"
|
||||
|
||||
_, type_args, _, recorded = executor._translate(
|
||||
"input_text", {"text": "wifi", "target": 2}, _indexed_state()
|
||||
)
|
||||
assert type_args == {"text": "wifi", "target": [500, 300], "clear_exist": True}
|
||||
assert recorded["target_resource_id"] == "com.android.settings:id/search"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name, label", [("click", "click"), ("long_press", "long press")])
|
||||
def test_out_of_range_index_is_refused_with_the_active_range(name, label):
|
||||
executor = _make_executor()
|
||||
with pytest.raises(_ArgError) as excinfo:
|
||||
executor._translate(name, {"target": 7}, _indexed_state())
|
||||
message = str(excinfo.value)
|
||||
assert message.startswith(f"Error during {label}: Invalid target index 7.")
|
||||
assert "1 to 2" in message
|
||||
assert "ask_explorer" in message
|
||||
|
||||
|
||||
def test_index_against_an_empty_list_is_refused():
|
||||
executor = _make_executor()
|
||||
with pytest.raises(_ArgError) as excinfo:
|
||||
executor._translate("click", {"target": 1}, _state())
|
||||
message = str(excinfo.value)
|
||||
assert "Invalid target index 1" in message
|
||||
assert "empty" in message
|
||||
|
||||
|
||||
def test_unusable_target_is_refused_with_both_syntaxes_named():
|
||||
executor = _make_executor()
|
||||
with pytest.raises(_ArgError) as excinfo:
|
||||
executor._translate("click", {"target": "the button"}, _indexed_state())
|
||||
message = str(excinfo.value)
|
||||
assert message.startswith("Error during click: Invalid target format")
|
||||
assert "element index" in message and "[x, y]" in message
|
||||
|
||||
|
||||
def test_index_resolution_uses_the_live_device_resolution():
|
||||
"""The element list is built against the device size ``observe()`` wrote
|
||||
on the context; a different resolution maps the same pixels differently."""
|
||||
executor = _make_executor(width=720, height=1600)
|
||||
state = _indexed_state()
|
||||
state.indexed_elements[0]["center"] = [360, 800]
|
||||
_, wire_args, _, _ = executor._translate("click", {"target": 1}, state)
|
||||
assert wire_args["target"] == [500, 500]
|
||||
|
||||
|
||||
# --- Coordinate targets: the model describes what it aims at -------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name, args, label",
|
||||
[
|
||||
|
||||
@@ -17,12 +17,11 @@
|
||||
Two kinds of protection:
|
||||
|
||||
* **Fixture pins** freeze the exact schema each model surface receives (operator
|
||||
shells, Validator/Flash declarations, the action server manifest). A schema change
|
||||
must be a conscious act: regenerate the fixture in the same commit and say why.
|
||||
* **Bridge derivation** re-derives every cross-dialect parameter difference from the
|
||||
generated schemas and requires it to be declared in ``ActionSpec.param_bridge``.
|
||||
Undeclared divergence between what one agent sees and what another sees -- the
|
||||
historical definition of drift -- fails here instead of shipping.
|
||||
shells, Flash declarations, the action server manifest). A schema change must be a
|
||||
conscious act: regenerate the fixture in the same commit and say why.
|
||||
* **Projection identity** checks that the Flash declaration of every shared action
|
||||
is the operator shell's own description and parameters, so the two profiles can
|
||||
never see different tools under one name -- the historical definition of drift.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -130,47 +129,71 @@ def test_validator_declaration_order_is_stable():
|
||||
]
|
||||
|
||||
|
||||
# --- Bridge derivation: dialect differences must be declared -------------------------
|
||||
# --- Projection identity: Flash binds the operator shell, projected ------------------
|
||||
|
||||
|
||||
def _operator_param_names(name: str) -> list[str]:
|
||||
schema = convert_to_openai_tool(operator_shell_tool(name))
|
||||
return list(schema["function"]["parameters"].get("properties", {}))
|
||||
def _without_optional_noise(schema: dict) -> dict:
|
||||
"""The pydantic-generated property schema minus what only marks optionality.
|
||||
|
||||
``default`` and the ``null`` alternative say "optional"; the declaration says
|
||||
that through ``required`` alone.
|
||||
"""
|
||||
out = {k: v for k, v in schema.items() if k not in ("anyOf", "default")}
|
||||
if "anyOf" in schema:
|
||||
members = [m for m in schema["anyOf"] if m.get("type") != "null"]
|
||||
out.update(members[0] if len(members) == 1 else {"anyOf": members})
|
||||
return out
|
||||
|
||||
|
||||
def _declaration_param_names(name: str) -> list[str]:
|
||||
return list(tool_declaration(name).parameters.get("properties", {}))
|
||||
|
||||
|
||||
def test_param_bridge_is_exact():
|
||||
"""Every operator<->declaration parameter difference is declared, none invented."""
|
||||
def test_flash_declarations_are_the_operator_shells_projected():
|
||||
"""One definition, two bindings: for every action with an operator shell, the
|
||||
Flash ``ToolDeclaration`` carries the shell's description, parameter names,
|
||||
descriptions, types (index-or-coordinates unions included) and required set."""
|
||||
for spec in ACTION_SPECS.values():
|
||||
if spec.operator is None or spec.declaration is None:
|
||||
assert spec.param_bridge == {}, (
|
||||
f"'{spec.name}' declares a param_bridge without both dialects."
|
||||
)
|
||||
if spec.operator is None:
|
||||
continue
|
||||
operator_names = _operator_param_names(spec.name)
|
||||
declaration_names = _declaration_param_names(spec.name)
|
||||
assert set(spec.param_bridge) == set(operator_names), (
|
||||
f"'{spec.name}': param_bridge keys must cover exactly the operator"
|
||||
f" params. bridge={sorted(spec.param_bridge)} operator={operator_names}"
|
||||
)
|
||||
assert set(spec.param_bridge.values()) == set(declaration_names), (
|
||||
f"'{spec.name}': param_bridge values must cover exactly the declaration"
|
||||
f" params. bridge={sorted(spec.param_bridge.values())}"
|
||||
f" declaration={declaration_names}"
|
||||
)
|
||||
|
||||
|
||||
def test_renamed_params_are_documented():
|
||||
"""A cross-dialect rename is a deliberate difference; it must carry prose."""
|
||||
for spec in ACTION_SPECS.values():
|
||||
renames = {k: v for k, v in spec.param_bridge.items() if k != v}
|
||||
if renames:
|
||||
assert spec.differences, (
|
||||
f"'{spec.name}' renames {renames} across dialects but documents no differences."
|
||||
shell = convert_to_openai_tool(operator_shell_tool(spec.name))["function"]
|
||||
declaration = tool_declaration(spec.name)
|
||||
assert declaration.description == shell["description"], spec.name
|
||||
shell_props = shell["parameters"]["properties"]
|
||||
decl_props = declaration.parameters["properties"]
|
||||
assert list(decl_props) == list(shell_props), spec.name
|
||||
for param, schema in shell_props.items():
|
||||
assert decl_props[param] == _without_optional_noise(schema), (
|
||||
f"'{spec.name}.{param}' diverges between the operator shell and the"
|
||||
" Flash declaration."
|
||||
)
|
||||
assert declaration.parameters["required"] == shell["parameters"].get("required", []), (
|
||||
spec.name
|
||||
)
|
||||
|
||||
|
||||
def test_point_targets_accept_an_index_or_coordinates_in_both_profiles():
|
||||
"""click / long_press / input_text address an element by index or by
|
||||
normalized coordinates; the description is required only for coordinates."""
|
||||
for name in ("click", "long_press", "input_text"):
|
||||
target = tool_declaration(name).parameters["properties"]["target"]
|
||||
assert target["anyOf"] == [
|
||||
{"type": "integer"},
|
||||
{"type": "array", "items": {"type": "integer"}},
|
||||
], name
|
||||
assert "target_description" not in tool_declaration(name).parameters["required"], name
|
||||
assert tool_declaration("click_sequence").parameters["required"] == [
|
||||
"sequence",
|
||||
"target_descriptions",
|
||||
]
|
||||
|
||||
|
||||
def test_shared_descriptions_are_identical_between_profiles():
|
||||
for name in ("swipe", "press_key", "manage_app", "wait_for_delay"):
|
||||
assert tool_declaration(name).description == operator_shell_tool(name).description
|
||||
|
||||
|
||||
def test_documented_differences_concern_the_wire_only():
|
||||
"""The agent dialect is single-sourced; a spec may only document how the
|
||||
agent dialect differs from the wire (spelling, vocabulary, resolution)."""
|
||||
for spec in ACTION_SPECS.values():
|
||||
assert "declaration dialect" not in spec.differences, spec.name
|
||||
|
||||
|
||||
# --- Manifest coverage ---------------------------------------------------------------
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -146,6 +146,7 @@ def test_mcp_stdio_handshake_immediate_input():
|
||||
"mobile_manage_task",
|
||||
"mobile_get_device_state",
|
||||
"mobile_inspect_trace",
|
||||
"mobile_diagnose",
|
||||
}.issubset(tool_names)
|
||||
|
||||
finally:
|
||||
|
||||
@@ -584,7 +584,8 @@ async def test_mobile_inspect_trace_step_details_replays_tool_calls_for_flash(te
|
||||
assert "error" not in res
|
||||
assert res["device_serial"] == "pixel-7"
|
||||
details = res["details"]
|
||||
assert "- **Step 1 (Start: 12.0s)**" in details
|
||||
# The header carries the session clock (session start 1000.0, step at 1012.0).
|
||||
assert "- **Step 1 (T+00:12)**" in details
|
||||
assert "[Screen]: Asked the explorer, then tapped Save." in details
|
||||
assert "Confirm the toggle first." in details
|
||||
assert "`ask_explorer(" in details
|
||||
|
||||
@@ -161,11 +161,13 @@ def _make(
|
||||
*,
|
||||
min_active: int = 2,
|
||||
max_steps: int = 12,
|
||||
min_steps: int = 3,
|
||||
target_tokens: int = 10**9,
|
||||
max_chunks: int = 8,
|
||||
max_eras: int | None = None,
|
||||
meter=None,
|
||||
auto_capsule: bool = False,
|
||||
start_ratio: float = 0.35,
|
||||
):
|
||||
ledger = TranscriptLedger(step_memory=StepMemoryService(ctx=None))
|
||||
engine = FakeEngine(steps)
|
||||
@@ -174,6 +176,7 @@ def _make(
|
||||
engine=engine,
|
||||
chunking_config=SimpleNamespace(
|
||||
max_steps=max_steps,
|
||||
min_steps=min_steps,
|
||||
target_source_tokens=target_tokens,
|
||||
model="test-model",
|
||||
max_chunks=max_chunks,
|
||||
@@ -181,12 +184,15 @@ def _make(
|
||||
),
|
||||
transcript_config=SimpleNamespace(
|
||||
context_budget_tokens=100_000,
|
||||
start_ratio=start_ratio,
|
||||
soft_ratio=0.7,
|
||||
hard_ratio=0.9,
|
||||
min_active_steps=min_active,
|
||||
),
|
||||
capsule_service=capsule,
|
||||
meter_getter=meter or (lambda: None),
|
||||
# No meter: the base comes from the ledger's operator-recorded prompt
|
||||
# size (None until recorded), exactly like production wiring.
|
||||
meter_getter=meter,
|
||||
goal="TEST GOAL",
|
||||
)
|
||||
ledger.attach_chunker(chunker)
|
||||
@@ -267,7 +273,7 @@ def test_action_ledger_renders_incident_and_burst_on_the_step_line():
|
||||
assert lines[0].startswith(
|
||||
"- Step 3 (T+00:30): Fast-action burst (2 actions, unvetted):"
|
||||
" Tapped 'player' at [5, 6] -> Tapped 'Skip' at [7, 8] -> Error: Execution failed"
|
||||
" (general, consecutive failure #1) on burst action 2/2 `Tapped 'Skip' at [7, 8]`:"
|
||||
" (general, consecutive failure #1) on burst action 2/2 `tap 'Skip' at [7, 8]`:"
|
||||
" Error: rejected"
|
||||
)
|
||||
assert "FA:" not in text
|
||||
@@ -654,7 +660,7 @@ def test_checkpoint_annotation_ignores_unmatched_subgoal():
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Era merging + recall-only overflow + L3 snapshot
|
||||
# Era merging + recall-only overflow + L3 fold
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -809,7 +815,7 @@ def test_period_paragraph_degrades_to_milestone_list_when_all_band1_missing():
|
||||
|
||||
|
||||
def test_independent_max_eras_cap_decouples_from_max_chunks():
|
||||
"""M5: ``chunking.max_eras`` caps eras independently; None follows
|
||||
"""``chunking.max_eras`` caps eras independently; None follows
|
||||
``max_chunks`` (the pre-M5 equal-value behavior, asserted by the
|
||||
overflow test above)."""
|
||||
# A generous era cap keeps every folded era's ledger despite max_chunks=2.
|
||||
@@ -822,42 +828,56 @@ def test_independent_max_eras_cap_decouples_from_max_chunks():
|
||||
assert [e for e in tight.eras if e.recall_only]
|
||||
|
||||
|
||||
def test_hard_threshold_renders_l3_snapshot_with_minimal_step_index():
|
||||
def _folded_steps(chunker) -> list[int]:
|
||||
return [
|
||||
n
|
||||
for era in chunker.eras
|
||||
if era.recall_only
|
||||
for c in era.chunks
|
||||
for n in range(c.start_step_number, c.end_step_number + 1)
|
||||
]
|
||||
|
||||
|
||||
def test_hard_threshold_folds_frozen_region_into_recall_only_era_with_step_index():
|
||||
"""L3 is a recall-only era: the period paragraph (① fields mechanically
|
||||
merged) plus the per-step minimal index of every folded chunk — step,
|
||||
session offset and action phrase, results only via search_history."""
|
||||
steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 13)]
|
||||
meter = {"value": None}
|
||||
ledger, chunker, _, capsule = _make(steps, min_active=2, meter=lambda: meter["value"])
|
||||
_run_turns(ledger, chunker, 1, 8, _hashes(4))
|
||||
capsule.resolve("chunk:1-4", _capsule(1, 4))
|
||||
|
||||
meter["value"] = 95_000 # ≥ 0.9 * 100k → L3
|
||||
meter["value"] = 95_000 # ≥ 0.9 * 100k → hard; the tiny turns cannot save 5k
|
||||
_run_turns(ledger, chunker, 9, 12, lambda i: "hash-c")
|
||||
|
||||
assert chunker.chunks == ()
|
||||
assert chunker.eras and all(era.recall_only for era in chunker.eras)
|
||||
frozen = _frozen_text(ledger)
|
||||
assert "[Session snapshot" in frozen
|
||||
assert "Overall goal: TEST GOAL" in frozen
|
||||
assert "Verified facts: fact-1" in frozen
|
||||
assert "Device/app state: Settings page open" in frozen
|
||||
# The minimal per-step index survives: step + time + action phrase for
|
||||
# every chunked step (steps 1–4 and the second event's chunk).
|
||||
assert "--- Step index (minimal width, per chunk) ---" in frozen
|
||||
for chunk in chunker.chunks:
|
||||
for n in range(chunk.start_step_number, chunk.end_step_number + 1):
|
||||
assert f"- Step {n} (T+" in frozen
|
||||
assert "-> dispatched" not in frozen.split("--- Step index")[1]
|
||||
assert "[Chunk" not in frozen
|
||||
assert "[Era 1 | Steps 1–" in frozen
|
||||
# The period paragraph carries the merged ① knowledge.
|
||||
assert "Working toward the login milestone." in frozen
|
||||
assert "Verified: fact-1" in frozen
|
||||
assert "(Step-level ledger via search_history for steps" in frozen
|
||||
# The minimal per-step index survives for every folded step.
|
||||
folded = _folded_steps(chunker)
|
||||
assert folded
|
||||
for n in folded:
|
||||
assert f"- Step {n} (T+" in frozen
|
||||
assert "-> dispatched" not in frozen # minimal width: no results
|
||||
|
||||
|
||||
def test_step_addressability_survives_every_compression_level():
|
||||
"""Hard invariant (§8, revised by §10 decision 5 final review): after L2
|
||||
chunks, era merges, and L3, every compressed step's number and
|
||||
session-offset time are directly readable; the extreme period-paragraph
|
||||
layer alone degrades to *period* addressability — step range and start/end
|
||||
offsets must be present in the header."""
|
||||
"""After L2 chunks, era merges, and recall-only periods,
|
||||
every compressed step's number and session-offset time stay directly
|
||||
readable — the recall-only layer keeps the per-step minimal index — and
|
||||
the period header pins the step range and start/end offsets."""
|
||||
ledger, chunker, _, _ = _many_chunks(6, max_chunks=3)
|
||||
frozen = _frozen_text(ledger)
|
||||
covered_steps = [
|
||||
n
|
||||
for era in chunker.eras
|
||||
if not era.recall_only
|
||||
for c in era.chunks
|
||||
for n in range(c.start_step_number, c.end_step_number + 1)
|
||||
] + [n for c in chunker.chunks for n in range(c.start_step_number, c.end_step_number + 1)]
|
||||
@@ -865,8 +885,9 @@ def test_step_addressability_survives_every_compression_level():
|
||||
for n in covered_steps:
|
||||
assert f"- Step {n} (T+" in frozen, f"step {n} lost its addressable line"
|
||||
|
||||
# Extreme layer: recall-only eras stay *period*-addressable — the header
|
||||
# regex pins step range and T+..→T+.. start/end offsets in place.
|
||||
# Extreme layer: recall-only eras stay period-addressable — the header
|
||||
# regex pins step range and T+..→T+.. start/end offsets in place — and
|
||||
# every member step keeps its index line.
|
||||
overflow_ledger, overflow_chunker, _, _ = _many_chunks(14, max_chunks=2)
|
||||
overflow_frozen = _frozen_text(overflow_ledger)
|
||||
recall_eras = [e for e in overflow_chunker.eras if e.recall_only]
|
||||
@@ -879,6 +900,8 @@ def test_step_addressability_survives_every_compression_level():
|
||||
assert re.search(pattern, overflow_frozen), (
|
||||
f"era {era.ordinal} lost its period-addressable header"
|
||||
)
|
||||
for n in _folded_steps(overflow_chunker):
|
||||
assert f"- Step {n} (T+" in overflow_frozen, f"folded step {n} lost its index line"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1077,9 +1100,11 @@ def test_failed_capsule_redispatches_and_retains_original():
|
||||
assert ledger.frozen_blocks == () # original text retained
|
||||
|
||||
|
||||
def test_hard_threshold_force_swaps_pending_chunks_into_l3():
|
||||
"""Emergency path: only the hard threshold may swap pending chunks — the
|
||||
frozen region becomes the L3 snapshot and ③'s minimal index survives."""
|
||||
def test_hard_threshold_force_swaps_pending_chunks_into_recall_only_era():
|
||||
"""Emergency path: only the hard threshold may swap pending chunks — they
|
||||
fold straight into a recall-only era (no chunk block ever), the period
|
||||
paragraph degrades to the milestone/step-range list, and ③'s minimal
|
||||
index survives for every folded step."""
|
||||
steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 9)]
|
||||
meter = {"value": None}
|
||||
ledger, chunker, _, _ = _make(steps, min_active=2, meter=lambda: meter["value"])
|
||||
@@ -1092,16 +1117,704 @@ def test_hard_threshold_force_swaps_pending_chunks_into_l3():
|
||||
ledger.render([_observation(9)])
|
||||
|
||||
assert chunker.awaiting_chunks == ()
|
||||
assert len(chunker.chunks) >= 1
|
||||
assert all(c.status == "pending" for c in chunker.chunks)
|
||||
assert chunker.chunks == ()
|
||||
assert len(chunker.eras) == 1 and chunker.eras[0].recall_only
|
||||
assert all(c.status == "pending" for c in chunker.eras[0].chunks)
|
||||
frozen = _frozen_text(ledger)
|
||||
assert "[Session snapshot" in frozen
|
||||
for chunk in chunker.chunks:
|
||||
for n in range(chunk.start_step_number, chunk.end_step_number + 1):
|
||||
assert f"- Step {n} (T+" in frozen
|
||||
assert "[Chunk" not in frozen and CHUNK_PENDING_NOTE not in frozen
|
||||
assert "[Era 1 | Steps 1–" in frozen
|
||||
assert "Milestones: segment (Steps 1–4)" in frozen # no band ① anywhere
|
||||
folded = _folded_steps(chunker)
|
||||
assert folded
|
||||
for n in folded:
|
||||
assert f"- Step {n} (T+" in frozen
|
||||
assert len(ledger.unchunked_turns()) < 8 # turns actually consumed
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Minimum chunk length for size / pressure closes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_pressure_close_waits_for_min_steps_instead_of_one_turn_chunks():
|
||||
"""Soft pressure with a single open segment: eligible turns age past the
|
||||
floor one per render; they accumulate to min_steps before closing, so the
|
||||
session does not spawn a one-turn chunk (one capsule call) per render."""
|
||||
steps = [_step(i) for i in range(1, 12)]
|
||||
ledger, chunker, _, _ = _make(steps, min_active=2, min_steps=3, meter=lambda: 75_000)
|
||||
_run_turns(ledger, chunker, 1, 5) # eligible = 2 at the step-5 render
|
||||
assert chunker.awaiting_chunks == ()
|
||||
_run_turns(ledger, chunker, 6, 6) # eligible = 3 → one 3-turn chunk
|
||||
assert [c.capsule_key for c in chunker.awaiting_chunks] == ["chunk:1-3"]
|
||||
assert chunker.awaiting_chunks[0].trigger == "pressure"
|
||||
_run_turns(ledger, chunker, 7, 8) # eligible beyond the claim: 1, then 2
|
||||
assert [c.capsule_key for c in chunker.awaiting_chunks] == ["chunk:1-3"]
|
||||
_run_turns(ledger, chunker, 9, 9)
|
||||
assert [c.capsule_key for c in chunker.awaiting_chunks] == ["chunk:1-3", "chunk:4-6"]
|
||||
|
||||
|
||||
def test_size_trigger_waits_for_min_steps_even_when_one_turn_exceeds_the_tokens():
|
||||
steps = [_step(i) for i in range(1, 9)]
|
||||
ledger, chunker, _, _ = _make(steps, min_active=2, min_steps=3, target_tokens=10)
|
||||
_run_turns(ledger, chunker, 1, 4) # eligible = 1, already over the token target
|
||||
assert chunker.awaiting_chunks == ()
|
||||
_run_turns(ledger, chunker, 5, 6) # eligible = 3
|
||||
assert [c.capsule_key for c in chunker.awaiting_chunks] == ["chunk:1-3"]
|
||||
assert chunker.awaiting_chunks[0].trigger == "size"
|
||||
|
||||
|
||||
def test_milestone_close_is_exempt_from_min_steps():
|
||||
steps = [_step(i, "hash-a" if i <= 1 else "hash-b") for i in range(1, 8)]
|
||||
ledger, chunker, _, _ = _make(steps, min_active=2, min_steps=3)
|
||||
_run_turns(ledger, chunker, 1, 4, _hashes(1)) # the 1-step segment aged past the floor
|
||||
assert [c.capsule_key for c in chunker.awaiting_chunks] == ["chunk:1-1"]
|
||||
assert chunker.awaiting_chunks[0].trigger == "milestone"
|
||||
|
||||
|
||||
def test_hard_threshold_waives_min_steps():
|
||||
steps = [_step(i) for i in range(1, 8)]
|
||||
meter = {"value": 75_000}
|
||||
ledger, chunker, _, _ = _make(steps, min_active=2, min_steps=3, meter=lambda: meter["value"])
|
||||
_run_turns(ledger, chunker, 1, 4) # eligible = 1 under soft only: held back
|
||||
assert chunker.awaiting_chunks == ()
|
||||
meter["value"] = 95_000
|
||||
_run_turns(ledger, chunker, 5, 5) # eligible = 2 < 3, but hard: closes (and folds)
|
||||
assert len(chunker.eras) == 1 and chunker.eras[0].recall_only
|
||||
assert chunker.eras[0].chunks[0].capsule_key == "chunk:1-2"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Start gate (swap held below budget*start_ratio) and the context base
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_start_gate_holds_ready_chunk_below_start_ratio_then_swaps():
|
||||
"""A closed segment prepares its capsule, but the original turns are
|
||||
replaced only once the operator's measured context reaches
|
||||
budget*start_ratio; until then the ready chunk waits in the queue."""
|
||||
steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 11)]
|
||||
meter = {"value": 20_000} # < 0.35 * 100k
|
||||
ledger, chunker, engine, _ = _make(
|
||||
steps, min_active=2, meter=lambda: meter["value"], auto_capsule=True
|
||||
)
|
||||
_run_turns(ledger, chunker, 1, 8, _hashes(4))
|
||||
|
||||
# Milestone closed steps 1–4 and the capsule is ready, yet nothing swapped.
|
||||
assert [c.status for c in chunker.awaiting_chunks] == ["ready"]
|
||||
assert chunker.chunks == ()
|
||||
assert ledger.frozen_blocks == ()
|
||||
assert len(ledger.unchunked_turns()) == 7
|
||||
# The timeline line stays running with a 'held' note carrying the gate.
|
||||
held = [
|
||||
t for t in engine.traces if t["status"] == "running" and t["payload"]["args"].get("note")
|
||||
]
|
||||
assert len(held) == 1
|
||||
args = held[0]["payload"]["args"]
|
||||
assert args["note"] == "held"
|
||||
assert (args["context_tokens"], args["context_budget"], args["swap_at_tokens"]) == (
|
||||
20_000,
|
||||
100_000,
|
||||
35_000,
|
||||
)
|
||||
|
||||
# Same render cadence, context now past the gate → the ready prefix swaps.
|
||||
meter["value"] = 40_000
|
||||
_run_turns(ledger, chunker, 9, 9, _hashes(4))
|
||||
assert len(chunker.chunks) == 1
|
||||
assert chunker.chunks[0].start_step_number == 1
|
||||
assert chunker.awaiting_chunks == ()
|
||||
assert ledger.frozen_blocks != ()
|
||||
done = engine.traces[-1]
|
||||
assert done["status"] == "success"
|
||||
assert done["payload"]["args"]["context_tokens"] == 40_000 - (
|
||||
chunker.chunks[0].source_chars // 4
|
||||
) + (chunker.chunks[0].summary_chars // 4)
|
||||
|
||||
|
||||
def test_leading_stampless_turn_joins_the_next_segment_instead_of_pinning_the_queue():
|
||||
"""A turn with no recorded step (a reply without a tool call) that lands
|
||||
right after a chunk boundary must not close as a stampless one-turn
|
||||
segment: such a chunk has no step records (``None``) and used to sit at
|
||||
the head of the awaiting queue, so no L2 swap ever happened and the hard
|
||||
line jumped straight to the fold. It joins the following segment."""
|
||||
steps = [_step(i) for i in range(1, 13)]
|
||||
meter = {"value": 50_000} # above the start gate
|
||||
ledger, chunker, _, capsule = _make(steps, min_active=2, meter=lambda: meter["value"])
|
||||
ledger.stage_turn([_observation(0), AIMessage(content="no tool call this turn")])
|
||||
_run_turns(ledger, chunker, 1, 7, _hashes(4)) # commits turn 0 without a step key
|
||||
|
||||
assert [c.capsule_key for c in chunker.awaiting_chunks] == ["chunk:1-4"]
|
||||
assert all(entry["chunk"] is not None for entry in chunker._awaiting)
|
||||
capsule.resolve("chunk:1-4", _capsule(1, 4))
|
||||
_run_turns(ledger, chunker, 8, 8, _hashes(4))
|
||||
|
||||
assert [c.capsule_key for c in chunker.chunks] == ["chunk:1-4"]
|
||||
assert chunker.awaiting_chunks == ()
|
||||
assert "[Chunk 1 | Steps 1–4" in _frozen_text(ledger)
|
||||
assert "no tool call this turn" not in _rendered_text(ledger.render([_observation(9)]))
|
||||
|
||||
|
||||
def test_partition_folds_leading_unstamped_turns_into_the_first_segment():
|
||||
ledger, chunker, _, _ = _make([_step(i) for i in range(1, 6)])
|
||||
chunker.on_step_stamped("s2", "a")
|
||||
chunker.on_step_stamped("s3", "a")
|
||||
chunker.on_step_stamped("s4", "b")
|
||||
turns = [
|
||||
{"step_key": None},
|
||||
{"step_key": "s1"},
|
||||
{"step_key": "s2"},
|
||||
{"step_key": "s3"},
|
||||
{"step_key": "s4"},
|
||||
{"step_key": None},
|
||||
]
|
||||
segments = chunker._partition(turns)
|
||||
assert [(h, [t["step_key"] for t in ts]) for h, ts in segments] == [
|
||||
("a", [None, "s1", "s2", "s3"]),
|
||||
("b", ["s4", None]),
|
||||
]
|
||||
# All unstamped: one stampless segment, never split.
|
||||
assert chunker._partition([{"step_key": None}, {"step_key": None}]) == [
|
||||
(None, [{"step_key": None}, {"step_key": None}])
|
||||
]
|
||||
|
||||
|
||||
def test_start_gate_zero_restores_swap_on_ready():
|
||||
steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 9)]
|
||||
ledger, chunker, _, _ = _make(
|
||||
steps, min_active=2, meter=lambda: 1_000, auto_capsule=True, start_ratio=0.0
|
||||
)
|
||||
_run_turns(ledger, chunker, 1, 8, _hashes(4))
|
||||
assert len(chunker.chunks) == 1
|
||||
assert chunker.awaiting_chunks == ()
|
||||
|
||||
|
||||
def test_start_gate_unknown_base_falls_through_to_swap_on_ready():
|
||||
"""No measured operator call (meter None, nothing recorded on the ledger):
|
||||
the gate cannot decide and the legacy swap-on-ready behavior stands."""
|
||||
steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 9)]
|
||||
ledger, chunker, _, _ = _make(steps, min_active=2, auto_capsule=True)
|
||||
assert ledger.last_prompt_tokens is None
|
||||
_run_turns(ledger, chunker, 1, 8, _hashes(4))
|
||||
assert len(chunker.chunks) == 1
|
||||
|
||||
|
||||
def _big_turn(i: int, chars: int = 4000) -> list:
|
||||
tool_call = {"name": "click", "args": {"target": i}, "id": f"tc{i}", "type": "tool_call"}
|
||||
return [
|
||||
_observation(i),
|
||||
AIMessage(content=f"thinking {i} " + "x" * chars, tool_calls=[tool_call]),
|
||||
ToolMessage(tool_call_id=f"tc{i}", content="Action Recorded"),
|
||||
]
|
||||
|
||||
|
||||
def _run_big_turns(ledger, chunker, start: int, upto: int, hash_for):
|
||||
for i in range(start, upto + 1):
|
||||
ledger.commit_staged(
|
||||
step_key=f"s{i - 1}" if i > 1 else None,
|
||||
validator_result={"status": "dispatched"} if i > 1 else None,
|
||||
)
|
||||
ledger.render([_observation(i)])
|
||||
ledger.stage_turn(_big_turn(i))
|
||||
chunker.on_step_stamped(f"s{i}", hash_for(i))
|
||||
|
||||
|
||||
def test_hard_threshold_swaps_ready_prefix_as_l2_when_that_is_enough():
|
||||
"""Hard ladder rung 1: with a ready prefix whose L2 swap brings the
|
||||
estimated context back under the hard line, the swap is an ordinary L2
|
||||
event — no fold, and the younger pending segment stays awaiting."""
|
||||
steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 13)]
|
||||
meter = {"value": None}
|
||||
ledger, chunker, engine, capsule = _make(steps, min_active=2, meter=lambda: meter["value"])
|
||||
# Steps 1–4 (hash-a) close at the switch; steps 5–8 (hash-b) close at the
|
||||
# next switch. Turns are ~1k tokens each so the L2 swap is worth ~4k.
|
||||
_run_big_turns(ledger, chunker, 1, 10, lambda i: "a" if i <= 4 else ("b" if i <= 8 else "c"))
|
||||
keys = [c.capsule_key for c in chunker.awaiting_chunks]
|
||||
assert keys[0] == "chunk:1-4"
|
||||
capsule.resolve("chunk:1-4", _capsule(1, 4)) # ready; chunk:5-8 stays pending
|
||||
|
||||
meter["value"] = 92_000 # ≥ 0.9 * 100k; minus ~4k of turns → under the line
|
||||
_run_big_turns(ledger, chunker, 11, 11, lambda i: "c")
|
||||
|
||||
assert [c.capsule_key for c in chunker.chunks] == ["chunk:1-4"]
|
||||
assert [c.capsule_key for c in chunker.awaiting_chunks] == ["chunk:5-8"]
|
||||
frozen = _frozen_text(ledger)
|
||||
assert "[Era" not in frozen # nothing folded
|
||||
assert "[Chunk 1" in frozen
|
||||
done = engine.traces[-1]
|
||||
assert done["status"] == "success"
|
||||
assert "forced" not in done["payload"]["args"]
|
||||
assert done["payload"]["args"]["context_tokens"] < 90_000
|
||||
|
||||
|
||||
def test_hard_threshold_escalates_to_fold_when_l2_swap_is_not_enough():
|
||||
"""Hard ladder rung 2: the ready prefix's L2 swap would not bring the
|
||||
context under the line → everything closed force-swaps, pending included,
|
||||
and the whole frozen region folds into one recall-only era whose index
|
||||
lists every folded step — the pending chunk's steps too."""
|
||||
steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 13)]
|
||||
meter = {"value": None}
|
||||
ledger, chunker, engine, capsule = _make(steps, min_active=2, meter=lambda: meter["value"])
|
||||
_run_turns(ledger, chunker, 1, 10, lambda i: "a" if i <= 4 else ("b" if i <= 8 else "c"))
|
||||
capsule.resolve("chunk:1-4", _capsule(1, 4))
|
||||
|
||||
meter["value"] = 99_000 # tiny test turns cannot shave 9k tokens
|
||||
_run_turns(ledger, chunker, 11, 11, lambda i: "c")
|
||||
|
||||
assert chunker.chunks == ()
|
||||
assert chunker.awaiting_chunks == ()
|
||||
assert len(chunker.eras) == 1 and chunker.eras[0].recall_only
|
||||
era = chunker.eras[0]
|
||||
assert [c.capsule_key for c in era.chunks] == ["chunk:1-4", "chunk:5-8"]
|
||||
assert [c.status for c in era.chunks] == ["ready", "pending"]
|
||||
frozen = _frozen_text(ledger)
|
||||
assert "[Chunk" not in frozen
|
||||
assert "[Era 1 | Steps 1–8" in frozen
|
||||
# The ready chunk's ① feeds the paragraph; the pending one falls back to
|
||||
# its step range — and both keep every step's index line.
|
||||
assert "Working toward the login milestone." in frozen
|
||||
assert "steps 5–8" in frozen
|
||||
for n in range(1, 9):
|
||||
assert f"- Step {n} (T+" in frozen, f"folded step {n} lost its index line"
|
||||
# Trace wording: only the chunk without a summary is a forced fold.
|
||||
done = {
|
||||
t["payload"]["args"]["start_step"]: t for t in engine.traces if t["status"] == "success"
|
||||
}
|
||||
assert "forced" not in done[1]["payload"]["args"]
|
||||
assert "folded into the period summary" in done[1]["payload"]["result"]
|
||||
assert done[5]["payload"]["args"]["forced"] is True
|
||||
assert done[5]["payload"]["result"] == (
|
||||
"Context budget reached before the summary was ready;"
|
||||
" 4 turns folded into the period summary."
|
||||
)
|
||||
|
||||
|
||||
def test_hard_fold_is_monotonic_later_l2_swaps_append_after_the_folded_era():
|
||||
"""After a hard fold, a later non-hard L2 swap appends a new chunk block
|
||||
while the folded steps stay inside the recall-only era — the fold never
|
||||
flips back to full chunk blocks."""
|
||||
steps = [_step(i) for i in range(1, 17)]
|
||||
hash_for = lambda i: "a" if i <= 4 else ("b" if i <= 8 else ("c" if i <= 12 else "d")) # noqa: E731
|
||||
meter = {"value": None}
|
||||
ledger, chunker, _, capsule = _make(steps, min_active=2, meter=lambda: meter["value"])
|
||||
_run_turns(ledger, chunker, 1, 10, hash_for)
|
||||
capsule.resolve("chunk:1-4", _capsule(1, 4))
|
||||
meter["value"] = 99_000
|
||||
_run_turns(ledger, chunker, 11, 11, hash_for) # hard: folds 1–4 and 5–8
|
||||
assert chunker.chunks == () and len(chunker.eras) == 1
|
||||
folded_block = _frozen_text(ledger)
|
||||
|
||||
# Pressure gone (above the start gate, below soft): the next milestone
|
||||
# switch closes 9–12; once ready it swaps as an ordinary L2 chunk.
|
||||
meter["value"] = 50_000
|
||||
_run_turns(ledger, chunker, 12, 15, hash_for)
|
||||
pending = [c for c in chunker.awaiting_chunks if c.status == "pending"]
|
||||
assert [c.capsule_key for c in pending] == ["chunk:9-12"]
|
||||
capsule.resolve("chunk:9-12", _capsule(9, 12))
|
||||
_run_turns(ledger, chunker, 16, 16, hash_for)
|
||||
|
||||
assert [c.capsule_key for c in chunker.chunks] == ["chunk:9-12"]
|
||||
assert len(chunker.eras) == 1 and chunker.eras[0].recall_only
|
||||
frozen = _frozen_text(ledger)
|
||||
assert frozen.startswith(folded_block) # the folded era is byte-identical
|
||||
assert "[Chunk 3 | Steps 9–12" in frozen
|
||||
assert "[Chunk 1" not in frozen and "[Chunk 2" not in frozen
|
||||
assert frozen.index("[Era 1") < frozen.index("[Chunk 3")
|
||||
for n in range(1, 9):
|
||||
assert f"- Step {n} (T+" in frozen
|
||||
|
||||
|
||||
def test_hard_threshold_with_nothing_awaiting_folds_frozen_region_once():
|
||||
"""Hard ladder last resort: the active window is already down to the
|
||||
sliding-window floor (nothing eligible to close, nothing awaiting), yet
|
||||
the prompt sits at the hard line — the existing L2 frozen region folds
|
||||
into a recall-only era without consuming turns, and repeated hard renders
|
||||
leave the frozen blocks byte-identical."""
|
||||
steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 13)]
|
||||
meter = {"value": 50_000}
|
||||
ledger, chunker, _, _ = _make(
|
||||
steps, min_active=2, meter=lambda: meter["value"], auto_capsule=True
|
||||
)
|
||||
_run_turns(ledger, chunker, 1, 7, _hashes(4))
|
||||
# Chunk 1–4 swapped as L2 at the step-7 render; s5..s6 remain = the floor.
|
||||
assert [c.capsule_key for c in chunker.chunks] == ["chunk:1-4"]
|
||||
assert chunker.awaiting_chunks == ()
|
||||
assert "[Chunk 1" in _frozen_text(ledger)
|
||||
assert len(ledger.unchunked_turns()) == 2
|
||||
|
||||
meter["value"] = 95_000
|
||||
ledger.render([_observation(8)]) # hard render, nothing eligible
|
||||
frozen = _frozen_text(ledger)
|
||||
assert "[Chunk 1" not in frozen and "[Era 1 | Steps 1–4" in frozen
|
||||
assert chunker.chunks == () and chunker.eras[0].recall_only
|
||||
for n in range(1, 5):
|
||||
assert f"- Step {n} (T+" in frozen
|
||||
assert len(ledger.unchunked_turns()) == 2 # no turns consumed
|
||||
folded_blocks = ledger.frozen_blocks
|
||||
ledger.render([_observation(8)])
|
||||
ledger.render([_observation(8)])
|
||||
assert ledger.frozen_blocks == folded_blocks # byte-stable across hard renders
|
||||
assert _frozen_text(ledger) == frozen
|
||||
|
||||
|
||||
def test_hard_threshold_overrides_start_gate():
|
||||
steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 9)]
|
||||
ledger, chunker, _, _ = _make(
|
||||
steps, min_active=2, meter=lambda: 95_000, auto_capsule=True, start_ratio=0.99
|
||||
)
|
||||
_run_turns(ledger, chunker, 1, 8, _hashes(4))
|
||||
assert chunker.awaiting_chunks == ()
|
||||
assert chunker.eras and all(era.recall_only for era in chunker.eras)
|
||||
assert "[Era 1 | Steps 1–" in _frozen_text(ledger)
|
||||
|
||||
|
||||
def test_context_base_reads_the_ledger_operator_measurement_not_the_session_meter():
|
||||
"""Production wiring: no meter_getter; the base is what the operator
|
||||
recorded on its own ledger. A session-wide meter reading from another
|
||||
agent's call (the 2026-09-08 trace: a 2.7k lite call right after the
|
||||
operator's 19.9k prompt) must not leak in."""
|
||||
from artemis.services.token_meter import get_meter
|
||||
|
||||
steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 11)]
|
||||
ledger, chunker, _, _ = _make(steps, min_active=2, auto_capsule=True)
|
||||
# Some other agent's small call lands on the session meter.
|
||||
get_meter("session-1").record({"prompt_tokens": 2_700, "completion_tokens": 10})
|
||||
# The operator's own measured prompt is well below the gate.
|
||||
ledger.record_prompt_tokens(20_000)
|
||||
assert chunker._context_base_tokens(ledger) == 20_000
|
||||
_run_turns(ledger, chunker, 1, 8, _hashes(4))
|
||||
assert chunker.chunks == () # held: 20k < 35k
|
||||
assert [c.status for c in chunker.awaiting_chunks] == ["ready"]
|
||||
|
||||
# Operator prompt grows past the gate → swap at the next render.
|
||||
ledger.record_prompt_tokens(36_000)
|
||||
_run_turns(ledger, chunker, 9, 9, _hashes(4))
|
||||
assert len(chunker.chunks) == 1
|
||||
|
||||
|
||||
def test_ledger_record_prompt_tokens_ignores_missing_or_non_positive():
|
||||
ledger = TranscriptLedger(step_memory=StepMemoryService(ctx=None))
|
||||
ledger.record_prompt_tokens(None)
|
||||
ledger.record_prompt_tokens(0)
|
||||
ledger.record_prompt_tokens("x")
|
||||
assert ledger.last_prompt_tokens is None
|
||||
ledger.record_prompt_tokens(1234)
|
||||
ledger.record_prompt_tokens(None)
|
||||
assert ledger.last_prompt_tokens == 1234
|
||||
|
||||
|
||||
def test_held_note_is_fresh_after_a_retry():
|
||||
"""A chunk whose capsule failed, was re-dispatched (``retrying`` note) and
|
||||
then readied while the start gate still holds it gets a fresh ``held``
|
||||
note: the re-dispatch resets the once-per-chunk held marker."""
|
||||
steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 14)]
|
||||
meter = {"value": 20_000} # < 0.35 * 100k: the gate holds ready chunks
|
||||
ledger, chunker, engine, capsule = _make(steps, min_active=2, meter=lambda: meter["value"])
|
||||
_run_turns(ledger, chunker, 1, 8, _hashes(4))
|
||||
entry = chunker._awaiting[0]
|
||||
key = entry["chunk"].capsule_key
|
||||
entry["held_announced"] = True # a stale marker must not survive a retry
|
||||
|
||||
capsule._failed.add(key)
|
||||
# The hash-b segment (5–8) ages past the floor at the step-11 render and
|
||||
# closes whole: a trigger render, which is a re-dispatch occasion.
|
||||
_run_turns(ledger, chunker, 9, 11, lambda i: "hash-c")
|
||||
assert capsule.submitted.count(key) == 2
|
||||
assert entry["held_announced"] is False
|
||||
notes = [t["payload"]["args"].get("note") for t in engine.traces if t["status"] == "running"]
|
||||
assert notes[-1] == "retrying"
|
||||
|
||||
capsule.resolve(key, _capsule(1, 4))
|
||||
_run_turns(ledger, chunker, 12, 12, lambda i: "hash-c")
|
||||
assert chunker.chunks == () # still held below the gate
|
||||
held = [
|
||||
t
|
||||
for t in engine.traces
|
||||
if t["status"] == "running"
|
||||
and t["payload"]["args"].get("note") == "held"
|
||||
and t["payload"]["args"]["start_step"] == 1
|
||||
]
|
||||
assert len(held) == 1
|
||||
assert engine.traces.index(held[0]) > max(
|
||||
i for i, t in enumerate(engine.traces) if t["payload"]["args"].get("note") == "retrying"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider-less prompt estimate + session-calibrated chars→tokens ratio
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_estimate_prompt_tokens_counts_text_images_and_tool_call_args():
|
||||
from artemis.memory.transcript import (
|
||||
IMAGE_BLOCK_TOKENS,
|
||||
estimate_prompt_tokens,
|
||||
measure_prompt_content,
|
||||
)
|
||||
|
||||
messages = [
|
||||
HumanMessage(content="a" * 400),
|
||||
HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "b" * 80},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG"}},
|
||||
{"type": "image", "source": {"data": "IMG2"}},
|
||||
{"type": "thinking", "thinking": "c" * 20},
|
||||
]
|
||||
),
|
||||
AIMessage(
|
||||
content="",
|
||||
tool_calls=[{"name": "click", "args": {"x": 1}, "id": "tc1", "type": "tool_call"}],
|
||||
),
|
||||
]
|
||||
text_chars, image_count = measure_prompt_content(messages)
|
||||
other_block = len(str({"type": "thinking", "thinking": "c" * 20}))
|
||||
assert image_count == 2
|
||||
assert text_chars == 400 + 80 + other_block + len('{"x": 1}')
|
||||
assert estimate_prompt_tokens(messages) == text_chars // 4 + 2 * IMAGE_BLOCK_TOKENS
|
||||
assert estimate_prompt_tokens([]) == 1 # never zero: an unknown base stays gated
|
||||
|
||||
|
||||
def _measured_prompt(text_chars: int, image_count: int = 0, char: str = "a"):
|
||||
"""One HumanMessage in the shape ``measure_prompt_content`` reads: a text
|
||||
block of ``text_chars`` characters and ``image_count`` image blocks."""
|
||||
blocks = [{"type": "text", "text": char * text_chars}]
|
||||
blocks += [
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,AAAA"}}
|
||||
for _ in range(image_count)
|
||||
]
|
||||
return [HumanMessage(content=blocks)]
|
||||
|
||||
|
||||
def test_ledger_calibrates_chars_per_token_from_measured_prompts():
|
||||
"""The first measured sample of a session goes through the whole-prompt
|
||||
fallback: ``text_chars / (prompt_tokens - images * IMAGE_BLOCK_TOKENS)``,
|
||||
clamped to [1.0, 6.0]."""
|
||||
|
||||
def fresh():
|
||||
return TranscriptLedger(step_memory=StepMemoryService(ctx=None))
|
||||
|
||||
ledger = fresh()
|
||||
assert ledger.chars_per_token == 4.0
|
||||
assert ledger.chars_to_tokens(1000) == 1000 // 4
|
||||
|
||||
# English-like prose at ~4 chars/token: the ratio is unchanged.
|
||||
ledger.record_prompt_tokens(1000, messages=[HumanMessage(content="a" * 4000)])
|
||||
assert ledger.chars_per_token == pytest.approx(4.0)
|
||||
assert ledger.chars_to_tokens(1000) == 250
|
||||
|
||||
# CJK-heavy prompt measured at ~1 token per char: the ratio drops to 1.
|
||||
ledger = fresh()
|
||||
ledger.record_prompt_tokens(4000, messages=[HumanMessage(content="中" * 4000)])
|
||||
assert ledger.chars_per_token == pytest.approx(1.0)
|
||||
assert ledger.chars_to_tokens(1000) == 1000
|
||||
|
||||
# Images are subtracted at their flat cost before dividing.
|
||||
ledger = fresh()
|
||||
ledger.record_prompt_tokens(1000 + 2 * 258, messages=_measured_prompt(4000, 2))
|
||||
assert ledger.chars_per_token == pytest.approx(4.0)
|
||||
|
||||
# Clamp bounds: [1.0, 6.0].
|
||||
ledger = fresh()
|
||||
ledger.record_prompt_tokens(10_000, messages=[HumanMessage(content="x" * 100)])
|
||||
assert ledger.chars_per_token == 1.0
|
||||
ledger = fresh()
|
||||
ledger.record_prompt_tokens(10, messages=[HumanMessage(content="x" * 10_000)])
|
||||
assert ledger.chars_per_token == 6.0
|
||||
|
||||
# Without messages only the base is recorded; the ratio is kept.
|
||||
ledger.record_prompt_tokens(777)
|
||||
assert ledger.last_prompt_tokens == 777 and ledger.chars_per_token == 6.0
|
||||
|
||||
|
||||
def test_ledger_first_sample_uses_the_whole_prompt_fallback():
|
||||
"""Before any usable difference exists the whole-prompt quotient is the
|
||||
only information available, so the first sample still applies it."""
|
||||
ledger = TranscriptLedger(step_memory=StepMemoryService(ctx=None))
|
||||
ledger.record_prompt_tokens(5000 + 3 * 258, messages=_measured_prompt(10_000, 3))
|
||||
assert ledger.chars_per_token == pytest.approx(2.0)
|
||||
assert ledger.last_prompt_tokens == 5000 + 3 * 258
|
||||
|
||||
|
||||
def test_ledger_differential_calibration_cancels_constant_overhead():
|
||||
"""Two same-image-count samples: the ratio is dchars/dtokens, and a large
|
||||
constant term in prompt_tokens (tool schemas, image tiling; 9000 tokens
|
||||
here) that the whole-prompt fallback mis-charges to the text cancels."""
|
||||
ledger = TranscriptLedger(step_memory=StepMemoryService(ctx=None))
|
||||
overhead = 9000
|
||||
ledger.record_prompt_tokens(overhead + 4000, messages=_measured_prompt(10_000, 2))
|
||||
fallback = 10_000 / (overhead + 4000 - 2 * 258)
|
||||
assert fallback < 1.0 # the fallback mis-reads ~0.8 and clamps to 1.0
|
||||
assert ledger.chars_per_token == 1.0
|
||||
|
||||
# +10000 chars, +4000 tokens, same two images: 2.5 exactly.
|
||||
ledger.record_prompt_tokens(overhead + 8000, messages=_measured_prompt(20_000, 2))
|
||||
assert ledger.chars_per_token == pytest.approx(2.5)
|
||||
|
||||
|
||||
def test_ledger_image_count_change_replaces_reference_without_updating():
|
||||
ledger = TranscriptLedger(step_memory=StepMemoryService(ctx=None))
|
||||
ledger.record_prompt_tokens(4000, messages=_measured_prompt(10_000, 2))
|
||||
# First usable pair (+10000 chars / +5000 tokens): pooled ratio 2.0.
|
||||
ledger.record_prompt_tokens(9000, messages=_measured_prompt(20_000, 2))
|
||||
assert ledger.chars_per_token == pytest.approx(2.0)
|
||||
|
||||
# Different image count: no ratio update, but it becomes the reference.
|
||||
ledger.record_prompt_tokens(14_000, messages=_measured_prompt(30_000, 3))
|
||||
assert ledger.chars_per_token == pytest.approx(2.0)
|
||||
|
||||
# The next 3-image sample differences against the 3-image reference
|
||||
# (+6000 chars / +2000 tokens), not against the 2-image one: pooled
|
||||
# (10000 + 6000) / (5000 + 2000).
|
||||
ledger.record_prompt_tokens(16_000, messages=_measured_prompt(36_000, 3))
|
||||
assert ledger.chars_per_token == pytest.approx(16_000 / 7000)
|
||||
|
||||
|
||||
def test_ledger_small_moves_hold_the_reference_until_they_accumulate():
|
||||
"""A sample that moved less than the delta floor (in either direction)
|
||||
neither updates the ratio nor replaces the reference: the movement keeps
|
||||
accumulating against the held reference until it is measurable."""
|
||||
from artemis.memory.transcript import MIN_CALIBRATION_DELTA_CHARS
|
||||
|
||||
ledger = TranscriptLedger(step_memory=StepMemoryService(ctx=None))
|
||||
ledger.record_prompt_tokens(2500, messages=_measured_prompt(10_000, 1))
|
||||
ledger.record_prompt_tokens(7500, messages=_measured_prompt(20_000, 1))
|
||||
assert ledger.chars_per_token == pytest.approx(2.0)
|
||||
|
||||
# Below the floor: no update, even though this pair's own quotient (1.0)
|
||||
# would differ wildly from the current ratio.
|
||||
small = MIN_CALIBRATION_DELTA_CHARS - 1
|
||||
ledger.record_prompt_tokens(7500 + small, messages=_measured_prompt(20_000 + small, 1))
|
||||
assert ledger.chars_per_token == pytest.approx(2.0)
|
||||
# A slight shrink in characters with a few more tokens (turn jitter) is
|
||||
# held the same way rather than resetting the baseline.
|
||||
ledger.record_prompt_tokens(7500 + small + 100, messages=_measured_prompt(20_000 + 100, 1))
|
||||
assert ledger.chars_per_token == pytest.approx(2.0)
|
||||
|
||||
# Reaching the floor differences against the HELD 20000-char / 7500-token
|
||||
# reference (+2000 chars / +2000 tokens), not the previous small sample:
|
||||
# pooled (10000 + 2000) / (5000 + 2000). Consecutive differencing would
|
||||
# have seen only +1900 chars here and never calibrated.
|
||||
ledger.record_prompt_tokens(
|
||||
9500, messages=_measured_prompt(20_000 + MIN_CALIBRATION_DELTA_CHARS, 1)
|
||||
)
|
||||
assert ledger.chars_per_token == pytest.approx(12_000 / 7000)
|
||||
|
||||
|
||||
def test_ledger_token_shrink_resets_the_reference():
|
||||
"""When the measured prompt shrinks in tokens (a compaction or scrub
|
||||
landed) the sample becomes a fresh baseline without touching the pooled
|
||||
ratio; text that grows at zero token cost is unusable and also resets."""
|
||||
ledger = TranscriptLedger(step_memory=StepMemoryService(ctx=None))
|
||||
ledger.record_prompt_tokens(2500, messages=_measured_prompt(10_000, 1))
|
||||
ledger.record_prompt_tokens(7500, messages=_measured_prompt(20_000, 1))
|
||||
assert ledger.chars_per_token == pytest.approx(2.0)
|
||||
|
||||
# Compaction: fewer tokens. Ratio untouched, baseline moves here.
|
||||
ledger.record_prompt_tokens(6000, messages=_measured_prompt(15_000, 1))
|
||||
assert ledger.chars_per_token == pytest.approx(2.0)
|
||||
# +2000 chars / +500 tokens against the new baseline: pooled 12000 / 5500.
|
||||
ledger.record_prompt_tokens(6500, messages=_measured_prompt(17_000, 1))
|
||||
assert ledger.chars_per_token == pytest.approx(12_000 / 5500)
|
||||
|
||||
# Grew past the floor at zero token cost: not a measurement; new baseline.
|
||||
ledger.record_prompt_tokens(6500, messages=_measured_prompt(19_000, 1))
|
||||
assert ledger.chars_per_token == pytest.approx(12_000 / 5500)
|
||||
ledger.record_prompt_tokens(7500, messages=_measured_prompt(21_000, 1))
|
||||
assert ledger.chars_per_token == pytest.approx(14_000 / 6500)
|
||||
|
||||
|
||||
def test_ledger_pools_deltas_rather_than_taking_the_last_pair():
|
||||
ledger = TranscriptLedger(step_memory=StepMemoryService(ctx=None))
|
||||
ledger.record_prompt_tokens(10_000, messages=_measured_prompt(10_000, 1))
|
||||
# Pair 1: +10000 chars / +5000 tokens (2.0).
|
||||
ledger.record_prompt_tokens(15_000, messages=_measured_prompt(20_000, 1))
|
||||
assert ledger.chars_per_token == pytest.approx(2.0)
|
||||
# Pair 2: +2000 chars / +2000 tokens (1.0). Pooled: 12000 / 7000.
|
||||
ledger.record_prompt_tokens(17_000, messages=_measured_prompt(22_000, 1))
|
||||
assert ledger.chars_per_token == pytest.approx(12_000 / 7000)
|
||||
assert ledger.chars_per_token != pytest.approx(1.0)
|
||||
|
||||
# A non-positive/None value neither disturbs the pooled ratio nor the
|
||||
# remembered reference: the next sample still differences against the
|
||||
# 22000-char one (+4000 / +1000 -> pooled 16000 / 8000).
|
||||
ledger.record_prompt_tokens(None, messages=_measured_prompt(50_000, 1))
|
||||
ledger.record_prompt_tokens(0, messages=_measured_prompt(50_000, 1))
|
||||
assert ledger.chars_per_token == pytest.approx(12_000 / 7000)
|
||||
ledger.record_prompt_tokens(18_000, messages=_measured_prompt(26_000, 1))
|
||||
assert ledger.chars_per_token == pytest.approx(2.0)
|
||||
|
||||
|
||||
def test_ledger_calibration_regression_on_real_operator_trace():
|
||||
"""Thirteen consecutive Gemini Operator prompts from a real 2026-09 run
|
||||
(session 28377d06, five full-size 1080x2424 screenshots on every call,
|
||||
tool schemas bound). Differencing against a held reference pools three
|
||||
measurable spans; the per-turn moves in between (a few hundred chars,
|
||||
some slightly negative with a few hundred more tokens) are held, not
|
||||
dropped. The whole-prompt quotient for the last call reads 1.62
|
||||
chars/token; the tail's own rate is denser, so the old estimator would
|
||||
have UNDER-counted the tail and fired the size trigger late here."""
|
||||
from artemis.memory.transcript import IMAGE_BLOCK_TOKENS
|
||||
|
||||
samples = [ # (text_chars, images, prompt_tokens) straight from the trace store
|
||||
(33262, 5, 20392),
|
||||
(33954, 5, 21231), # +692 chars: held
|
||||
(36094, 5, 22853), # +2832 chars / +2461 tokens vs 33262: pooled
|
||||
(36918, 5, 23332), # held
|
||||
(37519, 5, 23511), # held
|
||||
(38437, 5, 24345), # +2343 / +1492 vs 36094: pooled
|
||||
(38917, 5, 24741), # held
|
||||
(39332, 5, 25650), # held
|
||||
(42111, 5, 26248), # +3674 / +1903 vs 38437: pooled
|
||||
(43153, 5, 26910), # held
|
||||
(43023, 5, 27605), # chars down, tokens up (turn jitter): held
|
||||
(43027, 5, 27869), # held
|
||||
(43638, 5, 28176), # held
|
||||
]
|
||||
ledger = TranscriptLedger(step_memory=StepMemoryService(ctx=None))
|
||||
for chars, images, tokens in samples:
|
||||
ledger.record_prompt_tokens(tokens, messages=_measured_prompt(chars, images))
|
||||
|
||||
pooled = (2832 + 2343 + 3674) / (2461 + 1492 + 1903)
|
||||
assert ledger.chars_per_token == pytest.approx(pooled)
|
||||
assert ledger.chars_per_token == pytest.approx(1.51, abs=0.01)
|
||||
|
||||
chars, images, tokens = samples[-1]
|
||||
naive = chars / (tokens - images * IMAGE_BLOCK_TOKENS)
|
||||
assert naive == pytest.approx(1.62, abs=0.01)
|
||||
assert ledger.chars_per_token != pytest.approx(naive, abs=0.05)
|
||||
assert ledger.last_prompt_tokens == tokens
|
||||
|
||||
|
||||
def test_calibrated_ratio_scales_the_size_trigger():
|
||||
"""At a calibrated 1 char/token the size trigger closes a segment at about
|
||||
a quarter of the characters it needed at the default 4 chars/token."""
|
||||
steps = [_step(i) for i in range(1, 6)]
|
||||
|
||||
def eligible_chars(ledger):
|
||||
turns = ledger.unchunked_turns()
|
||||
return ledger.turn_text_chars(turns[: len(turns) - 2]) # min_active=2
|
||||
|
||||
# Default ratio: the eligible text (two ~4k-char turns once turn 5 is
|
||||
# staged) is worth ~chars/4 tokens, under the 5k target → nothing closes.
|
||||
ledger, chunker, _, _ = _make(
|
||||
steps, min_active=2, min_steps=2, max_steps=100, target_tokens=5000
|
||||
)
|
||||
ledger.record_prompt_tokens(1000, messages=[HumanMessage(content="a" * 4000)])
|
||||
_run_big_turns(ledger, chunker, 1, 5, lambda i: "hash-a")
|
||||
chars = eligible_chars(ledger)
|
||||
assert chars // 4 < 5000 <= chars
|
||||
assert chunker.awaiting_chunks == ()
|
||||
|
||||
# Calibrated to 1 char/token: the same text is worth ~chars tokens → closes.
|
||||
ledger, chunker, _, _ = _make(
|
||||
steps, min_active=2, min_steps=2, max_steps=100, target_tokens=5000
|
||||
)
|
||||
ledger.record_prompt_tokens(4000, messages=[HumanMessage(content="中" * 4000)])
|
||||
assert ledger.chars_per_token == pytest.approx(1.0)
|
||||
_run_big_turns(ledger, chunker, 1, 5, lambda i: "hash-a")
|
||||
assert [c.trigger for c in chunker.awaiting_chunks] == ["size"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Timeline announcement (compress_history trace)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1146,8 +1859,8 @@ def test_compression_announces_one_tool_line_from_dispatch_to_swap():
|
||||
assert "forced" not in done_args
|
||||
|
||||
|
||||
def test_forced_swap_announces_snapshot_without_a_ratio():
|
||||
"""Forced swaps report estimated context usage without a summary size."""
|
||||
def test_forced_swap_announces_fold_without_a_ratio():
|
||||
"""Forced folds report estimated context usage without a summary size."""
|
||||
steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 9)]
|
||||
meter = {"value": None}
|
||||
ledger, chunker, engine, _ = _make(steps, min_active=2, meter=lambda: meter["value"])
|
||||
@@ -1164,6 +1877,7 @@ def test_forced_swap_announces_snapshot_without_a_ratio():
|
||||
for trace in done:
|
||||
assert trace["payload"]["args"]["forced"] is True
|
||||
assert trace["payload"]["args"]["summary_tokens"] is None
|
||||
assert "folded into the period summary" in trace["payload"]["result"]
|
||||
last_args = done[-1]["payload"]["args"]
|
||||
assert last_args["context_budget"] == 100_000
|
||||
assert 0 < last_args["context_tokens"] < 95_000
|
||||
@@ -1171,6 +1885,63 @@ def test_forced_swap_announces_snapshot_without_a_ratio():
|
||||
assert all("context_tokens" not in t["payload"]["args"] for t in done[:-1])
|
||||
|
||||
|
||||
def test_compression_trace_carries_a_plain_language_phase_on_every_path():
|
||||
"""``args.phase`` says what the compression is doing without touching the
|
||||
trace status the rest of the UI keys on: summarizing (segment closed,
|
||||
capsule generating — also while retrying), ready (capsule on hand but the
|
||||
start gate holds the swap), applied (compressed block replaced the raw
|
||||
turns), failed (capsule generation gave up, full record kept)."""
|
||||
from artemis.memory.chunking import COMPRESSION_PHASES
|
||||
|
||||
assert COMPRESSION_PHASES == ("summarizing", "ready", "applied", "failed")
|
||||
|
||||
steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 15)]
|
||||
meter = {"value": 20_000} # < 0.35 * 100k: the start gate holds ready chunks
|
||||
ledger, chunker, engine, capsule = _make(steps, min_active=2, meter=lambda: meter["value"])
|
||||
_run_turns(ledger, chunker, 1, 8, _hashes(4))
|
||||
chunk = chunker.awaiting_chunks[0]
|
||||
key = chunk.capsule_key
|
||||
|
||||
def mine():
|
||||
return [
|
||||
(t["status"], t["payload"]["args"]["phase"])
|
||||
for t in engine.traces
|
||||
if t["trace_id"] == chunk.trace_id
|
||||
]
|
||||
|
||||
# Segment closed, capsule dispatched → summarizing.
|
||||
assert mine() == [("running", "summarizing")]
|
||||
|
||||
# Capsule generation exhausted → failed phase on the same trace.
|
||||
chunker._on_capsule_failed(key)
|
||||
assert mine()[-1] == ("failed", "failed")
|
||||
|
||||
# Re-dispatch at the next trigger render → summarizing again (note: retrying).
|
||||
capsule._failed.add(key)
|
||||
_run_turns(ledger, chunker, 9, 11, lambda i: "hash-c")
|
||||
assert capsule.submitted.count(key) == 2
|
||||
assert mine()[-1] == ("running", "summarizing")
|
||||
retrying = [t for t in engine.traces if t["trace_id"] == chunk.trace_id][-1]
|
||||
assert retrying["payload"]["args"]["note"] == "retrying"
|
||||
|
||||
# Capsule ready while the start gate holds → ready phase, status still running.
|
||||
capsule.resolve(key, _capsule(1, 4))
|
||||
_run_turns(ledger, chunker, 12, 12, lambda i: "hash-c")
|
||||
assert chunker.chunks == ()
|
||||
assert mine()[-1] == ("running", "ready")
|
||||
held = [t for t in engine.traces if t["trace_id"] == chunk.trace_id][-1]
|
||||
assert held["payload"]["args"]["note"] == "held"
|
||||
|
||||
# Context past the gate → the swap applies the summary.
|
||||
meter["value"] = 40_000
|
||||
_run_turns(ledger, chunker, 13, 13, lambda i: "hash-c")
|
||||
assert [c.start_step_number for c in chunker.chunks] == [1]
|
||||
assert mine()[-1] == ("success", "applied")
|
||||
|
||||
# Every announcement on every trace carries a known phase.
|
||||
assert all(t["payload"]["args"]["phase"] in COMPRESSION_PHASES for t in engine.traces)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Band ① note-linkage machine check
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -16,10 +16,13 @@
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import Mock
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.memory.step_memory import StepMemoryService
|
||||
from artemis.data_engine.context_vars import CURRENT_NODE_NAME, CURRENT_TRACE_ID
|
||||
from artemis.data_engine.trace import detached_trace
|
||||
from artemis.memory.step_memory import StepLens, StepMemoryService
|
||||
|
||||
|
||||
class _ScriptedService(StepMemoryService):
|
||||
@@ -161,6 +164,79 @@ async def test_flush_uses_configured_default_timeout():
|
||||
assert service._pending_tasks["slow"].done()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lens_job_runs_detached_from_the_submitters_trace_span():
|
||||
"""A job submitted while an operator step span is current (the task
|
||||
inherits that context) must not see the span: whatever the lens records
|
||||
can never be attributed to the caller's step. The caller's own context is
|
||||
untouched, and the job's node label names the lens."""
|
||||
seen: list[tuple] = []
|
||||
|
||||
class _Lens(StepLens):
|
||||
name = "probe"
|
||||
|
||||
async def render(self, key, payload):
|
||||
seen.append((CURRENT_TRACE_ID.get(), CURRENT_NODE_NAME.get()))
|
||||
return "rendered"
|
||||
|
||||
service = StepMemoryService(Mock(), lens=_Lens())
|
||||
operator_span = uuid.uuid4()
|
||||
token = CURRENT_TRACE_ID.set(operator_span)
|
||||
token_name = CURRENT_NODE_NAME.set("operator")
|
||||
try:
|
||||
service.submit("step-1", {"step_number": 1})
|
||||
await service.flush()
|
||||
assert CURRENT_TRACE_ID.get() == operator_span
|
||||
assert CURRENT_NODE_NAME.get() == "operator"
|
||||
finally:
|
||||
CURRENT_TRACE_ID.reset(token)
|
||||
CURRENT_NODE_NAME.reset(token_name)
|
||||
|
||||
assert service.get_summary("step-1") == "rendered"
|
||||
assert seen == [(None, "lens:probe")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lens_less_subclass_attempt_is_detached_too():
|
||||
"""The detachment lives in the runner, so a subclass with an inlined
|
||||
``_attempt`` (the Flash visual summarizer shape) gets it for free."""
|
||||
seen: list = []
|
||||
|
||||
async def attempt(svc, key):
|
||||
seen.append(CURRENT_TRACE_ID.get())
|
||||
svc._summaries[key] = "ok"
|
||||
return True
|
||||
|
||||
service = _ScriptedService(attempt)
|
||||
token = CURRENT_TRACE_ID.set(uuid.uuid4())
|
||||
try:
|
||||
service.submit("s", {"step_number": 1})
|
||||
await service.flush()
|
||||
finally:
|
||||
CURRENT_TRACE_ID.reset(token)
|
||||
assert seen == [None]
|
||||
assert service.trace_node_name == "lens:_scriptedservice"
|
||||
|
||||
|
||||
def test_detached_trace_restores_context_on_exception():
|
||||
span = uuid.uuid4()
|
||||
token = CURRENT_TRACE_ID.set(span)
|
||||
token_name = CURRENT_NODE_NAME.set("operator")
|
||||
try:
|
||||
with pytest.raises(RuntimeError):
|
||||
with detached_trace("lens:x"):
|
||||
assert CURRENT_TRACE_ID.get() is None
|
||||
assert CURRENT_NODE_NAME.get() == "lens:x"
|
||||
raise RuntimeError("boom")
|
||||
assert CURRENT_TRACE_ID.get() == span
|
||||
assert CURRENT_NODE_NAME.get() == "operator"
|
||||
with detached_trace(): # no node label: the name is left alone
|
||||
assert CURRENT_NODE_NAME.get() == "operator"
|
||||
finally:
|
||||
CURRENT_TRACE_ID.reset(token)
|
||||
CURRENT_NODE_NAME.reset(token_name)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resubmit_clears_failed_state():
|
||||
"""Re-submitting a failed job gives it a fresh retry budget."""
|
||||
|
||||
@@ -15,12 +15,12 @@
|
||||
"""Unit tests for the Pro session transcript ledger (history redesign §3.2, M2).
|
||||
|
||||
Covers the four-region discipline: S-region byte stability across turns, the
|
||||
append-only active region, depth-1 stripping of old UI lists and plan
|
||||
recitations, the depth-K screenshot scrub in the Pro message shape (grace /
|
||||
placeholder / freeze — the M1 race regressions re-run against HumanMessage
|
||||
observations keyed by step id), ``T+mm:ss`` session-offset timestamps, the
|
||||
tool-call/response pairing invariant, and the cold-start restored-history
|
||||
block.
|
||||
append-only active region, the single depth-K scrub edge in the Pro message
|
||||
shape (UI list / plan recitation strip, ephemeral-block deletion and the
|
||||
in-place screenshot swap in one pass; grace / placeholder / freeze — the M1
|
||||
race regressions re-run against HumanMessage observations keyed by step id),
|
||||
``T+mm:ss`` session-offset timestamps, the tool-call/response pairing
|
||||
invariant, the cold-start restored-history block, and the silent-turn flag.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -153,19 +153,30 @@ def test_validator_result_skipped_for_actionless_turn():
|
||||
assert not any(EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages)
|
||||
|
||||
|
||||
def test_old_turn_recitation_and_ui_list_stripped_at_depth_1():
|
||||
ledger = TranscriptLedger(step_memory=_service())
|
||||
def test_recitation_and_ui_list_stripped_at_the_text_edge_and_screenshot_at_k():
|
||||
"""Two edges (text at depth 1, screenshot at K=3): the plan recitation and
|
||||
UI list leave as soon as a newer observation exists, the screenshot stays
|
||||
until depth K and is then resolved in place; only the live tail ever
|
||||
carries an indexed element list."""
|
||||
ledger = TranscriptLedger(step_memory=_service(), image_scrub_depth=3)
|
||||
_play_turn(ledger, 1)
|
||||
rendered = _play_turn(ledger, 2, prev_key="step-1")
|
||||
obs1 = ledger.active_messages[0]
|
||||
# Turn 1 is now depth 2: text edge passed, screenshot kept.
|
||||
assert PLAN_RECITATION_MARKER not in str(obs1.content)
|
||||
assert PRO_UI_LIST_MARKER not in str(obs1.content)
|
||||
assert _has_image(obs1)
|
||||
original_obs1 = _fingerprint(obs1)
|
||||
|
||||
committed_obs = ledger.active_messages[0]
|
||||
committed_text = str(committed_obs.content)
|
||||
# Depth-1 scrub: the previous turn's plan recitation and UI list are gone.
|
||||
assert PLAN_RECITATION_MARKER not in committed_text
|
||||
assert PRO_UI_LIST_MARKER not in committed_text
|
||||
# Its screenshot is still inside the K-depth active window.
|
||||
assert _has_image(committed_obs)
|
||||
# The live tail keeps both.
|
||||
rendered = _play_turn(ledger, 3, prev_key="step-2")
|
||||
# Turn 1 reached depth 3: the screenshot is resolved.
|
||||
assert not _has_image(obs1)
|
||||
assert _fingerprint(obs1) != original_obs1
|
||||
# Turn 2 (depth 2): text edge passed, screenshot still there; the live tail keeps both.
|
||||
obs2 = next(m for m in ledger.active_messages if "OBSERVATION [T+00:02]" in str(m.content))
|
||||
assert PLAN_RECITATION_MARKER not in str(obs2.content)
|
||||
assert PRO_UI_LIST_MARKER not in str(obs2.content)
|
||||
assert _has_image(obs2)
|
||||
tail_text = str(rendered[-1].content)
|
||||
assert PLAN_RECITATION_MARKER in tail_text
|
||||
assert PRO_UI_LIST_MARKER in tail_text
|
||||
@@ -182,10 +193,16 @@ def test_depth_k_image_resolved_to_ready_visual_summary():
|
||||
|
||||
first_obs = ledger.active_messages[0]
|
||||
assert not _has_image(first_obs)
|
||||
assert {
|
||||
"type": "text",
|
||||
"text": "--- Historical Visual Transition ---\nObjective visual transition 1.",
|
||||
} in first_obs.content
|
||||
# (i) The summary sits where the image was, the label above it is gone,
|
||||
# recitation and UI list are stripped: exactly two blocks remain.
|
||||
assert first_obs.content == [
|
||||
{"type": "text", "text": "# CURRENT OBSERVATION [T+00:01]"},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "--- Historical Visual Transition ---\nObjective visual transition 1.",
|
||||
},
|
||||
]
|
||||
assert "--- Current Screenshot ---" not in str(first_obs.content)
|
||||
|
||||
|
||||
def test_pending_grace_then_placeholder_never_backfilled():
|
||||
@@ -418,3 +435,300 @@ def test_ledger_turn_transcript_reflects_the_scrubbed_active_region():
|
||||
assert "--- Action Execution Result (T+00:00) ---\nStatus: dispatched" in text
|
||||
assert "Status: failed" in ledger.turn_transcript(second)
|
||||
assert "thought 2" not in text # a turn transcript never leaks into another turn
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Occupancy-driven screenshot depth
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _image_turns(ledger) -> list[int]:
|
||||
"""1-based turn numbers whose observation still carries its screenshot."""
|
||||
return [
|
||||
i + 1
|
||||
for i, msg in enumerate(
|
||||
m for m in ledger.active_messages if "OBSERVATION" in str(m.content)
|
||||
)
|
||||
if _has_image(msg)
|
||||
]
|
||||
|
||||
|
||||
def _banded_ledger(**kwargs) -> TranscriptLedger:
|
||||
return TranscriptLedger(
|
||||
step_memory=_service(),
|
||||
image_scrub_depth=3,
|
||||
image_scrub_depth_relaxed=6,
|
||||
context_budget_tokens=100_000,
|
||||
start_ratio=0.35,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def test_relaxed_depth_below_start_gate_keeps_more_screenshots():
|
||||
"""Below the start gate the scrub edge sits at the relaxed depth. Depth K
|
||||
scrubs the K-th most recent screenshot with the live tail at depth 1, so
|
||||
K=6 keeps four historical screenshots in view (K=3 keeps one). An unknown
|
||||
occupancy counts as below the gate."""
|
||||
ledger = _banded_ledger()
|
||||
assert ledger.occupancy is None
|
||||
assert ledger.effective_image_scrub_depth == 6
|
||||
for i in range(1, 10):
|
||||
_play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
|
||||
# At the render of turn 9 its observation is the live tail (depth 1);
|
||||
# committed turns 8..5 are depths 2..5 and turn 4 (depth 6) is scrubbed.
|
||||
ledger.record_prompt_tokens(20_000) # 20% < 35%
|
||||
assert ledger.effective_image_scrub_depth == 6
|
||||
assert _image_turns(ledger) == [5, 6, 7, 8]
|
||||
|
||||
tight = TranscriptLedger(step_memory=_service(), image_scrub_depth=3)
|
||||
for i in range(1, 10):
|
||||
_play_turn(tight, i, prev_key=f"step-{i - 1}" if i > 1 else None)
|
||||
assert _image_turns(tight) == [8]
|
||||
|
||||
|
||||
def test_crossing_the_start_gate_tightens_once_and_relaxing_never_backfills():
|
||||
ledger = _banded_ledger()
|
||||
for i in range(1, 10):
|
||||
_play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
|
||||
assert _image_turns(ledger) == [5, 6, 7, 8]
|
||||
|
||||
# Past the gate: the next render scrubs the extra depths in one pass
|
||||
# (turns 5–8 were depths 3–6 at that render; only turn 9 at depth 2 stays).
|
||||
ledger.record_prompt_tokens(40_000) # 40% >= 35%
|
||||
assert ledger.effective_image_scrub_depth == 3
|
||||
_play_turn(ledger, 10, prev_key="step-9")
|
||||
assert _image_turns(ledger) == [9]
|
||||
|
||||
# Back under the gate: nothing is restored; new turns accumulate again
|
||||
# until four historical screenshots are in view.
|
||||
ledger.record_prompt_tokens(20_000)
|
||||
assert ledger.effective_image_scrub_depth == 6
|
||||
_play_turn(ledger, 11, prev_key="step-10")
|
||||
assert _image_turns(ledger) == [9, 10]
|
||||
_play_turn(ledger, 12, prev_key="step-11")
|
||||
assert _image_turns(ledger) == [9, 10, 11]
|
||||
_play_turn(ledger, 13, prev_key="step-12")
|
||||
assert _image_turns(ledger) == [9, 10, 11, 12]
|
||||
_play_turn(ledger, 14, prev_key="step-13") # turn 9 reaches depth 6
|
||||
assert _image_turns(ledger) == [10, 11, 12, 13]
|
||||
|
||||
|
||||
def test_no_band_configuration_keeps_the_fixed_depth():
|
||||
ledger = TranscriptLedger(step_memory=_service(), image_scrub_depth=3)
|
||||
ledger.record_prompt_tokens(1)
|
||||
assert ledger.effective_image_scrub_depth == 3
|
||||
relaxed_only = TranscriptLedger(
|
||||
step_memory=_service(), image_scrub_depth=3, image_scrub_depth_relaxed=6
|
||||
)
|
||||
assert relaxed_only.effective_image_scrub_depth == 3 # no gate → never relaxed
|
||||
|
||||
|
||||
def test_scrub_edge_runs_before_the_chunker_at_render():
|
||||
"""The chunker sees the active window after the scrub edge advanced, so a
|
||||
chunk closing over turns older than the floor never captures a live
|
||||
screenshot in its source transcript."""
|
||||
ledger = _banded_ledger()
|
||||
seen: list[list[int]] = []
|
||||
|
||||
class Probe:
|
||||
def on_render(self, led):
|
||||
seen.append(_image_turns(led))
|
||||
|
||||
ledger.attach_chunker(Probe())
|
||||
for i in range(1, 9):
|
||||
_play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None)
|
||||
# At the render of turn 8 the scrub edge (depth 6) had already resolved
|
||||
# turn 3 (depth 6); the probe saw turns 4–7 with images, never turn 3.
|
||||
assert seen[-1] == [4, 5, 6, 7]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ephemeral blocks and stable repeated renders
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _ready_service(n: int) -> StepMemoryService:
|
||||
service = _service()
|
||||
for i in range(1, n + 1):
|
||||
service._step_inputs[f"step-{i}"] = {"step_number": i}
|
||||
service._summaries[f"step-{i}"] = f"Visual transition {i}."
|
||||
return service
|
||||
|
||||
|
||||
def test_ephemeral_blocks_vanish_at_the_text_edge_and_not_before():
|
||||
"""(ii) Per-turn notices marked through ``mark_ephemeral`` stay only while
|
||||
the observation is the live tail; they are deleted at the text edge
|
||||
together with the strip, before the screenshot is resolved at K."""
|
||||
from artemis.memory.transcript import EPHEMERAL_BLOCKS_KEY, mark_ephemeral
|
||||
|
||||
ledger = TranscriptLedger(step_memory=_ready_service(6), image_scrub_depth=3)
|
||||
|
||||
def turn_with_notice(i: int) -> list:
|
||||
turn = _turn(i)
|
||||
obs = turn[0]
|
||||
obs.content = [
|
||||
obs.content[0],
|
||||
{"type": "text", "text": f"[Reminder for turn {i}: state your reasoning]"},
|
||||
*obs.content[1:],
|
||||
{"type": "text", "text": f"[User guidance for turn {i}]"},
|
||||
]
|
||||
mark_ephemeral(obs, [1, len(obs.content) - 1])
|
||||
return turn
|
||||
|
||||
ledger.commit_staged()
|
||||
ledger.render([turn_with_notice(1)[0]])
|
||||
ledger.stage_turn(turn_with_notice(1))
|
||||
for i in range(2, 4):
|
||||
ledger.commit_staged(step_key=f"step-{i - 1}")
|
||||
ledger.render([_observation(i)])
|
||||
ledger.stage_turn(turn_with_notice(i))
|
||||
|
||||
obs1 = ledger.active_messages[0]
|
||||
# At the render of turn 3, turn 1 is depth 3 (K): rewritten and frozen.
|
||||
assert obs1.content == [
|
||||
{"type": "text", "text": "# CURRENT OBSERVATION [T+00:01]"},
|
||||
{"type": "text", "text": "--- Historical Visual Transition ---\nVisual transition 1."},
|
||||
]
|
||||
assert EPHEMERAL_BLOCKS_KEY not in obs1.additional_kwargs
|
||||
# Turn 2 is depth 2: past the text edge, its notices are gone while the
|
||||
# screenshot is still there; the consumed indices are cleared.
|
||||
obs2 = next(m for m in ledger.active_messages if "OBSERVATION [T+00:02]" in str(m.content))
|
||||
assert "[Reminder for turn 2: state your reasoning]" not in str(obs2.content)
|
||||
assert "[User guidance for turn 2]" not in str(obs2.content)
|
||||
assert _has_image(obs2)
|
||||
assert EPHEMERAL_BLOCKS_KEY not in obs2.additional_kwargs
|
||||
|
||||
|
||||
def test_rendering_past_the_edge_is_byte_identical_and_prefix_stable():
|
||||
"""(iii) A render with nothing new is byte-identical to the previous one,
|
||||
and across turns exactly one message — the observation reaching depth K —
|
||||
differs from the previous render's prefix, so the request prefix before
|
||||
it is what the provider's prompt cache already holds."""
|
||||
ledger = TranscriptLedger(step_memory=_ready_service(12), image_scrub_depth=3)
|
||||
ledger.set_static_prefix([SystemMessage(content="STATIC")])
|
||||
|
||||
def snapshot(rendered: list) -> list[str]:
|
||||
return [_fingerprint(m) for m in rendered[:-1]] # everything but the live tail
|
||||
|
||||
previous: list[str] = []
|
||||
for i in range(1, 11):
|
||||
rendered = _play_turn(
|
||||
ledger,
|
||||
i,
|
||||
prev_key=f"step-{i - 1}" if i > 1 else None,
|
||||
prev_result={"status": "dispatched"} if i > 1 else None,
|
||||
)
|
||||
current = snapshot(rendered)
|
||||
changed = [idx for idx, fp in enumerate(previous) if current[idx] != fp]
|
||||
if i >= 3:
|
||||
# The live tail is depth 1 and committed turn i-1 depth 2, so the
|
||||
# turn reaching depth 3 is i-2; its observation heads its span.
|
||||
expected = next(
|
||||
idx
|
||||
for idx, m in enumerate(rendered[:-1])
|
||||
if f"OBSERVATION [T+00:0{(i - 2) % 10}]" in str(m.content)
|
||||
)
|
||||
assert changed == [expected], (i, changed)
|
||||
else:
|
||||
assert changed == [], (i, changed)
|
||||
previous = current
|
||||
|
||||
# Same state, rendered again: byte-identical, including the tail.
|
||||
tail = [_observation(11)]
|
||||
again = [_fingerprint(m) for m in ledger.render(tail)]
|
||||
assert again == [_fingerprint(m) for m in ledger.render(tail)]
|
||||
assert again[:-1] == previous
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Silent-turn flag
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _commit(messages: list) -> TranscriptLedger:
|
||||
ledger = TranscriptLedger(step_memory=_service())
|
||||
ledger.stage_turn(messages)
|
||||
ledger.commit_staged(step_key="s1")
|
||||
return ledger
|
||||
|
||||
|
||||
def test_last_turn_silent_when_ai_messages_show_no_text():
|
||||
click = [{"name": "click", "args": {"target": 1}, "id": "c1", "type": "tool_call"}]
|
||||
assert _commit([_observation(1), AIMessage(content="", tool_calls=click)]).last_turn_silent
|
||||
assert _commit([_observation(1), AIMessage(content=" \n", tool_calls=click)]).last_turn_silent
|
||||
assert _commit(
|
||||
[
|
||||
_observation(1),
|
||||
AIMessage(content=[{"type": "thinking", "thinking": "hidden"}], tool_calls=click),
|
||||
]
|
||||
).last_turn_silent
|
||||
assert _commit(
|
||||
[
|
||||
_observation(1),
|
||||
AIMessage(content=[{"type": "text", "text": " "}], tool_calls=click),
|
||||
ToolMessage(tool_call_id="c1", content="Action Recorded"),
|
||||
AIMessage(content=[], tool_calls=click),
|
||||
]
|
||||
).last_turn_silent
|
||||
|
||||
|
||||
def test_last_turn_not_silent_with_visible_text_or_without_ai_message():
|
||||
click = [{"name": "click", "args": {"target": 1}, "id": "c1", "type": "tool_call"}]
|
||||
assert not _commit(
|
||||
[_observation(1), AIMessage(content="I tap it.", tool_calls=click)]
|
||||
).last_turn_silent
|
||||
assert not _commit(
|
||||
[
|
||||
_observation(1),
|
||||
AIMessage(
|
||||
content=[{"type": "thinking", "thinking": "x"}, {"type": "text", "text": "Tap."}],
|
||||
tool_calls=click,
|
||||
),
|
||||
]
|
||||
).last_turn_silent
|
||||
# One silent call followed by a spoken one: the turn spoke.
|
||||
assert not _commit(
|
||||
[
|
||||
_observation(1),
|
||||
AIMessage(content="", tool_calls=click),
|
||||
ToolMessage(tool_call_id="c1", content="ok"),
|
||||
AIMessage(content="Now the real move.", tool_calls=click),
|
||||
]
|
||||
).last_turn_silent
|
||||
# No AI message at all is not a silent turn.
|
||||
assert not _commit([_observation(1)]).last_turn_silent
|
||||
# A tool result with empty text does not count as an AI message.
|
||||
assert not _commit(
|
||||
[_observation(1), ToolMessage(tool_call_id="c1", content="")]
|
||||
).last_turn_silent
|
||||
|
||||
|
||||
def test_last_turn_silent_tracks_each_commit():
|
||||
ledger = TranscriptLedger(step_memory=_service())
|
||||
click = [{"name": "click", "args": {}, "id": "c1", "type": "tool_call"}]
|
||||
assert ledger.last_turn_silent is False
|
||||
ledger.stage_turn([_observation(1), AIMessage(content="", tool_calls=click)])
|
||||
ledger.commit_staged(step_key="s1")
|
||||
assert ledger.last_turn_silent is True
|
||||
ledger.stage_turn([_observation(2), AIMessage(content="spoken", tool_calls=click)])
|
||||
ledger.commit_staged(step_key="s2")
|
||||
assert ledger.last_turn_silent is False
|
||||
|
||||
|
||||
def test_last_turn_silent_recognises_streamed_ai_message_chunks():
|
||||
"""Streamed replies are AIMessageChunk (type "AIMessageChunk"), not "ai"."""
|
||||
from langchain_core.messages import AIMessageChunk, HumanMessage
|
||||
|
||||
from artemis.memory.transcript import TranscriptLedger, render_turn_transcript
|
||||
|
||||
ledger = TranscriptLedger()
|
||||
ledger.stage_turn([HumanMessage(content="obs"), AIMessageChunk(content=[])])
|
||||
ledger.commit_staged(step_key=None, validator_result=None)
|
||||
assert ledger.last_turn_silent is True
|
||||
|
||||
ledger.stage_turn([HumanMessage(content="obs"), AIMessageChunk(content="I see the list.")])
|
||||
ledger.commit_staged(step_key=None, validator_result=None)
|
||||
assert ledger.last_turn_silent is False
|
||||
|
||||
rendered = render_turn_transcript([AIMessageChunk(content="I see the list.")])
|
||||
assert rendered.startswith("[operator]")
|
||||
@@ -0,0 +1,111 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Session and per-source cache-hit ratios.
|
||||
|
||||
A session whose operator calls never hit the provider's prompt cache must be
|
||||
visible without reading every ``llm_usage`` trace: the session snapshot and
|
||||
each source's totals carry ``cached_ratio`` (cached / prompt tokens).
|
||||
"""
|
||||
|
||||
import logging
|
||||
from unittest.mock import Mock
|
||||
from uuid import uuid4
|
||||
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
from artemis.services.token_meter import (
|
||||
SessionTokenMeter,
|
||||
cached_ratio,
|
||||
format_session_summary,
|
||||
get_meter,
|
||||
log_session_summary,
|
||||
record_llm_usage,
|
||||
)
|
||||
|
||||
|
||||
def _msg(prompt: int, cached: int = 0) -> AIMessage:
|
||||
usage = {"input_tokens": prompt, "output_tokens": 10, "total_tokens": prompt + 10}
|
||||
if cached:
|
||||
usage["input_token_details"] = {"cache_read": cached}
|
||||
return AIMessage(content="ok", usage_metadata=usage)
|
||||
|
||||
|
||||
def test_cached_ratio_is_zero_without_prompt_tokens():
|
||||
assert cached_ratio(0, 0) == 0.0
|
||||
assert cached_ratio(50, 0) == 0.0
|
||||
assert cached_ratio(250, 400) == 0.625
|
||||
|
||||
|
||||
def test_meter_tracks_cached_ratio_per_session_and_per_source():
|
||||
meter = SessionTokenMeter("s")
|
||||
meter.record({"prompt_tokens": 1000, "cached_tokens": 0}, source="google:operator")
|
||||
meter.record({"prompt_tokens": 1000, "cached_tokens": 0}, source="google:operator")
|
||||
snapshot = meter.record(
|
||||
{"prompt_tokens": 200, "cached_tokens": 150},
|
||||
source="lens:step_capsule:gemini",
|
||||
update_last_prompt=False,
|
||||
)
|
||||
|
||||
# Session ratio blends every call; the call's own source rides along.
|
||||
assert snapshot["session_cached_ratio"] == round(150 / 2200, 3)
|
||||
assert snapshot["source_cached_ratio"] == 0.75
|
||||
assert meter.last_prompt_tokens == 1000
|
||||
|
||||
summary = meter.summary()
|
||||
assert summary["session_id"] == "s"
|
||||
assert summary["sources"]["google:operator"] == {
|
||||
"llm_calls": 2,
|
||||
"prompt_tokens": 2000,
|
||||
"cached_tokens": 0,
|
||||
"cache_hit_calls": 0,
|
||||
"cached_ratio": 0.0, # the operator never hit the cache: visible at a glance
|
||||
}
|
||||
assert summary["sources"]["lens:step_capsule:gemini"]["cached_ratio"] == 0.75
|
||||
|
||||
|
||||
def test_record_llm_usage_payload_carries_both_ratios():
|
||||
engine = Mock()
|
||||
engine.current_session_id = uuid4()
|
||||
engine.current_step_id = None
|
||||
|
||||
record_llm_usage(engine, _msg(1000), source="google:op")
|
||||
payload = record_llm_usage(engine, _msg(1000, cached=800), source="google:op")
|
||||
|
||||
assert payload["session_cached_ratio"] == 0.4
|
||||
assert payload["source_cached_ratio"] == 0.4
|
||||
# A source-less call still reports the session ratio, never a stale source one.
|
||||
payload = record_llm_usage(engine, _msg(1000))
|
||||
assert payload["session_cached_ratio"] == round(800 / 3000, 3)
|
||||
assert "source_cached_ratio" not in payload
|
||||
|
||||
|
||||
def test_session_summary_logs_once_per_metered_session(caplog):
|
||||
session_id = f"ratio-{uuid4()}"
|
||||
assert log_session_summary(session_id) is None # never metered: nothing to say
|
||||
|
||||
meter = get_meter(session_id)
|
||||
meter.record({"prompt_tokens": 500, "cached_tokens": 0}, source="google:op")
|
||||
meter.record({"prompt_tokens": 100, "cached_tokens": 90}, source="lens:visual")
|
||||
|
||||
line = format_session_summary(session_id)
|
||||
assert f"LLM usage for session {session_id}: 2 calls, 600 prompt tokens, 90 cached" in line
|
||||
assert "cached_ratio=0.150" in line
|
||||
assert "google:op: 1 calls, cached_ratio=0.000" in line
|
||||
assert "lens:visual: 1 calls, cached_ratio=0.900" in line
|
||||
|
||||
with caplog.at_level(logging.INFO, logger="artemis.services.token_meter"):
|
||||
summary = log_session_summary(session_id)
|
||||
assert summary is not None and summary["session_cached_ratio"] == 0.15
|
||||
assert any(line in record.getMessage() for record in caplog.records)
|
||||
@@ -196,6 +196,7 @@ def test_cli_mcp_generate_config_antigravity():
|
||||
"mobile_manage_task",
|
||||
"mobile_get_device_state",
|
||||
"mobile_inspect_trace",
|
||||
"mobile_diagnose",
|
||||
}
|
||||
assert all(tool["eager"] is True for tool in legacy_config["tools"].values())
|
||||
|
||||
@@ -234,6 +235,7 @@ def test_cli_mcp_generate_config_codex():
|
||||
"mobile_manage_task",
|
||||
"mobile_get_device_state",
|
||||
"mobile_inspect_trace",
|
||||
"mobile_diagnose",
|
||||
]
|
||||
|
||||
|
||||
@@ -430,6 +432,7 @@ def test_cli_mcp_install_codex_preserves_config_and_is_idempotent(tmp_path, monk
|
||||
"mobile_manage_task",
|
||||
"mobile_get_device_state",
|
||||
"mobile_inspect_trace",
|
||||
"mobile_diagnose",
|
||||
]
|
||||
assert data["mcp_servers"]["artemis"]["env"]["PYTHONPATH"] == str(tmp_path)
|
||||
assert config_text.count("# BEGIN ARTEMIS MCP CONFIG") == 1
|
||||
@@ -680,3 +683,360 @@ def test_cli_server_lifecycle_aliases(monkeypatch):
|
||||
assert "restart" in help_result.output
|
||||
assert "stop" in help_result.output
|
||||
assert "status" in help_result.output
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# artemis doctor (readiness-engine backed)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_RAW_SECRET = "sk-RAWSECRET0123456789abcdef"
|
||||
|
||||
|
||||
def _probe(
|
||||
probe_id: str,
|
||||
*,
|
||||
status="pass",
|
||||
blocker: bool = True,
|
||||
summary: str = "OK",
|
||||
description: str = "fine",
|
||||
actions=(),
|
||||
metadata=None,
|
||||
):
|
||||
from artemis.core.diagnostics.schema import (
|
||||
ProbeAction,
|
||||
ProbeCategory,
|
||||
ProbeResult,
|
||||
ProbeStatus,
|
||||
)
|
||||
|
||||
return ProbeResult(
|
||||
id=probe_id,
|
||||
category=ProbeCategory.RUNTIME,
|
||||
title=f"Title {probe_id}",
|
||||
status=ProbeStatus(status),
|
||||
is_blocker=blocker,
|
||||
summary=summary,
|
||||
description=description,
|
||||
metadata=metadata or {},
|
||||
actions=[ProbeAction(action_type=t, label=lbl, payload=p) for t, lbl, p in actions],
|
||||
)
|
||||
|
||||
|
||||
def _all_pass_probes():
|
||||
return [
|
||||
_probe("vision_ocr_key", blocker=False, summary="Not Configured (Optional)"),
|
||||
_probe("toolchain", blocker=False),
|
||||
_probe("android_adb", metadata={"installed": True, "adb_keys": {"is_corrupted": False}}),
|
||||
_probe(
|
||||
"gemini_api_key",
|
||||
summary="Active (Gemini)",
|
||||
description="Gemini credential is active (sk-RAW...cdef).",
|
||||
metadata={
|
||||
"current_key": _RAW_SECRET,
|
||||
"api_keys": {"google": _RAW_SECRET},
|
||||
"providers": [{"raw_key": _RAW_SECRET, "key": _RAW_SECRET}],
|
||||
},
|
||||
),
|
||||
_probe("system_config"),
|
||||
_probe("python_runtime"),
|
||||
]
|
||||
|
||||
|
||||
def _install_doctor_fakes(monkeypatch, probes, host=None, *, heal_result=None):
|
||||
"""Patch the engine, the host probe, and the CLI-only extra rows."""
|
||||
from unittest.mock import AsyncMock
|
||||
import time
|
||||
|
||||
from artemis.core.diagnostics.engine import readiness_engine
|
||||
from artemis.core.diagnostics.probes.host_probe import IntegrationHostProbe
|
||||
from artemis.core.diagnostics.schema import ProbeStatus, SystemReadinessReport
|
||||
import artemis.interfaces.cli.commands.doctor as doctor_module
|
||||
|
||||
monkeypatch.setenv("COLUMNS", "200")
|
||||
blockers = [p for p in probes if p.is_blocker]
|
||||
report = SystemReadinessReport(
|
||||
overall_ready=all(p.status is ProbeStatus.PASS for p in blockers),
|
||||
blocker_count=len(blockers),
|
||||
passed_blocker_count=sum(p.status is ProbeStatus.PASS for p in blockers),
|
||||
probes=list(probes),
|
||||
os_type="windows",
|
||||
timestamp=time.time(),
|
||||
)
|
||||
run_all = AsyncMock(return_value=report)
|
||||
monkeypatch.setattr(readiness_engine, "run_all", run_all)
|
||||
heal = AsyncMock(return_value=heal_result or {"success": True, "message": "Keys regenerated."})
|
||||
monkeypatch.setattr(readiness_engine, "heal_adb_keys", heal)
|
||||
host_probe = AsyncMock(
|
||||
return_value=host or _probe("integration_host", summary="Host Ready", description="host ok")
|
||||
)
|
||||
monkeypatch.setattr(IntegrationHostProbe, "probe", host_probe)
|
||||
monkeypatch.setattr(
|
||||
doctor_module,
|
||||
"_npm_row",
|
||||
lambda: doctor_module.ExtraRow(
|
||||
key="nodejs_npm",
|
||||
title="Node.js / npm",
|
||||
status="pass",
|
||||
status_markup="[bold green]✔ Installed[/bold green]",
|
||||
summary="Installed",
|
||||
detail="/usr/bin/npm",
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
doctor_module,
|
||||
"_showcase_row",
|
||||
lambda: doctor_module.ExtraRow(
|
||||
key="showcase_ui",
|
||||
title="Showcase UI",
|
||||
status="missing",
|
||||
status_markup="[bold yellow]○ Not Compiled[/bold yellow]",
|
||||
summary="Not Compiled",
|
||||
detail="Run ./start.sh or artemis ui to auto-compile.",
|
||||
),
|
||||
)
|
||||
return {"run_all": run_all, "heal": heal, "host": host_probe}
|
||||
|
||||
|
||||
def test_cli_doctor_all_pass_renders_table_in_fix_order(monkeypatch):
|
||||
"""All-green run: engine rows in fix order, extras last, exit 0, ready footer."""
|
||||
_install_doctor_fakes(monkeypatch, _all_pass_probes())
|
||||
|
||||
result = runner.invoke(app, ["doctor"])
|
||||
assert result.exit_code == 0, result.output
|
||||
out = result.output
|
||||
assert "Artemis System & Environment Doctor" in out
|
||||
assert "Details & Recommendations" in out
|
||||
assert out.count("✔ OK") == 7
|
||||
assert "Node.js / npm" in out and "Showcase UI" in out
|
||||
assert "All system checks passed" in out
|
||||
assert 'artemis run "Open Settings and check Battery level"' in out
|
||||
|
||||
order = [
|
||||
"Title python_runtime",
|
||||
"Title system_config",
|
||||
"Title integration_host",
|
||||
"Title gemini_api_key",
|
||||
"Title android_adb",
|
||||
"Title toolchain",
|
||||
"Title vision_ocr_key",
|
||||
"Node.js / npm",
|
||||
"Showcase UI",
|
||||
]
|
||||
positions = [out.index(name) for name in order]
|
||||
assert positions == sorted(positions)
|
||||
|
||||
|
||||
def test_cli_doctor_blocker_failure_shows_run_lines_and_splits_chains(monkeypatch):
|
||||
"""A failing blocker prints each action on its own line; && chains are split."""
|
||||
probes = [p for p in _all_pass_probes() if p.id != "android_adb"]
|
||||
probes.append(
|
||||
_probe(
|
||||
"android_adb",
|
||||
status="fail",
|
||||
summary="ADB Missing",
|
||||
description="adb was not found.",
|
||||
actions=[
|
||||
("command", "Install", "winget install Google.PlatformTools && adb start-server"),
|
||||
("hint", "Enable USB", "Enable USB debugging on the phone."),
|
||||
("link", "Docs", "https://developer.android.com/tools/adb"),
|
||||
],
|
||||
metadata={"installed": False},
|
||||
)
|
||||
)
|
||||
_install_doctor_fakes(monkeypatch, probes)
|
||||
|
||||
result = runner.invoke(app, ["doctor"])
|
||||
assert result.exit_code == 1, result.output
|
||||
out = result.output
|
||||
assert "✖ ADB Missing" in out
|
||||
assert "Run: winget install Google.PlatformTools" in out
|
||||
assert "Run: adb start-server" in out
|
||||
assert "&&" not in out
|
||||
assert "Enable USB debugging on the phone." in out
|
||||
assert "https://developer.android.com/tools/adb" in out
|
||||
assert "Action Required" in out
|
||||
assert "artemis init" in out
|
||||
assert "mobile_diagnose" in out
|
||||
|
||||
|
||||
def test_cli_doctor_optional_failure_is_degraded_not_blocked(monkeypatch):
|
||||
"""Non-blocker FAIL renders as Optional, keeps exit 0, and yields 'degraded'."""
|
||||
import json
|
||||
|
||||
probes = [p for p in _all_pass_probes() if p.id != "toolchain"]
|
||||
probes.append(
|
||||
_probe(
|
||||
"toolchain",
|
||||
status="fail",
|
||||
blocker=False,
|
||||
summary="Missing FFmpeg",
|
||||
actions=[("command", "Install", "winget install Gyan.FFmpeg")],
|
||||
)
|
||||
)
|
||||
_install_doctor_fakes(monkeypatch, probes)
|
||||
|
||||
result = runner.invoke(app, ["doctor"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "⚪ Optional" in result.output
|
||||
assert "Run: winget install Gyan.FFmpeg" in result.output
|
||||
assert "optional gaps" in result.output
|
||||
|
||||
result = runner.invoke(app, ["doctor", "--json"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert json.loads(result.output)["verdict"] == "degraded"
|
||||
|
||||
|
||||
def test_cli_doctor_never_prints_raw_keys_from_metadata(monkeypatch):
|
||||
"""Probe metadata (which carries raw credentials) must not reach the terminal or JSON."""
|
||||
_install_doctor_fakes(monkeypatch, _all_pass_probes())
|
||||
|
||||
table = runner.invoke(app, ["doctor"])
|
||||
assert table.exit_code == 0, table.output
|
||||
assert "RAWSECRET" not in table.output
|
||||
assert "sk-RAW...cdef" in table.output # the masked description is still shown
|
||||
|
||||
as_json = runner.invoke(app, ["doctor", "--json"])
|
||||
assert as_json.exit_code == 0, as_json.output
|
||||
assert "RAWSECRET" not in as_json.output
|
||||
assert "metadata" not in as_json.output
|
||||
|
||||
|
||||
def test_cli_doctor_json_shape_and_ready_verdict(monkeypatch):
|
||||
"""--json emits the documented document shape and nothing else."""
|
||||
import json
|
||||
|
||||
_install_doctor_fakes(monkeypatch, _all_pass_probes())
|
||||
|
||||
result = runner.invoke(app, ["doctor", "--json"])
|
||||
assert result.exit_code == 0, result.output
|
||||
doc = json.loads(result.output)
|
||||
assert set(doc) == {"verdict", "checks", "extras", "fixes"}
|
||||
assert doc["verdict"] == "ready"
|
||||
assert doc["fixes"] == []
|
||||
assert [c["id"] for c in doc["checks"]] == [
|
||||
"python_runtime",
|
||||
"system_config",
|
||||
"integration_host",
|
||||
"gemini_api_key",
|
||||
"android_adb",
|
||||
"toolchain",
|
||||
"vision_ocr_key",
|
||||
]
|
||||
for check in doc["checks"]:
|
||||
assert set(check) == {"id", "title", "status", "required", "summary", "detail", "fix"}
|
||||
assert check["status"] == "pass"
|
||||
assert check["fix"] == []
|
||||
assert set(doc["extras"]) == {"nodejs_npm", "showcase_ui"}
|
||||
assert doc["extras"]["nodejs_npm"]["status"] == "pass"
|
||||
assert doc["extras"]["showcase_ui"]["status"] == "missing"
|
||||
|
||||
|
||||
def test_cli_doctor_json_blocked_verdict_and_exit_code(monkeypatch):
|
||||
"""A blocker that is not PASS makes the verdict 'blocked' and the exit code 1."""
|
||||
import json
|
||||
|
||||
probes = [p for p in _all_pass_probes() if p.id != "gemini_api_key"]
|
||||
probes.append(
|
||||
_probe(
|
||||
"gemini_api_key",
|
||||
status="fail",
|
||||
summary="Key Missing",
|
||||
description="No LLM credential found.",
|
||||
actions=[
|
||||
("command", "Run Artemis Init", "artemis init"),
|
||||
("link", "Get Key", "https://aistudio.google.com/app/apikey"),
|
||||
],
|
||||
)
|
||||
)
|
||||
_install_doctor_fakes(monkeypatch, probes)
|
||||
|
||||
result = runner.invoke(app, ["doctor", "--json"])
|
||||
assert result.exit_code == 1, result.output
|
||||
doc = json.loads(result.output)
|
||||
assert doc["verdict"] == "blocked"
|
||||
key_check = next(c for c in doc["checks"] if c["id"] == "gemini_api_key")
|
||||
assert key_check["status"] == "fail"
|
||||
assert key_check["required"] is True
|
||||
assert key_check["fix"] == [
|
||||
{"type": "command", "label": "Run Artemis Init", "payload": "artemis init"},
|
||||
{"type": "link", "label": "Get Key", "payload": "https://aistudio.google.com/app/apikey"},
|
||||
]
|
||||
|
||||
|
||||
def test_cli_doctor_host_warn_degrades_and_keeps_the_fix(monkeypatch):
|
||||
"""A host WARN is not a blocker (the runner works around it): the doctor
|
||||
exits 0 with a degraded verdict and still prints the fix."""
|
||||
host = _probe(
|
||||
"integration_host",
|
||||
status="warn",
|
||||
blocker=False,
|
||||
summary="Host Warning",
|
||||
description="MCP server runs on the wrong interpreter.",
|
||||
actions=[("command", "Regenerate", "uv run artemis mcp --install cursor")],
|
||||
)
|
||||
_install_doctor_fakes(monkeypatch, _all_pass_probes(), host=host)
|
||||
|
||||
result = runner.invoke(app, ["doctor"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "⚠ Host Warning" in result.output
|
||||
assert "Run: uv run artemis mcp --install cursor" in result.output
|
||||
|
||||
|
||||
def test_cli_doctor_host_fail_blocks_and_uses_summary(monkeypatch):
|
||||
host = _probe(
|
||||
"integration_host",
|
||||
status="fail",
|
||||
summary="Host Misconfigured",
|
||||
description="traces directory is not writable.",
|
||||
)
|
||||
_install_doctor_fakes(monkeypatch, _all_pass_probes(), host=host)
|
||||
|
||||
result = runner.invoke(app, ["doctor"])
|
||||
assert result.exit_code == 1, result.output
|
||||
assert "✖ Host Misconfigured" in result.output
|
||||
|
||||
|
||||
def test_cli_doctor_fix_heals_corrupted_keys_and_sweeps_locks(monkeypatch):
|
||||
"""--fix heals ADB keys when the probe reports corruption and sweeps stale locks."""
|
||||
from artemis.runtime.device_lock import DeviceExecutionLock
|
||||
|
||||
probes = [p for p in _all_pass_probes() if p.id != "android_adb"]
|
||||
probes.append(
|
||||
_probe(
|
||||
"android_adb",
|
||||
status="fail",
|
||||
summary="Keys Corrupted",
|
||||
metadata={"installed": True, "adb_keys": {"is_corrupted": True}},
|
||||
)
|
||||
)
|
||||
fakes = _install_doctor_fakes(monkeypatch, probes)
|
||||
cleanup_calls: list = []
|
||||
monkeypatch.setattr(
|
||||
DeviceExecutionLock,
|
||||
"cleanup_stale_locks",
|
||||
classmethod(lambda cls, device_id=None: cleanup_calls.append(device_id) or 3),
|
||||
)
|
||||
|
||||
result = runner.invoke(app, ["doctor", "--fix"])
|
||||
fakes["heal"].assert_awaited_once()
|
||||
assert cleanup_calls == [None]
|
||||
assert fakes["run_all"].await_count == 2 # report is re-collected after the repairs
|
||||
assert "Repairs (--fix)" in result.output
|
||||
assert "Keys regenerated." in result.output
|
||||
assert "Removed 3 stale device lock(s)" in result.output
|
||||
|
||||
|
||||
def test_cli_doctor_fix_skips_heal_when_keys_are_healthy(monkeypatch):
|
||||
"""--fix does not touch healthy ADB keys but still sweeps stale locks."""
|
||||
from artemis.runtime.device_lock import DeviceExecutionLock
|
||||
|
||||
fakes = _install_doctor_fakes(monkeypatch, _all_pass_probes())
|
||||
monkeypatch.setattr(
|
||||
DeviceExecutionLock, "cleanup_stale_locks", classmethod(lambda cls, device_id=None: 0)
|
||||
)
|
||||
|
||||
result = runner.invoke(app, ["doctor", "--fix"])
|
||||
assert result.exit_code == 0, result.output
|
||||
fakes["heal"].assert_not_awaited()
|
||||
assert "nothing to repair" in result.output
|
||||
assert "No stale device locks" in result.output
|
||||
@@ -14,6 +14,8 @@
|
||||
|
||||
"""Unit tests for the agent.memory configuration block (M1/M2)."""
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.config import AgentGlobalConfig
|
||||
|
||||
|
||||
@@ -24,12 +26,39 @@ def test_memory_defaults():
|
||||
assert cfg.memory.runtime.max_concurrency == 2
|
||||
assert cfg.memory.runtime.retry_limit == 3
|
||||
assert cfg.memory.runtime.flush_timeout_s == 30.0
|
||||
# M5 (2026-09-01): the transcript path ships ON by default; explicit
|
||||
# false is the byte-for-byte rollback switch (see the flip test below).
|
||||
# Transcript history is enabled by default; false selects the legacy path.
|
||||
assert cfg.memory.transcript.enabled is True
|
||||
assert cfg.memory.transcript.image_scrub_depth == 3
|
||||
assert cfg.memory.transcript.pending_grace_steps == 3
|
||||
# Text cleanup runs independently of the screenshot depth.
|
||||
assert cfg.memory.transcript.xml_scrub_depth == 1
|
||||
# Ready chunks replace their turns only once the
|
||||
# operator's measured context reaches budget*start_ratio.
|
||||
assert cfg.memory.chunking.min_steps == 3
|
||||
assert cfg.memory.transcript.image_scrub_depth_relaxed == 6
|
||||
assert cfg.memory.transcript.start_ratio == 0.35
|
||||
assert cfg.memory.transcript.soft_ratio == 0.7
|
||||
assert cfg.memory.transcript.hard_ratio == 0.9
|
||||
|
||||
|
||||
def test_transcript_ratio_ladder_is_validated():
|
||||
"""start <= soft <= hard, all within [0, 1]: the defaults pass, a
|
||||
crossed pair is rejected with a clear message."""
|
||||
import pydantic
|
||||
|
||||
from artemis.config.agent import MemoryTranscriptConfig
|
||||
|
||||
MemoryTranscriptConfig()
|
||||
MemoryTranscriptConfig(start_ratio=0.0, soft_ratio=0.5, hard_ratio=0.5)
|
||||
|
||||
with pytest.raises(pydantic.ValidationError, match="start_ratio <= soft_ratio"):
|
||||
MemoryTranscriptConfig(start_ratio=0.8, soft_ratio=0.7)
|
||||
with pytest.raises(pydantic.ValidationError, match="soft_ratio <= hard_ratio"):
|
||||
MemoryTranscriptConfig(soft_ratio=0.95, hard_ratio=0.9)
|
||||
with pytest.raises(pydantic.ValidationError):
|
||||
AgentGlobalConfig.model_validate(
|
||||
{"memory": {"transcript": {"start_ratio": 0.9, "soft_ratio": 0.7}}}
|
||||
)
|
||||
|
||||
|
||||
def test_memory_transcript_flag_explicit_disable_is_rollback_switch():
|
||||
@@ -57,9 +86,40 @@ def test_memory_explicit_values():
|
||||
assert cfg.memory.runtime.flush_timeout_s == 10.0
|
||||
assert cfg.memory.transcript.image_scrub_depth == 5
|
||||
assert cfg.memory.transcript.pending_grace_steps == 1
|
||||
# Legacy key: still accepted (back-compat), value retained as given.
|
||||
assert cfg.memory.transcript.xml_scrub_depth == 2
|
||||
|
||||
|
||||
def test_xml_scrub_depth_positions_the_text_edge():
|
||||
"""The text edge is separate from the screenshot edge: with K=3 and
|
||||
xml_scrub_depth=1 the observation at depth 2 has already lost its UI
|
||||
list while it still carries its screenshot."""
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from artemis.memory.transcript import PRO_UI_LIST_MARKER, TranscriptLedger
|
||||
|
||||
cfg = AgentGlobalConfig.model_validate({"memory": {"transcript": {"xml_scrub_depth": 1}}})
|
||||
ledger = TranscriptLedger(
|
||||
image_scrub_depth=3, xml_scrub_depth=cfg.memory.transcript.xml_scrub_depth
|
||||
)
|
||||
|
||||
def obs(i):
|
||||
return HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": f"obs {i}"},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{i}"}},
|
||||
{"type": "text", "text": f"{PRO_UI_LIST_MARKER}\n[{i}] item"},
|
||||
]
|
||||
)
|
||||
|
||||
ledger.stage_turn([obs(1)])
|
||||
ledger.commit_staged(step_key="s1")
|
||||
ledger.render([obs(2)])
|
||||
committed = ledger.active_messages[0]
|
||||
assert PRO_UI_LIST_MARKER not in str(committed.content)
|
||||
assert any(isinstance(b, dict) and b.get("type") == "image_url" for b in committed.content)
|
||||
|
||||
|
||||
def test_legacy_step_summarizer_retry_limit_seeds_memory_runtime():
|
||||
"""An explicit legacy flash.step_summarizer.retry_limit keeps working."""
|
||||
cfg = AgentGlobalConfig.model_validate({"flash": {"step_summarizer": {"retry_limit": 7}}})
|
||||
@@ -103,7 +163,7 @@ def test_recall_and_similarity_defaults():
|
||||
|
||||
|
||||
def test_chunking_era_cap_follows_max_chunks_by_default():
|
||||
"""M5: max_eras is an independent key; None (default) follows max_chunks."""
|
||||
"""max_eras is an independent key; None (default) follows max_chunks."""
|
||||
cfg = AgentGlobalConfig()
|
||||
assert cfg.memory.chunking.max_chunks == 8
|
||||
assert cfg.memory.chunking.max_eras is None
|
||||
@@ -133,3 +193,25 @@ def test_recall_and_policies_overridable():
|
||||
assert cfg.memory.replay.max_tokens == 4000
|
||||
assert cfg.memory.transcript.similarity_hint is False
|
||||
assert cfg.memory.policies == {"planner": {"last_n_detailed": 3}}
|
||||
|
||||
|
||||
def test_memory_chunking_min_steps_must_not_exceed_max_steps():
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
AgentGlobalConfig.model_validate({"memory": {"chunking": {"min_steps": 4, "max_steps": 4}}})
|
||||
with pytest.raises(ValidationError):
|
||||
AgentGlobalConfig.model_validate({"memory": {"chunking": {"min_steps": 5, "max_steps": 4}}})
|
||||
|
||||
|
||||
def test_memory_transcript_relaxed_image_depth_must_not_be_shallower():
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
AgentGlobalConfig.model_validate(
|
||||
{"memory": {"transcript": {"image_scrub_depth": 3, "image_scrub_depth_relaxed": 3}}}
|
||||
)
|
||||
with pytest.raises(ValidationError):
|
||||
AgentGlobalConfig.model_validate(
|
||||
{"memory": {"transcript": {"image_scrub_depth": 4, "image_scrub_depth_relaxed": 3}}}
|
||||
)
|
||||
@@ -82,6 +82,7 @@ def test_meter_accumulates_session_totals():
|
||||
"session_completion_tokens": 30,
|
||||
"session_cached_tokens": 250,
|
||||
"session_cache_hit_calls": 1,
|
||||
"session_cached_ratio": 0.625,
|
||||
}
|
||||
assert meter.last_prompt_tokens == 300
|
||||
|
||||
|
||||
@@ -24,10 +24,14 @@ from artemis.core.tool_failure import is_tool_failure
|
||||
from artemis.tools.history import ReplayStepsTool, replay_steps, replay_steps_text
|
||||
|
||||
|
||||
SESSION_START = 1000.0
|
||||
|
||||
|
||||
def _friendly_step(number, tool_result="The toggle is ON", **overrides):
|
||||
step = {
|
||||
"step_id": f"step_{number}",
|
||||
"step_number": number,
|
||||
"timestamp": SESSION_START + number * 3,
|
||||
"relative_time": f"{number * 3}.0s",
|
||||
"summary": f"Step {number} summary.",
|
||||
"action_taken": {"action": "click", "target": [0.5, 0.5], "target_text": "Save"},
|
||||
@@ -46,8 +50,9 @@ def _friendly_step(number, tool_result="The toggle is ON", **overrides):
|
||||
return step
|
||||
|
||||
|
||||
def _reader(steps):
|
||||
def _reader(steps, session_start=SESSION_START):
|
||||
reader = MagicMock()
|
||||
reader.session_start_time = session_start
|
||||
reader.get_agent_friendly_steps_in_range.side_effect = lambda s, e: [
|
||||
st for st in steps if s <= st["step_number"] <= e
|
||||
]
|
||||
@@ -59,19 +64,38 @@ def test_replay_range_renders_every_step_in_full():
|
||||
out = replay_steps_text(reader, 3, 4)
|
||||
|
||||
reader.get_agent_friendly_steps_in_range.assert_called_once_with(3, 4)
|
||||
assert "- **Step 3 (Start: 9.0s)**" in out
|
||||
assert "- **Step 4 (Start: 12.0s)**" in out
|
||||
# Step headers carry the session clock every agent prompt uses.
|
||||
assert "- **Step 3 (T+00:09)**" in out
|
||||
assert "- **Step 4 (T+00:12)**" in out
|
||||
assert "Start:" not in out
|
||||
assert "[Screen]: Step 3 summary." in out
|
||||
assert "Thinking at step 4." in out
|
||||
assert "`ask_explorer(" in out
|
||||
# Loose clamp: a ~1.8k-char tool result is replayed in full.
|
||||
assert "The toggle is OFF " * 100 in out
|
||||
assert "[Planned Action]: Tapped 'Save' at [0.5, 0.5]" in out
|
||||
assert "[Action]: Tapped 'Save' at [0.5, 0.5]" in out
|
||||
assert "[Reasoning & tool calls]:" in out
|
||||
assert "Operator" not in out
|
||||
|
||||
|
||||
def test_replay_header_falls_back_without_a_session_clock():
|
||||
"""No session start (or a mocked, non-numeric one) / no step timestamp:
|
||||
the header keeps the stored relative time instead of ``T+??:??``."""
|
||||
out = replay_steps_text(_reader([_friendly_step(2)], session_start=None), 2)
|
||||
assert "- **Step 2 (Start: 6.0s)**" in out
|
||||
|
||||
reader = MagicMock() # session_start_time is a MagicMock, not a number
|
||||
reader.get_agent_friendly_steps_in_range.return_value = [_friendly_step(2)]
|
||||
assert "- **Step 2 (Start: 6.0s)**" in replay_steps_text(reader, 2)
|
||||
|
||||
no_timestamp = _friendly_step(2)
|
||||
del no_timestamp["timestamp"]
|
||||
assert "- **Step 2 (Start: 6.0s)**" in replay_steps_text(_reader([no_timestamp]), 2)
|
||||
|
||||
|
||||
def test_replay_single_step_reversed_bounds_and_missing_step():
|
||||
reader = _reader([_friendly_step(2)])
|
||||
assert "- **Step 2 (Start: 6.0s)**" in replay_steps_text(reader, 2)
|
||||
assert "- **Step 2 (T+00:06)**" in replay_steps_text(reader, 2)
|
||||
reader.get_agent_friendly_steps_in_range.assert_called_once_with(2, 2)
|
||||
|
||||
reader = _reader([_friendly_step(2), _friendly_step(3)])
|
||||
@@ -149,7 +173,7 @@ def test_replay_without_history():
|
||||
async def test_tool_execute_reads_the_context_engine_and_degrades_without_one():
|
||||
reader = _reader([_friendly_step(2)])
|
||||
out = await ReplayStepsTool().execute(ctx=SimpleNamespace(data_engine=reader), start_step=2)
|
||||
assert "- **Step 2 (Start: 6.0s)**" in out
|
||||
assert "- **Step 2 (T+00:06)**" in out
|
||||
|
||||
out = await replay_steps.execute(ctx=SimpleNamespace(data_engine=None), start_step=1)
|
||||
assert "no execution history" in out
|
||||
|
||||
@@ -21,7 +21,9 @@ from unittest.mock import MagicMock
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from artemis.core.tool_failure import is_tool_failure
|
||||
from artemis.tools.history import get_step_screenshot, load_step_screenshot
|
||||
from artemis.tools.tool_wrapper import tool_result_messages
|
||||
|
||||
|
||||
def _jpeg_file(tmp_path, name="pre.jpg", color="white"):
|
||||
@@ -78,21 +80,70 @@ def test_overlay_falls_back_to_plain_pre_without_action(tmp_path):
|
||||
assert "plain screenshot" in shot.description
|
||||
|
||||
|
||||
def test_missing_file_and_missing_step_return_plain_text():
|
||||
def test_missing_file_is_an_answer_but_missing_step_is_a_failure():
|
||||
shot = load_step_screenshot(_reader(pre=None, post=None), 3, "post")
|
||||
assert shot.image_bytes is None
|
||||
assert shot.to_content_blocks() == "No post-action screenshot recorded for step 3."
|
||||
assert shot.is_error is False
|
||||
out = shot.to_content_blocks()
|
||||
assert out == "No post-action screenshot recorded for step 3."
|
||||
assert not is_tool_failure(out)
|
||||
|
||||
shot = load_step_screenshot(_reader(pre=None, record=False), 9, "pre")
|
||||
assert shot.to_content_blocks() == "Error: step 9 not found."
|
||||
assert shot.is_error is True
|
||||
out = shot.to_content_blocks()
|
||||
assert out == "Error: step 9 not found."
|
||||
assert is_tool_failure(out)
|
||||
|
||||
|
||||
def test_invalid_variant_and_step_number_are_rejected():
|
||||
shot = load_step_screenshot(_reader(), 1, "bogus")
|
||||
assert shot.image_bytes is None
|
||||
assert shot.is_error is True
|
||||
assert "'pre', 'post' or 'overlay'" in shot.description
|
||||
assert "must be an integer" in load_step_screenshot(_reader(), "one").description
|
||||
assert "no execution history" in load_step_screenshot(None, 1).description
|
||||
assert is_tool_failure(shot.to_content_blocks())
|
||||
|
||||
shot = load_step_screenshot(_reader(), "one")
|
||||
assert "must be an integer" in shot.description and shot.is_error is True
|
||||
shot = load_step_screenshot(None, 1)
|
||||
assert "no execution history" in shot.description and shot.is_error is True
|
||||
|
||||
|
||||
def test_unreadable_file_and_reader_errors_are_failures(tmp_path):
|
||||
shot = load_step_screenshot(_reader(pre=tmp_path / "missing.jpg"), 2, "pre")
|
||||
assert shot.is_error is True
|
||||
assert "Error reading screenshot" in shot.description
|
||||
|
||||
reader = _reader()
|
||||
reader.get_step_image_path.side_effect = ValueError("corrupt row")
|
||||
shot = load_step_screenshot(reader, 2, "pre")
|
||||
assert shot.is_error is True
|
||||
assert "Error looking up step 2: corrupt row" in shot.description
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_error_answers_reach_the_model_with_status_error():
|
||||
"""The executor loops derive the ToolMessage status from ``is_tool_failure``:
|
||||
an unknown step arrives as status=error, a recorded-but-absent screenshot
|
||||
as a normal answer."""
|
||||
ctx = SimpleNamespace(data_engine=_reader(pre=None, record=False))
|
||||
out = await get_step_screenshot.execute(ctx=ctx, step_number=999, which="pre")
|
||||
assert out == "Error: step 999 not found."
|
||||
assert is_tool_failure(out)
|
||||
status = "error" if is_tool_failure(out) else "success"
|
||||
(msg,) = tool_result_messages("tc-1", out, name="get_step_screenshot", status=status)
|
||||
assert msg.status == "error"
|
||||
assert msg.content == "Error: step 999 not found."
|
||||
|
||||
ctx = SimpleNamespace(data_engine=_reader(pre=None, post=None))
|
||||
out = await get_step_screenshot.execute(ctx=ctx, step_number=3, which="post")
|
||||
assert not is_tool_failure(out)
|
||||
(msg,) = tool_result_messages(
|
||||
"tc-2",
|
||||
out,
|
||||
name="get_step_screenshot",
|
||||
status="error" if is_tool_failure(out) else "success",
|
||||
)
|
||||
assert msg.status == "success"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -346,6 +346,101 @@ def test_availability_requires_engine_and_config():
|
||||
assert search_history_available(SimpleNamespace(data_engine=object())) is True
|
||||
|
||||
|
||||
def test_excerpts_never_show_raw_json(engine):
|
||||
"""The raw action / result JSON only scores; the ``Match:`` excerpt is the
|
||||
ledger action phrase plus the result's status/error words."""
|
||||
engine.record_step(
|
||||
summary="Started playback.",
|
||||
action_taken={
|
||||
"action": "click",
|
||||
"coordinates": [500, 520],
|
||||
"coordinate_space": "normalized",
|
||||
"target_description": "Wi-Fi jsonneedle row",
|
||||
"args": {"target": [500, 520], "target_description": "Wi-Fi jsonneedle row"},
|
||||
},
|
||||
last_execution_result={"status": "failed", "error": "Error: tap rejected"},
|
||||
)
|
||||
_flush(engine)
|
||||
|
||||
out = search_history_text(engine, query="jsonneedle", recall_config=_cfg())
|
||||
match = next(line for line in out.splitlines() if line.strip().startswith("Match:"))
|
||||
assert "Tapped 'Wi-Fi jsonneedle row' (self-described) at [500, 520]" in match
|
||||
assert "status failed" in match
|
||||
assert "Error: tap rejected" in match
|
||||
for token in ('{"action"', '"coordinates"', '"coordinate_space"', '"args"', "{", "}"):
|
||||
assert token not in match, match
|
||||
|
||||
|
||||
def test_raw_only_fields_still_score(engine):
|
||||
"""A hit that only exists in the raw record (a resource id / package name
|
||||
under ``args``) is still found; its excerpt is the readable action line."""
|
||||
engine.record_step(
|
||||
summary="Opened the drawer.",
|
||||
action_taken={
|
||||
"action": "click",
|
||||
"coordinates": [10, 20],
|
||||
"coordinate_space": "normalized",
|
||||
"args": {"target": [10, 20], "resource_id": "com.example:id/rawonlyneedle"},
|
||||
},
|
||||
last_execution_result={"status": "dispatched"},
|
||||
)
|
||||
_flush(engine)
|
||||
out = search_history_text(engine, query="rawonlyneedle", recall_config=_cfg())
|
||||
assert "[Step 4 " in out
|
||||
assert "Match: Tapped element at [10, 20] | status dispatched" in out
|
||||
assert '"resource_id"' not in out
|
||||
|
||||
|
||||
def test_summary_hit_is_not_repeated_in_the_match_line(engine):
|
||||
out = search_history_text(engine, query="promo popup", recall_config=_cfg())
|
||||
block = out[out.index("[Step 3 ") :]
|
||||
assert "Screen: A promo popup appeared and was dismissed." in block
|
||||
# The summary heads the result once; the Match line (if any) shows the
|
||||
# other readable fields, never the summary again.
|
||||
assert block.count("A promo popup appeared") == 1
|
||||
|
||||
|
||||
def test_screen_hits_render_readable_values_not_ui_tree_json(tmp_path):
|
||||
engine = _make_engine(tmp_path)
|
||||
engine.record_step(
|
||||
pre_screenshot_bytes=_jpeg("red"),
|
||||
summary="On the network settings screen.",
|
||||
action_taken={"action": "click", "target_text": "Network"},
|
||||
ui_tree=[
|
||||
{
|
||||
"text": "",
|
||||
"content-desc": "Navigate up",
|
||||
"resource-id": "com.android.settings:id/toolbar_up",
|
||||
"bounds": "[0,0][100,100]",
|
||||
"class": "android.widget.ImageButton",
|
||||
"clickable": "true",
|
||||
"package": "com.android.settings",
|
||||
},
|
||||
{
|
||||
"text": "Wi-Fi treeneedle",
|
||||
"content-desc": "",
|
||||
"resource-id": "android:id/title",
|
||||
"bounds": "[40,560][600,640]",
|
||||
"class": "android.widget.TextView",
|
||||
"clickable": "false",
|
||||
"children": [{"text": "Connected", "bounds": "[40,640][600,700]"}],
|
||||
},
|
||||
],
|
||||
ocr_result=[{"text": "Bluetooth ocrword", "bounds": [0, 0, 10, 10]}],
|
||||
)
|
||||
_flush(engine)
|
||||
|
||||
out = search_history_text(engine, query="treeneedle", recall_config=_cfg())
|
||||
match = next(line for line in out.splitlines() if line.strip().startswith("Match:"))
|
||||
assert "Wi-Fi treeneedle" in match
|
||||
assert "Navigate up | com.android.settings:id/toolbar_up | Wi-Fi treeneedle" in match
|
||||
assert "android:id/title | Connected | Bluetooth ocrword" in match
|
||||
for token in ('"bounds"', '"class"', '"clickable"', "{", "}"):
|
||||
assert token not in match, match
|
||||
# Bounds / class names still score (they stay in the raw haystack).
|
||||
assert "[Step 1 " in search_history_text(engine, query="ImageButton", recall_config=_cfg())
|
||||
|
||||
|
||||
def test_action_matches_keep_target_provenance(engine):
|
||||
"""The action haystack is the shared ledger rendering, so an excerpt shows a
|
||||
self-described coordinate target with its marker and an observed index
|
||||
|
||||
@@ -62,7 +62,7 @@ async def test_append_note_pure_direct_execution_success(mock_ctx, tmp_path):
|
||||
key="pure_append",
|
||||
content="First line",
|
||||
)
|
||||
assert result1 == "Successfully appended note to pure_append.md."
|
||||
assert result1 == "Appended to note 'pure_append'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "pure_append.md"
|
||||
assert note_path.exists()
|
||||
@@ -74,7 +74,7 @@ async def test_append_note_pure_direct_execution_success(mock_ctx, tmp_path):
|
||||
key="pure_append",
|
||||
content="Second line",
|
||||
)
|
||||
assert result2 == "Successfully appended note to pure_append.md."
|
||||
assert result2 == "Appended to note 'pure_append'."
|
||||
assert note_path.read_text(encoding="utf-8") == "First line\nSecond line\n"
|
||||
|
||||
|
||||
@@ -86,7 +86,7 @@ async def test_append_note_pure_callable_execution(mock_ctx, tmp_path):
|
||||
key="callable_pure_append",
|
||||
content="Pure callable append",
|
||||
)
|
||||
assert result == "Successfully appended note to callable_pure_append.md."
|
||||
assert result == "Appended to note 'callable_pure_append'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "callable_pure_append.md"
|
||||
assert note_path.exists()
|
||||
@@ -120,7 +120,7 @@ async def test_get_append_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"content": "Saved via pure append tool",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully appended note to lc_pure_append.md."
|
||||
assert result == "Appended to note 'lc_pure_append'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "lc_pure_append.md"
|
||||
assert note_path.exists()
|
||||
|
||||
@@ -70,7 +70,7 @@ async def test_append_note_direct_execution_success(mock_ctx, tmp_path):
|
||||
key="test_append",
|
||||
content="First line",
|
||||
)
|
||||
assert result1 == "Successfully appended note to test_append.md."
|
||||
assert result1 == "Appended to note 'test_append'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "test_append.md"
|
||||
assert note_path.exists()
|
||||
@@ -82,7 +82,7 @@ async def test_append_note_direct_execution_success(mock_ctx, tmp_path):
|
||||
key="test_append",
|
||||
content="Second line",
|
||||
)
|
||||
assert result2 == "Successfully appended note to test_append.md."
|
||||
assert result2 == "Appended to note 'test_append'."
|
||||
assert note_path.read_text(encoding="utf-8") == "First line\nSecond line\n"
|
||||
|
||||
|
||||
@@ -94,7 +94,7 @@ async def test_append_note_callable_execution(mock_ctx, tmp_path):
|
||||
key="callable_append",
|
||||
content="Callable append test",
|
||||
)
|
||||
assert result == "Successfully appended note to callable_append.md."
|
||||
assert result == "Appended to note 'callable_append'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "callable_append.md"
|
||||
assert note_path.exists()
|
||||
@@ -116,7 +116,7 @@ async def test_append_note_with_state_tool_message(mock_ctx, tmp_path):
|
||||
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert result.tool_call_id == "call_append_999"
|
||||
assert result.content == "Successfully appended note to state_append_note.md."
|
||||
assert result.content == "Appended to note 'state_append_note'."
|
||||
assert result.status == "success"
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "state_append_note.md"
|
||||
@@ -151,7 +151,7 @@ async def test_get_append_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"content": "Appended via LangChain BaseTool",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully appended note to lc_append_note.md."
|
||||
assert result == "Appended to note 'lc_append_note'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "lc_append_note.md"
|
||||
assert note_path.exists()
|
||||
@@ -170,7 +170,7 @@ async def test_get_append_note_tool_pure(mock_ctx, tmp_path):
|
||||
"content": "Appended via pure tool",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully appended note to pure_append_note.md."
|
||||
assert result == "Appended to note 'pure_append_note'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "pure_append_note.md"
|
||||
assert note_path.exists()
|
||||
|
||||
@@ -255,9 +255,25 @@ async def test_run_command_kill(mock_exec, mock_ctx):
|
||||
"TaskId": task_id,
|
||||
}
|
||||
)
|
||||
assert f"Task {task_id} successfully terminated" in kill_res
|
||||
assert f"Terminated task {task_id}" in kill_res
|
||||
assert "successfully" not in kill_res
|
||||
assert task_id not in registry.background
|
||||
|
||||
# Refusals (unknown or already-finished task) are reported structurally.
|
||||
from artemis.core.tool_failure import is_tool_failure
|
||||
|
||||
again = await manage_task.ainvoke({"Action": "kill", "TaskId": task_id})
|
||||
assert is_tool_failure(again)
|
||||
assert f"Task {task_id} is not active or already finished." in str(again)
|
||||
|
||||
missing = await manage_task.ainvoke({"Action": "status", "TaskId": "task_nope"})
|
||||
assert is_tool_failure(missing)
|
||||
assert "Error: Task task_nope not found." in str(missing)
|
||||
|
||||
no_input = await manage_task.ainvoke({"Action": "send_input", "TaskId": task_id, "Input": "y"})
|
||||
assert is_tool_failure(no_input)
|
||||
assert f"Task {task_id} is not active." in str(no_input)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("asyncio.create_subprocess_exec")
|
||||
@@ -305,8 +321,8 @@ async def test_run_short_command_timeout(mock_exec, mock_ctx):
|
||||
@pytest.mark.asyncio
|
||||
@patch("asyncio.create_subprocess_exec")
|
||||
async def test_run_command_sync_long_output(mock_exec, mock_ctx):
|
||||
# Generate long output (> 500 lines)
|
||||
long_output = "\n".join(f"Output line {i}" for i in range(1, 600))
|
||||
# Generate long output (> 50000 chars): the trigger is size, not line count.
|
||||
long_output = "\n".join(f"Output line {i} {'x' * 90}" for i in range(1, 600))
|
||||
mock_process = MockProcess(
|
||||
output_bytes=long_output.encode() + b"\n===EXIT_CODE===0\n===ENV_START===\n",
|
||||
exit_code=0,
|
||||
@@ -326,9 +342,36 @@ async def test_run_command_sync_long_output(mock_exec, mock_ctx):
|
||||
assert "ADB command completed with exit code 0." in result
|
||||
assert "has been truncated" in result
|
||||
assert "Output line 599" in result
|
||||
assert "Output line 1" not in result # since it was truncated to last 200 lines
|
||||
assert "analyze_task_output" in result
|
||||
assert "Output line 1 " not in result # since it was truncated to last 200 lines
|
||||
assert "TaskId: task_sync_" in result
|
||||
task_id = result.split("TaskId: ")[1].split(".")[0]
|
||||
assert f"use analyze_task_output with TaskId {task_id}" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("asyncio.create_subprocess_exec")
|
||||
async def test_run_command_many_short_lines_is_shown_inline(mock_exec, mock_ctx):
|
||||
"""1000 short lines (~4 KB) are not 'long': the whole output stays inline."""
|
||||
output = "\n".join(str(i) for i in range(1, 1001))
|
||||
mock_exec.return_value = MockProcess(output_bytes=output.encode(), exit_code=0)
|
||||
|
||||
run_command = get_run_adb_command_tool(mock_ctx)
|
||||
result = await run_command.ainvoke({"CommandLine": "seq 1 1000", "WaitMsBeforeAsync": 500})
|
||||
|
||||
assert "has been truncated" not in result
|
||||
assert "\n1\n2\n" in result
|
||||
assert result.endswith("\n1000")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("asyncio.create_subprocess_exec")
|
||||
async def test_run_command_empty_output_is_labelled(mock_exec, mock_ctx):
|
||||
mock_exec.return_value = MockProcess(output_bytes=b"", exit_code=0)
|
||||
|
||||
run_command = get_run_adb_command_tool(mock_ctx)
|
||||
result = await run_command.ainvoke({"CommandLine": "true", "WaitMsBeforeAsync": 500})
|
||||
|
||||
assert result == "ADB command completed with exit code 0.\nOutput:\n(empty output)"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -704,7 +747,7 @@ def test_registry_is_per_context_and_notifications_do_not_leak():
|
||||
@pytest.mark.asyncio
|
||||
@patch("asyncio.create_subprocess_exec")
|
||||
async def test_sync_long_output_is_not_reannounced(mock_exec, mock_ctx):
|
||||
long_output = "\n".join(f"line {i}" for i in range(600))
|
||||
long_output = "\n".join(f"line {i} {'x' * 90}" for i in range(600))
|
||||
mock_exec.return_value = MockProcess(output_bytes=long_output.encode(), exit_code=0)
|
||||
run_command = get_run_adb_command_tool(mock_ctx)
|
||||
registry = get_adb_task_registry(mock_ctx)
|
||||
|
||||
@@ -288,10 +288,32 @@ def test_render_text_found_lists_indices_and_coordinates():
|
||||
text = render_text("Find buttons", outcome, registered)
|
||||
assert "Explorer located 1 candidate(s) for 'Find buttons'" in text
|
||||
assert "- [2] 'First Button' at normalized [500, 500]" in text
|
||||
assert "target=[index]" in text
|
||||
assert "Explorer notes: Please choose carefully." in text
|
||||
|
||||
|
||||
def test_render_text_targeting_guidance_follows_the_callers_click_dialect():
|
||||
"""Flash / Validator / MCP ``click`` takes coordinates only; the Pro Operator
|
||||
takes a bare integer index. Neither dialect accepts ``target=[index]``."""
|
||||
outcome = ExplorerOutcome(candidates=[ExplorerCandidate("S1", (500, 500), "First Button")])
|
||||
registered = [RegisteredCandidate(2, (540, 1200), (500, 500), "First Button")]
|
||||
|
||||
coordinate_text = render_text("Find buttons", outcome, registered, index_targets=False)
|
||||
assert (
|
||||
"by its normalized coordinate (target=[x, y]) with a target_description" in coordinate_text
|
||||
)
|
||||
assert "bare integer" not in coordinate_text
|
||||
# The keyword defaults to the coordinate-only dialect (text entry points).
|
||||
assert render_text("Find buttons", outcome, registered) == coordinate_text
|
||||
|
||||
index_text = render_text("Find buttons", outcome, registered, index_targets=True)
|
||||
assert "by its index (target=3, a bare integer) or by its normalized coordinate" in index_text
|
||||
assert "target=[x, y]" not in index_text
|
||||
|
||||
for text in (coordinate_text, index_text):
|
||||
assert "target=[index]" not in text
|
||||
assert "Candidates are ranked by confidence." in text
|
||||
|
||||
|
||||
def test_render_text_not_found_explains_and_suggests_rephrasing():
|
||||
text = render_text("hidden button", ExplorerOutcome(message="Covered by keyboard."), [])
|
||||
assert text.startswith("Explorer could not locate 'hidden button'. Covered by keyboard.")
|
||||
@@ -329,6 +351,9 @@ def test_render_operator_blocks_adds_the_annotated_image(tmp_path):
|
||||
assert isinstance(result, list) and len(result) == 2
|
||||
assert result[0]["type"] == "text"
|
||||
assert "Explorer located 1 candidate(s)" in result[0]["text"]
|
||||
# The Operator's click takes a bare integer index.
|
||||
assert "target=3, a bare integer" in result[0]["text"]
|
||||
assert "target=[x, y]" not in result[0]["text"]
|
||||
assert result[1]["type"] == "image_url"
|
||||
assert result[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
||||
|
||||
@@ -485,6 +510,9 @@ async def test_ask_explorer_text_end_to_end():
|
||||
assert "Explorer located 2 candidate(s) for 'Find buttons'" in text
|
||||
assert "- [2] 'First Button' at normalized [500, 500]" in text
|
||||
assert "- [3] 'Second Button' at normalized [250, 750]" in text
|
||||
# The text entry serves Flash / Validator / MCP, whose click takes coordinates only.
|
||||
assert "target=[x, y]" in text
|
||||
assert "bare integer" not in text
|
||||
assert "Explorer notes: Please choose carefully." in text
|
||||
|
||||
|
||||
@@ -604,10 +632,16 @@ def test_register_candidates_reuses_an_indexed_element_that_contains_the_point()
|
||||
description="Gmail app icon",
|
||||
bounds=(200, 300, 400, 500),
|
||||
reused=True,
|
||||
existing_text="Gmail",
|
||||
)
|
||||
]
|
||||
assert len(state.indexed_points) == 2 # nothing appended
|
||||
assert "already in your indexed list" in render_text("Gmail app icon", outcome, registered)
|
||||
# The reused line names the element the way the agent's own list shows it.
|
||||
text = render_text("Gmail app icon", outcome, registered)
|
||||
assert (
|
||||
"- [1] 'Gmail app icon' at normalized [310, 210] (matches your indexed element [1] 'Gmail')"
|
||||
) in text
|
||||
assert "already in your indexed list" not in text
|
||||
|
||||
|
||||
def test_register_candidates_reuses_a_bare_center_within_the_dedup_radius_only():
|
||||
@@ -622,7 +656,9 @@ def test_register_candidates_reuses_a_bare_center_within_the_dedup_radius_only()
|
||||
registered = register_candidates(ctx, state, ExplorerOutcome(candidates=[near, far]))
|
||||
assert [r.reused for r in registered] == [True, False]
|
||||
assert registered[0].index == 1
|
||||
assert registered[0].existing_text == "Earlier candidate"
|
||||
assert registered[1].index == 2
|
||||
assert registered[1].existing_text == ""
|
||||
assert state.indexed_points == [[500, 1000], [560, 1000]]
|
||||
|
||||
|
||||
|
||||
@@ -60,7 +60,7 @@ async def test_read_note_pure_direct_execution_success(mock_ctx, tmp_path):
|
||||
save_note_content(tmp_path, "sample_pure", "line 1\nline 2\nline 3")
|
||||
|
||||
result = await read_note_pure.execute(ctx=mock_ctx, key="sample_pure")
|
||||
assert "Successfully read note 'sample_pure'." in result
|
||||
assert "Note 'sample_pure' (" in result
|
||||
assert "line 1\nline 2\nline 3" in result
|
||||
|
||||
|
||||
@@ -75,7 +75,7 @@ async def test_read_note_pure_slice(mock_ctx, tmp_path):
|
||||
start_line=2,
|
||||
end_line=3,
|
||||
)
|
||||
assert "Successfully read note 'slice_pure' (lines 2 to 3)." in result
|
||||
assert "Note 'slice_pure' (lines 2 to 3):" in result
|
||||
assert "line 1" not in result
|
||||
assert "line 2\nline 3" in result
|
||||
assert "line 4" not in result
|
||||
@@ -87,7 +87,7 @@ async def test_read_note_pure_callable_execution(mock_ctx, tmp_path):
|
||||
save_note_content(tmp_path, "callable_pure", "hello from pure callable")
|
||||
|
||||
result = await read_note_pure(ctx=mock_ctx, key="callable_pure")
|
||||
assert "Successfully read note 'callable_pure'." in result
|
||||
assert "Note 'callable_pure' (" in result
|
||||
assert "hello from pure callable" in result
|
||||
|
||||
|
||||
@@ -118,5 +118,5 @@ async def test_get_read_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
assert pure_tool.name == "read_note"
|
||||
|
||||
result = await pure_tool.ainvoke({"key": "pure_export"})
|
||||
assert "Successfully read note 'pure_export'." in result
|
||||
assert "Note 'pure_export' (" in result
|
||||
assert "exported pure note" in result
|
||||
@@ -68,7 +68,7 @@ async def test_read_note_direct_execution_success(mock_ctx, tmp_path):
|
||||
save_note_content(tmp_path, "sample", "line 1\nline 2\nline 3\nline 4\nline 5")
|
||||
|
||||
result = await read_note.execute(ctx=mock_ctx, key="sample")
|
||||
assert "Successfully read note 'sample'." in result
|
||||
assert "Note 'sample' (" in result
|
||||
assert "line 1\nline 2\nline 3\nline 4\nline 5" in result
|
||||
|
||||
|
||||
@@ -83,7 +83,7 @@ async def test_read_note_slice(mock_ctx, tmp_path):
|
||||
start_line=2,
|
||||
end_line=4,
|
||||
)
|
||||
assert "Successfully read note 'sample' (lines 2 to 4)." in result
|
||||
assert "Note 'sample' (lines 2 to 4):" in result
|
||||
assert "line 1" not in result
|
||||
assert "line 2\nline 3\nline 4" in result
|
||||
assert "line 5" not in result
|
||||
@@ -95,7 +95,7 @@ async def test_read_note_callable_execution(mock_ctx, tmp_path):
|
||||
save_note_content(tmp_path, "callable_sample", "hello from callable")
|
||||
|
||||
result = await read_note(ctx=mock_ctx, key="callable_sample")
|
||||
assert "Successfully read note 'callable_sample'." in result
|
||||
assert "Note 'callable_sample' (" in result
|
||||
assert "hello from callable" in result
|
||||
|
||||
|
||||
@@ -115,7 +115,7 @@ async def test_read_note_with_state_tool_message(mock_ctx, tmp_path):
|
||||
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert result.tool_call_id == "call_read_999"
|
||||
assert "Successfully read note 'state_sample'." in result.content
|
||||
assert "Note 'state_sample' (" in result.content
|
||||
assert "state note content" in result.content
|
||||
assert result.status == "success"
|
||||
|
||||
@@ -147,7 +147,7 @@ async def test_get_read_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
assert lc_tool.name == "read_note"
|
||||
|
||||
result = await lc_tool.ainvoke({"key": "lc_sample"})
|
||||
assert "Successfully read note 'lc_sample'." in result
|
||||
assert "Note 'lc_sample' (" in result
|
||||
assert "langchain content" in result
|
||||
|
||||
|
||||
@@ -160,5 +160,5 @@ async def test_get_read_note_tool_pure(mock_ctx, tmp_path):
|
||||
assert pure_tool.name == "read_note"
|
||||
|
||||
result = await pure_tool.ainvoke({"key": "pure_sample"})
|
||||
assert "Successfully read note 'pure_sample'." in result
|
||||
assert "Note 'pure_sample' (" in result
|
||||
assert "pure content" in result
|
||||
@@ -61,7 +61,7 @@ async def test_save_note_pure_direct_execution_success(mock_ctx, tmp_path):
|
||||
key="test_pure_save",
|
||||
content="Pure saved content",
|
||||
)
|
||||
assert result == "Successfully saved note to test_pure_save.md."
|
||||
assert result == "Saved note 'test_pure_save'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "test_pure_save.md"
|
||||
assert note_path.exists()
|
||||
@@ -76,7 +76,7 @@ async def test_save_note_pure_callable_execution(mock_ctx, tmp_path):
|
||||
key="callable_pure_save",
|
||||
content="Callable pure content",
|
||||
)
|
||||
assert result == "Successfully saved note to callable_pure_save.md."
|
||||
assert result == "Saved note 'callable_pure_save'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "callable_pure_save.md"
|
||||
assert note_path.exists()
|
||||
@@ -110,7 +110,7 @@ async def test_get_save_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"content": "Saved via LangChain pure tool",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully saved note to lc_pure_save.md."
|
||||
assert result == "Saved note 'lc_pure_save'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "lc_pure_save.md"
|
||||
assert note_path.exists()
|
||||
|
||||
@@ -69,7 +69,7 @@ async def test_save_note_direct_execution_success(mock_ctx, tmp_path):
|
||||
key="test_note",
|
||||
content="Hello Artemis memory",
|
||||
)
|
||||
assert result == "Successfully saved note to test_note.md."
|
||||
assert result == "Saved note 'test_note'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "test_note.md"
|
||||
assert note_path.exists()
|
||||
@@ -84,7 +84,7 @@ async def test_save_note_callable_execution(mock_ctx, tmp_path):
|
||||
key="callable_note",
|
||||
content="Callable invocation test",
|
||||
)
|
||||
assert result == "Successfully saved note to callable_note.md."
|
||||
assert result == "Saved note 'callable_note'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "callable_note.md"
|
||||
assert note_path.exists()
|
||||
@@ -106,7 +106,7 @@ async def test_save_note_with_state_tool_message(mock_ctx, tmp_path):
|
||||
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert result.tool_call_id == "call_save_999"
|
||||
assert result.content == "Successfully saved note to state_note.md."
|
||||
assert result.content == "Saved note 'state_note'."
|
||||
assert result.status == "success"
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "state_note.md"
|
||||
@@ -141,7 +141,7 @@ async def test_get_save_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"content": "Saved via LangChain BaseTool",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully saved note to lc_note.md."
|
||||
assert result == "Saved note 'lc_note'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "lc_note.md"
|
||||
assert note_path.exists()
|
||||
@@ -160,7 +160,7 @@ async def test_get_save_note_tool_pure(mock_ctx, tmp_path):
|
||||
"content": "Saved via pure tool",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully saved note to pure_note.md."
|
||||
assert result == "Saved note 'pure_note'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "pure_note.md"
|
||||
assert note_path.exists()
|
||||
|
||||
@@ -133,6 +133,32 @@ def test_search_and_merge_logs_regex():
|
||||
assert result == expected
|
||||
|
||||
|
||||
def test_search_and_merge_logs_header_compresses_consecutive_matches():
|
||||
# 100 consecutive matches at lines 699-798 collapse into one range.
|
||||
logs = "\n".join("ERROR: failure" if 699 <= i <= 798 else f"line {i}" for i in range(900))
|
||||
result = search_and_merge_logs(logs, "ERROR", context_lines=0)
|
||||
|
||||
assert result.splitlines()[0] == "--- Match at line(s) 699-798 ---"
|
||||
assert "699, 700" not in result
|
||||
|
||||
|
||||
def test_search_and_merge_logs_header_mixes_singles_and_ranges():
|
||||
# Matches at 6 and 69-79 sit in one block once context windows overlap.
|
||||
hits = {6} | set(range(69, 80))
|
||||
logs = "\n".join("ERROR: failure" if i in hits else f"line {i}" for i in range(100))
|
||||
result = search_and_merge_logs(logs, "ERROR", context_lines=40)
|
||||
|
||||
assert result.splitlines()[0] == "--- Match at line(s) 6, 69-79 ---"
|
||||
|
||||
|
||||
def test_search_and_merge_logs_header_caps_range_count():
|
||||
# Every other line matches: 50 single-line ranges, capped at 8 plus "...".
|
||||
logs = "\n".join("ERROR: failure" if i % 2 == 0 else f"line {i}" for i in range(100))
|
||||
result = search_and_merge_logs(logs, "ERROR", context_lines=1)
|
||||
|
||||
assert result.splitlines()[0] == "--- Match at line(s) 0, 2, 4, 6, 8, 10, 12, 14, ... ---"
|
||||
|
||||
|
||||
def test_search_and_merge_logs_invalid_regex():
|
||||
logs = "line 0"
|
||||
result = search_and_merge_logs(logs, "[invalid", is_regex=True)
|
||||
|
||||
@@ -66,7 +66,7 @@ async def test_update_note_pure_direct_execution_success(mock_ctx, tmp_path):
|
||||
target="Step 1: Pending",
|
||||
replacement="Step 1: Completed",
|
||||
)
|
||||
assert result == "Successfully updated note 'plan'."
|
||||
assert result == "Updated note 'plan'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "Step 1: Completed\nStep 2: Done"
|
||||
@@ -87,7 +87,7 @@ async def test_update_note_pure_with_warning(mock_ctx, tmp_path):
|
||||
target="target string",
|
||||
replacement="New String",
|
||||
)
|
||||
assert "Successfully updated note 'fuzzy_note'." in result
|
||||
assert "Updated note 'fuzzy_note'." in result
|
||||
assert "WARNING: Fuzzy match applied" in result
|
||||
|
||||
|
||||
@@ -102,7 +102,7 @@ async def test_update_note_pure_callable_execution(mock_ctx, tmp_path):
|
||||
target="beta",
|
||||
replacement="gamma",
|
||||
)
|
||||
assert result == "Successfully updated note 'callable_plan'."
|
||||
assert result == "Updated note 'callable_plan'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "callable_plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "alpha gamma"
|
||||
@@ -135,7 +135,7 @@ async def test_get_update_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"replacement": "updated",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully updated note 'export_plan'."
|
||||
assert result == "Updated note 'export_plan'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "export_plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "updated value"
|
||||
@@ -74,7 +74,7 @@ async def test_update_note_direct_execution_success(mock_ctx, tmp_path):
|
||||
target="- [ ] Step 1",
|
||||
replacement="- [x] Step 1",
|
||||
)
|
||||
assert result == "Successfully updated note 'plan'."
|
||||
assert result == "Updated note 'plan'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "- [x] Step 1\n- [ ] Step 2"
|
||||
@@ -95,7 +95,7 @@ async def test_update_note_with_warning(mock_ctx, tmp_path):
|
||||
target="Step 1",
|
||||
replacement="Step 1 Done",
|
||||
)
|
||||
assert "Successfully updated note 'fuzzy_plan'." in result
|
||||
assert "Updated note 'fuzzy_plan'." in result
|
||||
assert "WARNING: Fuzzy match applied" in result
|
||||
|
||||
|
||||
@@ -110,7 +110,7 @@ async def test_update_note_callable_execution(mock_ctx, tmp_path):
|
||||
target="- [ ] Task 2",
|
||||
replacement="- [x] Task 2",
|
||||
)
|
||||
assert result == "Successfully updated note 'callable_plan'."
|
||||
assert result == "Updated note 'callable_plan'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "callable_plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "- [ ] Task 1\n- [x] Task 2"
|
||||
@@ -134,7 +134,7 @@ async def test_update_note_with_state_tool_message(mock_ctx, tmp_path):
|
||||
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert result.tool_call_id == "call_update_999"
|
||||
assert result.content == "Successfully updated note 'state_plan'."
|
||||
assert result.content == "Updated note 'state_plan'."
|
||||
assert result.status == "success"
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "state_plan.md"
|
||||
@@ -168,7 +168,7 @@ async def test_get_update_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
|
||||
"replacement": "delta",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully updated note 'lc_plan'."
|
||||
assert result == "Updated note 'lc_plan'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "lc_plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "alpha delta gamma"
|
||||
@@ -189,7 +189,7 @@ async def test_get_update_note_tool_pure(mock_ctx, tmp_path):
|
||||
"replacement": "Artemis",
|
||||
}
|
||||
)
|
||||
assert result == "Successfully updated note 'pure_plan'."
|
||||
assert result == "Updated note 'pure_plan'."
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "pure_plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "hello Artemis"
|
||||
@@ -15,9 +15,94 @@
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.utils.app_launch_utils import launch_app_with_retries
|
||||
from artemis.utils.app_launch_utils import (
|
||||
ForegroundTask,
|
||||
_handle_initial_app_launch,
|
||||
get_foreground_task,
|
||||
launch_app_with_retries,
|
||||
parse_foreground_task,
|
||||
)
|
||||
import pytest
|
||||
|
||||
SETTINGS = "com.android.settings"
|
||||
SETTINGS_SEARCH = "com.google.android.settings.intelligence"
|
||||
LAUNCHER = "com.google.android.apps.nexuslauncher"
|
||||
YOUTUBE = "com.google.android.youtube"
|
||||
|
||||
# Trimmed from a real Pixel 10 ``dumpsys activity activities`` right after
|
||||
# ``monkey -p com.android.settings -c android.intent.category.LAUNCHER 1`` re-opened a
|
||||
# Settings task whose previous run had left the search activity (a different package)
|
||||
# resumed on top. The focused window is settings.intelligence; the task is Settings'.
|
||||
DUMP_SETTINGS_TASK_WITH_SEARCH_ON_TOP = """\
|
||||
ACTIVITY MANAGER ACTIVITIES (dumpsys activity activities)
|
||||
Display #0 (activities from top to bottom):
|
||||
* Task{97d2927 #121 type=standard A=1000:com.android.settings.root U=0 visible=true visibleRequested=true mode=fullscreen translucent=false sz=2}
|
||||
mLastPausedActivity: ActivityRecord{248426470 u0 com.android.settings/.Settings t121}
|
||||
isSleeping=false
|
||||
topResumedActivity=ActivityRecord{55960290 u0 com.google.android.settings.intelligence/.modules.search.SearchActivity t121}
|
||||
* Hist #1: ActivityRecord{55960290 u0 com.google.android.settings.intelligence/.modules.search.SearchActivity t121}
|
||||
packageName=com.google.android.settings.intelligence processName=com.google.android.settings.intelligence
|
||||
launchedFromUid=1000 launchedFromPackage=com.android.settings launchedFromFeature=null userId=0
|
||||
rootOfTask=false task=Task{97d2927 #121 type=standard A=1000:com.android.settings.root}
|
||||
taskAffinity=10164:com.google.android.settings.intelligence
|
||||
state=RESUMED delayedResume=false finishing=false
|
||||
* Hist #0: ActivityRecord{248426470 u0 com.android.settings/.Settings t121}
|
||||
packageName=com.android.settings processName=com.android.settings
|
||||
rootOfTask=true task=Task{97d2927 #121 type=standard A=1000:com.android.settings.root}
|
||||
taskAffinity=1000:com.android.settings.root
|
||||
state=STOPPED delayedResume=false finishing=false
|
||||
* Task{9580295 #7 type=home U=0 visible=false visibleRequested=false mode=fullscreen translucent=true sz=1}
|
||||
* Task{a3c724c #8 type=home I=com.google.android.apps.nexuslauncher/.NexusLauncherActivity U=0 rootTaskId=7 visible=false visibleRequested=false mode=fullscreen translucent=true sz=2}
|
||||
* Hist #1: ActivityRecord{230175776 u0 com.google.android.apps.nexuslauncher/.NexusLauncherActivity t8}
|
||||
state=STOPPED delayedResume=false finishing=false
|
||||
* Task{ebe93a1 #108 type=standard A=10194:com.google.android.apps.maps U=0 visible=false visibleRequested=false mode=fullscreen translucent=true sz=1}
|
||||
* Hist #0: ActivityRecord{210087944 u0 com.google.android.apps.maps/com.google.android.maps.MapsActivity t108}
|
||||
ResumedActivity: ActivityRecord{55960290 u0 com.google.android.settings.intelligence/.modules.search.SearchActivity t121}
|
||||
mFocusedApp=ActivityRecord{55960290 u0 com.google.android.settings.intelligence/.modules.search.SearchActivity t121}
|
||||
WindowContainer hierarchy:
|
||||
* Task{97d2927 #121 type=standard A=1000:com.android.settings.root U=0 visible=true visibleRequested=true mode=fullscreen translucent=false sz=2}
|
||||
* ActivityRecord{55960290 u0 com.google.android.settings.intelligence/.modules.search.SearchActivity t121}
|
||||
* ActivityRecord{248426470 u0 com.android.settings/.Settings t121}
|
||||
"""
|
||||
|
||||
# Same device after KEYCODE_HOME: the launcher's (nested) home task is resumed.
|
||||
DUMP_HOME = """\
|
||||
ACTIVITY MANAGER ACTIVITIES (dumpsys activity activities)
|
||||
Display #0 (activities from top to bottom):
|
||||
* Task{9580295 #7 type=home U=0 visible=true visibleRequested=true mode=fullscreen translucent=false sz=1}
|
||||
* Task{a3c724c #8 type=home I=com.google.android.apps.nexuslauncher/.NexusLauncherActivity U=0 rootTaskId=7 visible=true visibleRequested=true mode=fullscreen translucent=false sz=2}
|
||||
topResumedActivity=ActivityRecord{230175776 u0 com.google.android.apps.nexuslauncher/.NexusLauncherActivity t8}
|
||||
* Hist #1: ActivityRecord{230175776 u0 com.google.android.apps.nexuslauncher/.NexusLauncherActivity t8}
|
||||
state=RESUMED delayedResume=false finishing=false
|
||||
* Task{97d2927 #121 type=standard A=1000:com.android.settings.root U=0 visible=false visibleRequested=false mode=fullscreen translucent=true sz=2}
|
||||
* Hist #1: ActivityRecord{55960290 u0 com.google.android.settings.intelligence/.modules.search.SearchActivity t121}
|
||||
* Hist #0: ActivityRecord{248426470 u0 com.android.settings/.Settings t121}
|
||||
ResumedActivity: ActivityRecord{230175776 u0 com.google.android.apps.nexuslauncher/.NexusLauncherActivity t8}
|
||||
"""
|
||||
|
||||
# Search activity started directly with ``am start -n``: it lands in its own task, so
|
||||
# Settings is *not* the foreground app even though its task still exists underneath.
|
||||
DUMP_SEARCH_IN_OWN_TASK = """\
|
||||
Display #0 (activities from top to bottom):
|
||||
* Task{7f6102 #120 type=standard A=10164:com.google.android.settings.intelligence U=0 visible=true visibleRequested=true mode=fullscreen translucent=false sz=1}
|
||||
topResumedActivity=ActivityRecord{53985869 u0 com.google.android.settings.intelligence/.modules.search.SearchActivity t120}
|
||||
* Hist #0: ActivityRecord{53985869 u0 com.google.android.settings.intelligence/.modules.search.SearchActivity t120}
|
||||
* Task{c38138b #119 type=standard A=1000:com.android.settings.root U=0 visible=false visibleRequested=false mode=fullscreen translucent=true sz=3}
|
||||
* Hist #2: ActivityRecord{24290640 u0 com.android.settings/.SubSettings t119}
|
||||
* Hist #1: ActivityRecord{199753729 u0 com.android.settings/.SubSettings t119}
|
||||
* Hist #0: ActivityRecord{33224026 u0 com.android.settings/.Settings t119}
|
||||
ResumedActivity: ActivityRecord{53985869 u0 com.google.android.settings.intelligence/.modules.search.SearchActivity t120}
|
||||
"""
|
||||
|
||||
# Mid-transition: nothing is resumed yet, the visible task must be used instead.
|
||||
DUMP_NO_RESUMED = """\
|
||||
Display #0 (activities from top to bottom):
|
||||
* Task{97d2927 #121 type=standard A=1000:com.android.settings.root U=0 visible=true visibleRequested=true mode=fullscreen translucent=false sz=1}
|
||||
* Hist #0: ActivityRecord{248426470 u0 com.android.settings/.Settings t121}
|
||||
* Task{ebe93a1 #108 type=standard A=10194:com.google.android.apps.maps U=0 visible=false visibleRequested=false mode=fullscreen translucent=true sz=1}
|
||||
* Hist #0: ActivityRecord{210087944 u0 com.google.android.apps.maps/com.google.android.maps.MapsActivity t108}
|
||||
"""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
@@ -29,120 +114,358 @@ def mock_context():
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_success_immediate(mock_get_foreground, mock_controller_cls, mock_context):
|
||||
"""Test successful immediate app launch."""
|
||||
mock_controller = Mock()
|
||||
mock_controller.launch_app = AsyncMock(return_value=True)
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
|
||||
# App immediately in foreground
|
||||
mock_get_foreground.return_value = "com.google.android.youtube"
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context,
|
||||
"com.google.android.youtube",
|
||||
max_retries=1,
|
||||
max_poll_seconds=2,
|
||||
)
|
||||
|
||||
assert success is True
|
||||
assert error_msg is None
|
||||
mock_controller.launch_app.assert_called_once_with("com.google.android.youtube")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.get_focused_task_package")
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_success_permission_overlay(
|
||||
mock_get_foreground,
|
||||
mock_controller_cls,
|
||||
mock_get_focused_task,
|
||||
mock_context,
|
||||
):
|
||||
"""Test successful launch if overlayed by system permission controller and task is matching."""
|
||||
mock_controller = Mock()
|
||||
mock_controller.launch_app = AsyncMock(return_value=True)
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
|
||||
# System permission controller is focused foreground
|
||||
mock_get_foreground.return_value = "com.google.android.permissioncontroller"
|
||||
# Second layer check: Task stack shows com.google.android.youtube is focused
|
||||
mock_get_focused_task.return_value = "com.google.android.youtube"
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context,
|
||||
"com.google.android.youtube",
|
||||
max_retries=1,
|
||||
max_poll_seconds=2,
|
||||
)
|
||||
|
||||
assert success is True
|
||||
assert error_msg is None
|
||||
mock_controller.launch_app.assert_called_once_with("com.google.android.youtube")
|
||||
mock_get_focused_task.assert_called_once_with(mock_context)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.get_focused_task_package")
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_failure_permission_overlay_wrong_task(
|
||||
mock_get_foreground,
|
||||
mock_controller_cls,
|
||||
mock_get_focused_task,
|
||||
mock_context,
|
||||
):
|
||||
"""Test launch failure if overlayed by system permission controller but task does not match."""
|
||||
def _controller(mock_controller_cls):
|
||||
mock_controller = Mock()
|
||||
mock_controller.launch_app = AsyncMock(return_value=True)
|
||||
mock_controller.terminate_app = AsyncMock()
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
return mock_controller
|
||||
|
||||
# System permission controller is focused foreground
|
||||
mock_get_foreground.return_value = "com.google.android.permissioncontroller"
|
||||
# Second layer check: Task stack is some other app (e.g. com.android.settings)
|
||||
mock_get_focused_task.return_value = "com.android.settings"
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context,
|
||||
"com.google.android.youtube",
|
||||
max_retries=1,
|
||||
max_poll_seconds=1,
|
||||
# ---------------------------------------------------------------------------
|
||||
# parse_foreground_task / ForegroundTask
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_settings_task_with_foreign_activity_on_top():
|
||||
task = parse_foreground_task(DUMP_SETTINGS_TASK_WITH_SEARCH_ON_TOP)
|
||||
|
||||
assert task is not None
|
||||
assert task.task_id == 121
|
||||
assert task.affinity == "com.android.settings.root"
|
||||
assert task.resumed_package == SETTINGS_SEARCH
|
||||
assert task.resumed_component == f"{SETTINGS_SEARCH}/.modules.search.SearchActivity"
|
||||
assert task.base_package == SETTINGS
|
||||
assert task.packages == frozenset({SETTINGS, SETTINGS_SEARCH})
|
||||
|
||||
assert task.owns(SETTINGS) # base activity of the top task
|
||||
assert task.owns(SETTINGS_SEARCH) # resumed activity itself
|
||||
assert not task.owns(LAUNCHER)
|
||||
assert not task.owns("com.google.android.apps.maps") # present, but not foreground
|
||||
|
||||
|
||||
def test_parse_home_screen_uses_nested_home_task():
|
||||
task = parse_foreground_task(DUMP_HOME)
|
||||
|
||||
assert task is not None
|
||||
assert task.task_id == 8
|
||||
assert task.affinity is None
|
||||
assert task.base_package == LAUNCHER
|
||||
assert task.resumed_package == LAUNCHER
|
||||
assert task.owns(LAUNCHER)
|
||||
assert not task.owns(SETTINGS)
|
||||
assert not task.owns(SETTINGS_SEARCH)
|
||||
|
||||
|
||||
def test_parse_search_started_in_its_own_task_does_not_count_as_settings():
|
||||
task = parse_foreground_task(DUMP_SEARCH_IN_OWN_TASK)
|
||||
|
||||
assert task is not None
|
||||
assert task.task_id == 120
|
||||
assert task.base_package == SETTINGS_SEARCH
|
||||
assert task.owns(SETTINGS_SEARCH)
|
||||
assert not task.owns(SETTINGS)
|
||||
|
||||
|
||||
def test_parse_falls_back_to_visible_task_when_nothing_is_resumed():
|
||||
task = parse_foreground_task(DUMP_NO_RESUMED)
|
||||
|
||||
assert task is not None
|
||||
assert task.task_id == 121
|
||||
assert task.resumed_package is None
|
||||
assert task.base_package == SETTINGS
|
||||
assert task.owns(SETTINGS)
|
||||
assert not task.owns("com.google.android.apps.maps")
|
||||
|
||||
|
||||
def test_parse_returns_none_without_task_information():
|
||||
assert parse_foreground_task("") is None
|
||||
assert parse_foreground_task("ACTIVITY MANAGER ACTIVITIES\nnothing here\n") is None
|
||||
|
||||
|
||||
def test_describe_names_task_base_and_resumed_activity():
|
||||
task = parse_foreground_task(DUMP_SETTINGS_TASK_WITH_SEARCH_ON_TOP)
|
||||
|
||||
description = task.describe()
|
||||
|
||||
assert "task #121" in description
|
||||
assert f"base={SETTINGS}" in description
|
||||
assert f"resumed={SETTINGS_SEARCH}/.modules.search.SearchActivity" in description
|
||||
assert "affinity=com.android.settings.root" in description
|
||||
|
||||
|
||||
def test_describe_lists_other_member_packages():
|
||||
task = ForegroundTask(
|
||||
task_id=5,
|
||||
resumed_component="a.b/.Dialog",
|
||||
resumed_package="a.b",
|
||||
base_package="c.d",
|
||||
packages=frozenset({"a.b", "c.d", "e.f"}),
|
||||
)
|
||||
|
||||
assert success is False
|
||||
assert "Failed to launch com.google.android.youtube" in error_msg
|
||||
mock_get_focused_task.assert_called_once_with(mock_context)
|
||||
assert task.describe() == "task #5 (base=c.d, resumed=a.b/.Dialog, also=e.f)"
|
||||
|
||||
|
||||
@patch("artemis.utils.app_launch_utils.get_adb_device")
|
||||
def test_get_foreground_task_reads_dumpsys(mock_get_device, mock_context):
|
||||
device = Mock()
|
||||
device.shell.return_value = DUMP_SETTINGS_TASK_WITH_SEARCH_ON_TOP
|
||||
mock_get_device.return_value = device
|
||||
|
||||
task = get_foreground_task(mock_context)
|
||||
|
||||
# The dump is pre-filtered on the device; the parser only needs task
|
||||
# headers, history entries and the resumed-activity lines.
|
||||
device.shell.assert_called_once()
|
||||
command = device.shell.call_args.args[0]
|
||||
assert command.startswith("dumpsys activity activities | grep -E ")
|
||||
assert task is not None and task.base_package == SETTINGS
|
||||
|
||||
|
||||
@patch("artemis.utils.app_launch_utils.get_adb_device")
|
||||
def test_get_foreground_task_falls_back_to_the_full_dump_without_grep(
|
||||
mock_get_device, mock_context
|
||||
):
|
||||
device = Mock()
|
||||
device.shell.side_effect = [
|
||||
"/system/bin/sh: grep: not found",
|
||||
DUMP_SETTINGS_TASK_WITH_SEARCH_ON_TOP,
|
||||
]
|
||||
mock_get_device.return_value = device
|
||||
|
||||
task = get_foreground_task(mock_context)
|
||||
|
||||
assert [c.args[0] for c in device.shell.call_args_list][-1] == "dumpsys activity activities"
|
||||
assert task is not None and task.base_package == SETTINGS
|
||||
|
||||
|
||||
def test_task_header_accepts_android_11_taskrecord_form():
|
||||
from artemis.utils.app_launch_utils import parse_foreground_task
|
||||
|
||||
dump = "\n".join(
|
||||
[
|
||||
" * TaskRecord{a1b2c3 #7 A=com.example.app U=0 StackId=1 sz=1}",
|
||||
" * Hist #0: ActivityRecord{deadbeef u0 com.example.app/.Main t7}",
|
||||
" ResumedActivity: ActivityRecord{deadbeef u0 com.example.app/.Main t7}",
|
||||
]
|
||||
)
|
||||
task = parse_foreground_task(dump)
|
||||
assert task is not None
|
||||
assert task.task_id == 7 and task.affinity == "com.example.app"
|
||||
assert task.owns("com.example.app")
|
||||
|
||||
|
||||
@patch("artemis.utils.app_launch_utils.get_adb_device")
|
||||
def test_get_foreground_task_tolerates_missing_device_and_errors(mock_get_device, mock_context):
|
||||
mock_get_device.return_value = None
|
||||
assert get_foreground_task(mock_context) is None
|
||||
|
||||
device = Mock()
|
||||
device.shell.side_effect = RuntimeError("adb gone")
|
||||
mock_get_device.return_value = device
|
||||
assert get_foreground_task(mock_context) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# launch_app_with_retries
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.get_foreground_task")
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_failure_wrong_app(mock_get_foreground, mock_controller_cls, mock_context):
|
||||
"""Test launch failure and retry when wrong app remains in foreground."""
|
||||
mock_controller = Mock()
|
||||
mock_controller.launch_app = AsyncMock(return_value=True)
|
||||
mock_controller.terminate_app = AsyncMock()
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
|
||||
# A different app (e.g., settings or home screen) is in foreground
|
||||
mock_get_foreground.return_value = "com.android.settings"
|
||||
async def test_launch_app_success_immediate(
|
||||
mock_get_foreground, mock_controller_cls, mock_get_task, mock_context
|
||||
):
|
||||
"""String-equal foreground package is the fast path; the task stack is not read."""
|
||||
mock_controller = _controller(mock_controller_cls)
|
||||
mock_get_foreground.return_value = YOUTUBE
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context,
|
||||
"com.google.android.youtube",
|
||||
max_retries=2,
|
||||
max_poll_seconds=1,
|
||||
mock_context, YOUTUBE, max_retries=1, max_poll_seconds=2
|
||||
)
|
||||
|
||||
assert success is True
|
||||
assert error_msg is None
|
||||
mock_controller.launch_app.assert_called_once_with(YOUTUBE)
|
||||
mock_get_task.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.get_foreground_task")
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_success_when_top_task_belongs_to_app(
|
||||
mock_get_foreground, mock_controller_cls, mock_get_task, mock_context
|
||||
):
|
||||
"""The Settings search activity on top of the Settings task counts as loaded."""
|
||||
mock_controller = _controller(mock_controller_cls)
|
||||
mock_get_foreground.return_value = SETTINGS_SEARCH
|
||||
mock_get_task.return_value = parse_foreground_task(DUMP_SETTINGS_TASK_WITH_SEARCH_ON_TOP)
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context, SETTINGS, max_retries=1, max_poll_seconds=2
|
||||
)
|
||||
|
||||
assert success is True
|
||||
assert error_msg is None
|
||||
mock_controller.launch_app.assert_called_once_with(SETTINGS)
|
||||
mock_controller.terminate_app.assert_not_called()
|
||||
mock_get_task.assert_called_once_with(mock_context)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.get_foreground_task")
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_success_permission_overlay_in_app_task(
|
||||
mock_get_foreground, mock_controller_cls, mock_get_task, mock_context
|
||||
):
|
||||
"""A permission dialog launched into the app's task is accepted without a package list."""
|
||||
_controller(mock_controller_cls)
|
||||
mock_get_foreground.return_value = "com.google.android.permissioncontroller"
|
||||
mock_get_task.return_value = ForegroundTask(
|
||||
task_id=42,
|
||||
resumed_component="com.google.android.permissioncontroller/.GrantPermissionsActivity",
|
||||
resumed_package="com.google.android.permissioncontroller",
|
||||
base_package=YOUTUBE,
|
||||
packages=frozenset({YOUTUBE, "com.google.android.permissioncontroller"}),
|
||||
)
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context, YOUTUBE, max_retries=1, max_poll_seconds=2
|
||||
)
|
||||
|
||||
assert success is True
|
||||
assert error_msg is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.get_foreground_task")
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_failure_when_top_task_is_another_app(
|
||||
mock_get_foreground, mock_controller_cls, mock_get_task, mock_context
|
||||
):
|
||||
"""Foreground task rooted in another package is still a failure, and the error names both."""
|
||||
mock_controller = _controller(mock_controller_cls)
|
||||
mock_get_foreground.return_value = LAUNCHER
|
||||
mock_get_task.return_value = parse_foreground_task(DUMP_HOME)
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context, SETTINGS, max_retries=2, max_poll_seconds=1
|
||||
)
|
||||
|
||||
assert success is False
|
||||
assert "Failed to launch com.google.android.youtube" in error_msg
|
||||
|
||||
# Should have attempted launching twice
|
||||
assert f"Failed to launch {SETTINGS}" in error_msg
|
||||
assert mock_controller.launch_app.call_count == 2
|
||||
# Should have attempted force-stopping once before retry
|
||||
assert mock_controller.terminate_app.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.get_foreground_task")
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_timeout_message_names_foreground_package_and_task(
|
||||
mock_get_foreground, mock_controller_cls, mock_get_task, mock_context
|
||||
):
|
||||
_controller(mock_controller_cls)
|
||||
mock_get_foreground.return_value = SETTINGS_SEARCH
|
||||
mock_get_task.return_value = parse_foreground_task(DUMP_SEARCH_IN_OWN_TASK)
|
||||
|
||||
with patch("artemis.utils.app_launch_utils.logger") as mock_logger:
|
||||
success, _ = await launch_app_with_retries(
|
||||
mock_context, SETTINGS, max_retries=1, max_poll_seconds=1
|
||||
)
|
||||
|
||||
assert success is False
|
||||
timeout_messages = [
|
||||
call.args[0]
|
||||
for call in mock_logger.error.call_args_list
|
||||
if "Timeout waiting" in call.args[0]
|
||||
]
|
||||
assert len(timeout_messages) == 1
|
||||
message = timeout_messages[0]
|
||||
assert f"Current foreground: {SETTINGS_SEARCH}" in message
|
||||
assert "task #120" in message
|
||||
assert f"base={SETTINGS_SEARCH}" in message
|
||||
assert f"resumed={SETTINGS_SEARCH}/.modules.search.SearchActivity" in message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.get_foreground_task")
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_null_focus_keeps_polling_without_reading_task(
|
||||
mock_get_foreground, mock_controller_cls, mock_get_task, mock_context
|
||||
):
|
||||
"""mCurrentFocus=null is a loading state: no task lookup, then success once focused."""
|
||||
_controller(mock_controller_cls)
|
||||
mock_get_foreground.side_effect = [None, None, YOUTUBE]
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context, YOUTUBE, max_retries=1, max_poll_seconds=3
|
||||
)
|
||||
|
||||
assert success is True
|
||||
assert error_msg is None
|
||||
mock_get_task.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _handle_initial_app_launch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
|
||||
@patch("artemis.utils.app_launch_utils.get_foreground_task")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_initial_launch_skips_launch_when_app_task_is_already_foreground(
|
||||
mock_get_foreground, mock_get_task, mock_launch, mock_context
|
||||
):
|
||||
mock_get_foreground.return_value = SETTINGS_SEARCH
|
||||
mock_get_task.return_value = parse_foreground_task(DUMP_SETTINGS_TASK_WITH_SEARCH_ON_TOP)
|
||||
|
||||
result = await _handle_initial_app_launch(mock_context, SETTINGS)
|
||||
|
||||
assert result.locked_app_initial_launch_success is True
|
||||
assert result.locked_app_initial_launch_error is None
|
||||
mock_launch.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.launch_app_with_retries", new_callable=AsyncMock)
|
||||
@patch("artemis.utils.app_launch_utils.get_foreground_task")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_initial_launch_launches_when_another_app_is_foreground(
|
||||
mock_get_foreground, mock_get_task, mock_launch, mock_context
|
||||
):
|
||||
mock_get_foreground.return_value = LAUNCHER
|
||||
mock_get_task.return_value = parse_foreground_task(DUMP_HOME)
|
||||
mock_launch.return_value = (True, None)
|
||||
|
||||
result = await _handle_initial_app_launch(mock_context, SETTINGS)
|
||||
|
||||
assert result.locked_app_initial_launch_success is True
|
||||
mock_launch.assert_awaited_once_with(mock_context, SETTINGS)
|
||||
|
||||
|
||||
def test_task_affinity_alone_identifies_the_owning_app():
|
||||
"""Seen on a Pixel 10: the Settings task had lost its root activity and only
|
||||
the search companion remained in its history, but the task affinity
|
||||
(``com.android.settings.root``) still named the app."""
|
||||
from artemis.utils.app_launch_utils import ForegroundTask
|
||||
|
||||
task = ForegroundTask(
|
||||
task_id=121,
|
||||
affinity="com.android.settings.root",
|
||||
resumed_component="com.google.android.settings.intelligence/.modules.search.SearchActivity",
|
||||
resumed_package="com.google.android.settings.intelligence",
|
||||
base_package="com.google.android.settings.intelligence",
|
||||
packages=frozenset({"com.google.android.settings.intelligence"}),
|
||||
)
|
||||
assert task.owns("com.android.settings")
|
||||
assert not task.owns("com.android.settingsx")
|
||||
assert not task.owns("com.google.android.apps.nexuslauncher")
|
||||
@@ -205,8 +205,8 @@ def test_restore_missing_check_items_keeps_additions():
|
||||
|
||||
|
||||
def test_render_plan_grammar_spec_conditional():
|
||||
base = render_plan_grammar_spec(include_checks=False)
|
||||
extended = render_plan_grammar_spec(include_checks=True)
|
||||
base = render_plan_grammar_spec(midway=False, final=False)
|
||||
extended = render_plan_grammar_spec(midway=True, final=True)
|
||||
# Both-gates-off output carries zero trace of the checking feature
|
||||
assert base == PLAN_GRAMMAR_SPEC
|
||||
assert "verify" not in base and "assert" not in base
|
||||
@@ -214,6 +214,27 @@ def test_render_plan_grammar_spec_conditional():
|
||||
assert "- verify:" in extended and "- assert:" in extended and "@end" in extended
|
||||
# The capability boundary is declared honestly
|
||||
assert "POST-HOC" in extended.upper() or "post-hoc" in extended
|
||||
# Compatibility shim: ``include_checks`` means "both gates".
|
||||
assert render_plan_grammar_spec(include_checks=True) == extended
|
||||
assert render_plan_grammar_spec(include_checks=False) == base
|
||||
|
||||
|
||||
def test_render_plan_grammar_spec_words_the_judgment_moment_per_gate():
|
||||
"""The grammar tells the truth about when a check line is judged: a
|
||||
completion-time Checker with a repair loop only exists with midway checks;
|
||||
the factory layering (final review only) judges once at task exit."""
|
||||
midway = render_plan_grammar_spec(midway=True, final=True)
|
||||
assert "at the moment the milestone is marked completed" in midway
|
||||
assert "reopens the milestone for repair" in midway
|
||||
|
||||
final_only = render_plan_grammar_spec(midway=False, final=True)
|
||||
assert (
|
||||
"judged once at task exit from the recorded evidence around the milestone's"
|
||||
" completion; there is no midway repair loop" in final_only
|
||||
)
|
||||
assert "at the moment the milestone is marked completed" not in final_only
|
||||
assert "reopens the milestone" not in final_only
|
||||
assert "- assert:" in final_only and "@end" in final_only
|
||||
|
||||
|
||||
def test_grammar_spec_renders_constants():
|
||||
@@ -291,7 +312,10 @@ def test_apply_finding_lines_drops_entries_for_missing_subgoals():
|
||||
|
||||
|
||||
def test_finding_grammar_doc_is_conditional():
|
||||
base = render_plan_grammar_spec(include_checks=False)
|
||||
extended = render_plan_grammar_spec(include_checks=True)
|
||||
base = render_plan_grammar_spec(midway=False, final=False)
|
||||
midway = render_plan_grammar_spec(midway=True, final=False)
|
||||
final_only = render_plan_grammar_spec(midway=False, final=True)
|
||||
assert "finding" not in base
|
||||
assert "- finding:" in extended and "checker-" in extended
|
||||
assert "- finding:" in midway and "checker-" in midway
|
||||
# Finding lines are projected by the midway checkpoint machinery only.
|
||||
assert "- finding:" not in final_only
|
||||
@@ -48,7 +48,7 @@ def test_build_plan_and_history_separated_layout():
|
||||
assert "--- Execution History ---" in output
|
||||
assert "- **Step 1 (Most Recent Step, Start: 2.5s)**" in output
|
||||
assert " - Thought 1" in output
|
||||
assert " * [Planned Action]: Launched app 'None'" in output
|
||||
assert " * [Action]: Launched app 'None'" in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_granularities():
|
||||
@@ -298,7 +298,7 @@ def test_build_plan_and_history_with_tool_calls():
|
||||
]
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
|
||||
assert "* [Operator Decision Loop]:" in output
|
||||
assert "* [Reasoning & tool calls]:" in output
|
||||
assert '`read_note({"key": "other_note"})` -> - [ ] Milestone' in output
|
||||
|
||||
|
||||
@@ -339,7 +339,7 @@ def test_build_plan_and_history_with_interleaved_events():
|
||||
assert ' - [Tool Call]: `read_note({"file": "notes.txt"})` -> Milestone plan' in output
|
||||
|
||||
# It should use the new section header
|
||||
assert "* [Operator Decision Loop]:" in output
|
||||
assert "* [Reasoning & tool calls]:" in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_concatenates_thoughts():
|
||||
@@ -396,14 +396,14 @@ def test_build_plan_and_history_interleaved_decision_loop():
|
||||
assert plan in output
|
||||
assert "--- Execution History ---" in output
|
||||
assert "- **Step 1 (Most Recent Step, Start: 2.5s)**" in output
|
||||
assert "* [Operator Decision Loop]:" in output
|
||||
assert "* [Reasoning & tool calls]:" in output
|
||||
assert " - I should find search icon." in output
|
||||
assert (
|
||||
'- [Tool Call]: `ask_explorer({"query": "search icon"})` -> {"center":'
|
||||
" [500, 600]}" in output
|
||||
)
|
||||
assert " - Clicking coordinates [500, 600] now." in output
|
||||
assert "* [Planned Action]: Tapped 'Search' at None" in output
|
||||
assert "* [Action]: Tapped 'Search' at None" in output
|
||||
assert "* [Validator Execution Result]" not in output
|
||||
|
||||
|
||||
@@ -460,12 +460,9 @@ def test_build_plan_and_history_safety_net_and_incident():
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
|
||||
assert "- **Step 1 (Most Recent Step, Start: 5.0s)**" in output
|
||||
assert "* [Operator Decision Loop]:" in output
|
||||
assert "* [Reasoning & tool calls]:" in output
|
||||
assert " - Let's click search." in output
|
||||
assert (
|
||||
"* [Planned Action]: Tapped 'Search' at None (Intercepted by"
|
||||
" Pre-Execution Safety Net)" in output
|
||||
)
|
||||
assert "* [Action]: Tapped 'Search' at None (Intercepted by Pre-Execution Safety Net)" in output
|
||||
assert "* [Pre-Execution Safety Net]:" in output
|
||||
assert (
|
||||
"- [Safety Net Check]: (Intercepted by Pre-Execution Safety Net: Target button is"
|
||||
@@ -475,7 +472,7 @@ def test_build_plan_and_history_safety_net_and_incident():
|
||||
assert "Failure Analyzer" not in output
|
||||
assert (
|
||||
"* [Result]: Error: Intercepted by Pre-Execution Safety Net"
|
||||
" (target_disappeared, consecutive failure #2) on action `Tapped 'Search' at"
|
||||
" (target_disappeared, consecutive failure #2) on action `tap 'Search' at"
|
||||
" None`: Target button is not visible" in output
|
||||
)
|
||||
|
||||
@@ -528,16 +525,14 @@ def test_build_plan_and_history_fast_action_burst():
|
||||
]
|
||||
# Detailed view: every member with its outcome.
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
assert (
|
||||
"* [Planned Fast-Action Burst]: 3 actions fired back to back without the"
|
||||
" safety net" in output
|
||||
)
|
||||
assert "* [Fast-Action Burst]: 3 actions fired back to back without the safety net" in output
|
||||
assert " 1. Tapped element at [500, 900] (dispatched)" in output
|
||||
assert " 2. Tapped 'Skip' at [880, 120] (FAILED: Error: tap rejected)" in output
|
||||
assert " 3. Pressed key 'BACK' (skipped)" in output
|
||||
# The failed member is quoted in intent form: it did not happen.
|
||||
assert (
|
||||
"* [Result]: Error: Execution failed (general, consecutive failure #1) on burst"
|
||||
" action 2/3 `Tapped 'Skip' at [880, 120]`: Error: tap rejected; the remaining"
|
||||
" action 2/3 `tap 'Skip' at [880, 120]`: Error: tap rejected; the remaining"
|
||||
" burst actions were not executed" in output
|
||||
)
|
||||
|
||||
@@ -662,7 +657,7 @@ def test_render_step_replay_shows_summary_thoughts_and_full_tool_results():
|
||||
assert "`ask_explorer(" in out
|
||||
assert long_result in out
|
||||
# Action tools are the planned action, not a tool-call line.
|
||||
assert "[Planned Action]: Tapped 'Save' at [0.5, 0.5]" in out
|
||||
assert "[Action]: Tapped 'Save' at [0.5, 0.5]" in out
|
||||
assert "`click(" not in out
|
||||
|
||||
|
||||
@@ -715,6 +710,174 @@ def test_render_step_replay_shows_described_images_in_tool_results():
|
||||
assert "[screenshot: pre-action of Step 1]" in out
|
||||
|
||||
|
||||
def test_render_step_replay_header_uses_the_session_clock():
|
||||
"""The replay header carries the same ``T+mm:ss`` offset the prompts and
|
||||
``search_history`` use; the legacy ``Start:`` text only remains when no
|
||||
session clock / step timestamp exists."""
|
||||
from artemis.utils.task_tree import render_step_replay, replay_time_label
|
||||
|
||||
step = _replay_step("ok", timestamp=1000.0 + 61.0)
|
||||
out = render_step_replay(step, session_start=1000.0)
|
||||
assert "- **Step 4 (T+01:01)**" in out
|
||||
assert "Start:" not in out
|
||||
|
||||
# No session start (or a non-numeric one, e.g. a mocked reader): fallback.
|
||||
assert "- **Step 4 (Start: 12.0s)**" in render_step_replay(step)
|
||||
assert "- **Step 4 (Start: 12.0s)**" in render_step_replay(step, session_start=object())
|
||||
# A step without a timestamp cannot be placed on the session clock.
|
||||
assert replay_time_label(_replay_step("ok"), 1000.0) == "Start: 12.0s"
|
||||
|
||||
|
||||
def test_render_step_replay_labels_are_role_neutral():
|
||||
"""Flash has no Operator, so the block labels name what they hold."""
|
||||
from artemis.utils.task_tree import render_step_replay
|
||||
|
||||
out = render_step_replay(_replay_step("ok"))
|
||||
assert " * [Reasoning & tool calls]:" in out
|
||||
assert " * [Action]: Tapped 'Save' at [0.5, 0.5]" in out
|
||||
assert "Operator Decision Loop" not in out
|
||||
assert "Planned Action" not in out
|
||||
|
||||
|
||||
def test_render_step_replay_prints_a_failure_once():
|
||||
"""A failed step shows its error on the [Result] line only, not appended
|
||||
to the action line as well."""
|
||||
from artemis.utils.task_tree import render_step_replay
|
||||
|
||||
out = render_step_replay(
|
||||
_replay_step(
|
||||
"ok",
|
||||
last_execution_result={
|
||||
"status": "failed",
|
||||
"execution": [{"action": "click", "attempts": ["Error: tap rejected"]}],
|
||||
},
|
||||
)
|
||||
)
|
||||
assert " * [Action]: Tapped 'Save' at [0.5, 0.5]\n" in out
|
||||
assert " * [Result]: Error: tap rejected" in out
|
||||
assert out.count("tap rejected") == 1
|
||||
assert "Execution failed:" not in out
|
||||
|
||||
|
||||
def test_render_step_replay_indents_multiline_reasoning():
|
||||
from artemis.utils.task_tree import render_step_replay
|
||||
|
||||
step = _replay_step(
|
||||
"ok",
|
||||
interleaved_events=[
|
||||
{"type": "thought", "content": "- Save findings first.\nTap Wi-Fi.\nThen verify."}
|
||||
],
|
||||
)
|
||||
out = render_step_replay(step)
|
||||
assert "\n - - Save findings first.\n Tap Wi-Fi.\n Then verify." in out
|
||||
|
||||
|
||||
def test_format_action_clean_directional_swipe_shows_direction_first():
|
||||
"""``swipe(direction="up")`` replays as the direction the agent issued; the
|
||||
recorded path is a parenthesised detail (Flash records both)."""
|
||||
from artemis.utils.task_tree import format_action_clean
|
||||
|
||||
flash_record = {
|
||||
"action": "swipe",
|
||||
"coordinates": [600, 700, 600, 300],
|
||||
"coordinate_space": "normalized",
|
||||
"normalized_start_coordinates": [600, 700],
|
||||
"normalized_end_coordinates": [600, 300],
|
||||
"args": {"direction": "up", "duration": 800},
|
||||
}
|
||||
assert (
|
||||
format_action_clean(flash_record) == "Swiped up (from [600, 700] to [600, 300]) over 800ms"
|
||||
)
|
||||
assert format_action_clean({"action": "swipe", "direction": "down"}) == "Swiped down"
|
||||
assert (
|
||||
format_action_clean(
|
||||
{
|
||||
"action": "swipe",
|
||||
"gesture": "left",
|
||||
"start_coordinates": [750, 500],
|
||||
"end_coordinates": [250, 500],
|
||||
}
|
||||
)
|
||||
== "Swiped left (from [750, 500] to [250, 500])"
|
||||
)
|
||||
# Without a direction the path is the headline, as before.
|
||||
assert (
|
||||
format_action_clean({"action": "swipe", "coordinates": [600, 700, 600, 300]})
|
||||
== "Swiped from [600, 700] to [600, 300]"
|
||||
)
|
||||
|
||||
|
||||
def test_action_intent_phrase_and_incident_line_use_intent_form():
|
||||
"""Failure contexts describe what the agent tried to do, not an outcome."""
|
||||
from artemis.utils.task_tree import (
|
||||
action_intent_phrase,
|
||||
format_action_intent,
|
||||
format_incident_clean,
|
||||
)
|
||||
|
||||
assert format_action_intent({"action": "launch_app", "app_name": "NonexistentApp"}) == (
|
||||
"launch app 'NonexistentApp'"
|
||||
)
|
||||
assert (
|
||||
format_action_intent(
|
||||
{"action": "click", "coordinates": [500, 520], "target_description": "Wi-Fi row"}
|
||||
)
|
||||
== "tap 'Wi-Fi row' (self-described) at [500, 520]"
|
||||
)
|
||||
assert format_action_intent({"action": "swipe", "direction": "up"}) == "swipe up"
|
||||
assert (
|
||||
format_action_intent(
|
||||
{
|
||||
"action": "input_text",
|
||||
"text": "hello",
|
||||
"target_text": "Search",
|
||||
"coordinates": [1, 2],
|
||||
}
|
||||
)
|
||||
== "type 'hello' into 'Search' at [1, 2]"
|
||||
)
|
||||
assert format_action_intent({"action": "press_key", "keycode": "BACK"}) == "press key 'BACK'"
|
||||
assert format_action_intent({"action": "wait_for_delay", "delay_ms": 200}) == "wait 200ms"
|
||||
assert format_action_intent({"action": "long_press", "coordinates": [1, 2]}) == (
|
||||
"long press element at [1, 2]"
|
||||
)
|
||||
assert format_action_intent({"action": "click", "coordinates": [1, 2], "times": 2}) == (
|
||||
"double tap element at [1, 2]"
|
||||
)
|
||||
# Unknown phrases pass through untouched.
|
||||
assert action_intent_phrase("Fast-action burst (2 actions, unvetted): x") == (
|
||||
"Fast-action burst (2 actions, unvetted): x"
|
||||
)
|
||||
assert action_intent_phrase("") == ""
|
||||
|
||||
line = format_incident_clean(
|
||||
{
|
||||
"kind": "exec_error",
|
||||
"category": "general",
|
||||
"consecutive_failures": 2,
|
||||
"action": {"action": "launch_app", "app_name": "NonexistentApp"},
|
||||
"action_description": "Launched app 'NonexistentApp'",
|
||||
"reason": "Error finding package for app: NonexistentApp",
|
||||
}
|
||||
)
|
||||
assert line == (
|
||||
"Error: Execution failed (general, consecutive failure #2) on action"
|
||||
" `launch app 'NonexistentApp'`: Error finding package for app: NonexistentApp"
|
||||
)
|
||||
intercepted = format_incident_clean(
|
||||
{
|
||||
"kind": "safety_net",
|
||||
"category": "target_disappeared",
|
||||
"action": {"action": "click", "target_text": "Skip", "coordinates": [880, 120]},
|
||||
"reason": "Target element 'Skip' was not found on the screen.",
|
||||
}
|
||||
)
|
||||
assert intercepted.startswith(
|
||||
"Error: Intercepted by Pre-Execution Safety Net (target_disappeared, consecutive"
|
||||
" failure #1) on action `tap 'Skip' at [880, 120]`:"
|
||||
)
|
||||
|
||||
|
||||
def test_format_action_clean_reads_flash_arguments_and_maps_manage_app():
|
||||
"""Read Flash action arguments from ``args`` when formatting app launches."""
|
||||
from artemis.utils.task_tree import format_action_clean
|
||||
|
||||
Reference in new issue
Block a user