feat: improve operator progress and recording accuracy

tests/unit/agents/test_operator.py

@@ -1188,140 +1188,6 @@ async def test_operator_background_tasks_prompt_injection():
         adb_registry.background.pop("task_99999", None)
 
 
-@pytest.mark.asyncio
-async def test_operator_task_plan_warning_prompt_injection():
-    from artemis.agents.operator.operator import OperatorNode
-    from artemis.context import ArtemisContext
-    from unittest.mock import AsyncMock, MagicMock, patch
-
-    mock_ctx = MagicMock(spec=ArtemisContext)
-    mock_ctx.execution_setup = None
-    mock_ctx.data_engine = None
-
-    mock_state = MagicMock()
-    mock_state.subagent_calls = []
-    mock_state.initial_goal = "Test goal"
-
-    mock_controller = MagicMock()
-    mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=")
-    mock_device_data = MagicMock()
-    mock_device_data.elements = []
-    mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
-
-    mock_llm = MagicMock()
-    mock_response = MagicMock()
-    mock_response.tool_calls = []
-
-    mock_llm.ainvoke = AsyncMock(return_value=mock_response)
-    mock_llm.bind_tools.return_value = mock_llm
-
-    # Case 1: task_plan updated in the last step (using the "key" argument)
-    steps_updated = [
-        {
-            "step_id": "step_1",
-            "step_number": 1,
-            "tool_calls": [{"name": "click", "args": {"target": 1}}],
-        },
-        {
-            "step_id": "step_2",
-            "step_number": 2,
-            "tool_calls": [
-                {
-                    "name": "update_note",
-                    "args": {"key": "task_plan", "target": "a", "replacement": "b"},
-                }
-            ],
-        },
-    ]
-
-    captured_messages_1 = []
-
-    async def mock_invoke_llm_loop_1(*args, **kwargs):
-        messages = await node._build_prompt(
-            state=mock_state,
-            latest_screenshot_b64="YmFzZTY0X3NjcmVlbnNob3Q=",
-            fused_xml=[],
-            minimal_list="",
-            current_step_num=3,
-            steps=steps_updated,
-            task_plan="",
-        )
-        captured_messages_1.extend(messages)
-        return [], None, None, False
-
-    with (
-        patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
-        patch(
-            "artemis.agents.operator.operator.OperatorNode._invoke_llm_loop",
-            side_effect=mock_invoke_llm_loop_1,
-        ),
-    ):
-        node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
-        node_update = await node(mock_state)
-
-        full_content_1 = ""
-        for m in captured_messages_1:
-            if isinstance(m.content, list):
-                for part in m.content:
-                    if isinstance(part, dict) and "text" in part:
-                        full_content_1 += part["text"] + "\n"
-            elif isinstance(m.content, str):
-                full_content_1 += m.content + "\n"
-
-        assert "Reminder: your last turn executed a Turn-Ending Action" not in full_content_1
-
-    # Case 2: an action turn that did NOT update task_plan
-    steps_not_updated = [
-        {
-            "step_id": "step_1",
-            "step_number": 1,
-            "tool_calls": [{"name": "click", "args": {"target": 1}}],
-        },
-        {
-            "step_id": "step_2",
-            "step_number": 2,
-            "tool_calls": [{"name": "click", "args": {"target": 2}}],
-            "action_taken": {"action": "click", "target": 2},
-        },
-    ]
-
-    captured_messages_2 = []
-
-    async def mock_invoke_llm_loop_2(*args, **kwargs):
-        messages = await node._build_prompt(
-            state=mock_state,
-            latest_screenshot_b64="YmFzZTY0X3NjcmVlbnNob3Q=",
-            fused_xml=[],
-            minimal_list="",
-            current_step_num=3,
-            steps=steps_not_updated,
-            task_plan="",
-        )
-        captured_messages_2.extend(messages)
-        return [], None, None, False
-
-    with (
-        patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
-        patch(
-            "artemis.agents.operator.operator.OperatorNode._invoke_llm_loop",
-            side_effect=mock_invoke_llm_loop_2,
-        ),
-    ):
-        node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
-        node_update = await node(mock_state)
-
-        full_content_2 = ""
-        for m in captured_messages_2:
-            if isinstance(m.content, list):
-                for part in m.content:
-                    if isinstance(part, dict) and "text" in part:
-                        full_content_2 += part["text"] + "\n"
-            elif isinstance(m.content, str):
-                full_content_2 += m.content + "\n"
-
-        assert "Reminder: your last turn executed a Turn-Ending Action" in full_content_2
-
-
 @pytest.mark.asyncio
 async def test_operator_tool_limit_exceeded_warning():
     from artemis.agents.operator.operator import OperatorNode

tests/unit/agents/test_operator_plan_ledger.py

@@ -0,0 +1,277 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for operator plan-ledger handling."""
+
+import json
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from artemis.agents.operator.operator import OperatorNode
+from artemis.agents.operator.prompts import (
+    apply_operator_prompt_contract,
+    load_operator_prompts,
+    render_plan_ledger_bounce,
+    unwritten_action_streak,
+)
+from artemis.config.agent import MemoryTranscriptConfig
+from artemis.context import ArtemisContext
+from artemis.utils.plan_grammar import parse_plan, render_plan_grammar_spec
+
+LEGACY_TRANSCRIPT = MemoryTranscriptConfig(enabled=False)
+
+PLAN_NO_LEAF = """- [x] Open Google Maps and search for SFO
+- [/] Read the commute duration and record it into note `commute_eta_info`
+  - verify: duration recorded
+- [ ] Draft the ETA message
+"""
+
+PLAN_WITH_LEAF = """- [x] Open Google Maps and search for SFO
+- [/] Read the commute duration and record it into note `commute_eta_info`
+  - [x] Open route options for SFO
+  - [/] Read the driving duration
+    - [/] Tap the "Driving" tab so the fastest route is highlighted
+  - verify: duration recorded
+- [ ] Draft the ETA message
+"""
+
+
+def test_grammar_spec_teaches_deeper_nesting():
+    spec = render_plan_grammar_spec(include_checks=False)
+    assert "4 spaces for a sub-sub-goal" in spec
+    assert "Only zero-indent lines count as milestones" in spec
+
+
+def test_active_milestone_and_leaf():
+    snap = parse_plan(PLAN_WITH_LEAF)
+    milestone = snap.active_milestone()
+    assert milestone is not None and milestone.text.startswith("Read the commute duration")
+    children = snap.children_of(milestone)
+    assert [c.text for c in children] == [
+        "Open route options for SFO",
+        "Read the driving duration",
+        'Tap the "Driving" tab so the fastest route is highlighted',
+    ]
+    leaf = snap.active_leaf(milestone)
+    assert leaf is not None and leaf.text.startswith('Tap the "Driving" tab')
+    assert leaf.indent == 4
+
+    snap = parse_plan(PLAN_NO_LEAF)
+    milestone = snap.active_milestone()
+    assert milestone is not None
+    assert snap.active_leaf(milestone) is None
+    assert snap.children_of(milestone) == ()
+
+
+def test_active_milestone_fallbacks():
+    all_pending = parse_plan("- [ ] First\n- [ ] Second\n")
+    assert all_pending.active_milestone().text == "First"
+    nested_only = parse_plan("- [ ] First\n  - [/] Doing it\n- [ ] Second\n")
+    assert nested_only.active_milestone().text == "First"
+    assert parse_plan("- [x] First\n- [x] Second\n").active_milestone() is None
+
+
+def _action_step(n, wrote_plan=False, action=True):
+    tool_calls = []
+    if wrote_plan:
+        tool_calls.append(
+            {"name": "update_note", "args": {"key": "task_plan", "target": "a", "replacement": "b"}}
+        )
+    step = {"step_id": f"s{n}", "step_number": n, "tool_calls": tool_calls}
+    if action:
+        step["action_taken"] = [{"action": "tap"}]
+    return step
+
+
+def test_streak_counts_trailing_action_turns_without_plan_write():
+    assert unwritten_action_streak([]) == 0
+    assert unwritten_action_streak([_action_step(1, wrote_plan=True)]) == 0
+    assert unwritten_action_streak([_action_step(1)]) == 1
+    steps = [
+        _action_step(1),
+        _action_step(2, wrote_plan=True),
+        _action_step(3),
+        _action_step(4, action=False),
+        _action_step(5),
+    ]
+    assert unwritten_action_streak(steps) == 2
+
+
+def test_bounce_only_when_ledger_is_broken_or_stale():
+    assert render_plan_ledger_bounce(PLAN_WITH_LEAF, streak=0, stale_turns=4) is None
+    assert render_plan_ledger_bounce(PLAN_WITH_LEAF, streak=2, stale_turns=4) is None
+    stale = render_plan_ledger_bounce(PLAN_WITH_LEAF, streak=3, stale_turns=4)
+    assert stale and "stayed in progress for 4 action turns" in stale
+    assert 'Tap the "Driving" tab' in stale
+    assert render_plan_ledger_bounce(PLAN_WITH_LEAF, streak=9, stale_turns=0) is None
+
+    broken = render_plan_ledger_bounce(PLAN_NO_LEAF, streak=0, stale_turns=4)
+    assert broken and "has no in-progress `[/]` sub-goal beneath it" in broken
+    assert "Read the commute duration" in broken
+
+    assert render_plan_ledger_bounce("No task plan yet.", 0, 4) is None
+    assert render_plan_ledger_bounce("- [x] Done\n- [x] Also done\n", 0, 4) is None
+
+
+def test_prompt_teaches_the_ledger_once_without_per_turn_reminder():
+    prompt = apply_operator_prompt_contract(load_operator_prompts()["main_template"])
+    assert prompt.count("**Live Sub-goal Ledger**") == 1
+    assert "mandatory every turn" not in prompt
+    assert "need no plan write" in prompt
+    assert "sub-sub-goal (4 spaces)" in prompt
+    assert "structurally broken" in prompt
+
+
+def _make_node(tmp_path, plan_text, steps=None):
+    notes = tmp_path / "notes"
+    notes.mkdir(parents=True)
+    (notes / "task_plan.md").write_text(plan_text, encoding="utf-8")
+
+    ctx = MagicMock(spec=ArtemisContext)
+    ctx.execution_setup = None
+    ctx.data_engine = MagicMock()
+    ctx.data_engine.base_dir = str(tmp_path)
+    ctx.data_engine.current_session_id = "s"
+    ctx.data_engine.get_agent_friendly_steps.return_value = steps or []
+
+    state = MagicMock()
+    state.subagent_calls = []
+    state.initial_goal = "goal"
+    state.open_incident = None
+    node = OperatorNode(ctx, transcript_config=LEGACY_TRANSCRIPT)
+    return node, state, notes / "task_plan.md"
+
+
+def _click_response(n_actions=1):
+    response = MagicMock()
+    response.content = ""
+    response.tool_calls = [
+        {"name": "click", "args": {"target": [50, 50]}, "id": f"call_{i}"} for i in range(n_actions)
+    ]
+    return response
+
+
+def _llm(side_effect=None, return_value=None):
+    llm = MagicMock()
+    llm.ainvoke = AsyncMock(side_effect=side_effect, return_value=return_value)
+    llm.bind_tools.return_value = llm
+    return llm
+
+
+@pytest.mark.asyncio
+async def test_gate_bounces_broken_ledger_once_then_executes(tmp_path):
+    node, state, _ = _make_node(tmp_path, PLAN_NO_LEAF)
+    seen = []
+
+    async def ainvoke(messages, *args, **kwargs):
+        seen.append(list(messages))
+        return _click_response()
+
+    with patch("artemis.agents.operator.operator.get_llm", return_value=_llm(ainvoke)):
+        update = await node(state)
+
+    assert len(seen) == 2, "first submission bounced, second executed"
+    bounce = [
+        m
+        for m in seen[1]
+        if isinstance(getattr(m, "content", None), str)
+        and m.content.startswith("Not executed: the active milestone")
+    ]
+    assert bounce and bounce[0].status == "error"
+    assert json.loads(update["structured_decisions"])[0]["action"] == "tap"
+
+
+@pytest.mark.asyncio
+async def test_gate_lets_healthy_ledger_through_without_a_write(tmp_path):
+    node, state, _ = _make_node(tmp_path, PLAN_WITH_LEAF, steps=[_action_step(1)])
+    llm = _llm(return_value=_click_response())
+    with patch("artemis.agents.operator.operator.get_llm", return_value=llm):
+        update = await node(state)
+    assert llm.ainvoke.await_count == 1
+    assert update["structured_decisions"] is not None
+
+
+@pytest.mark.asyncio
+async def test_gate_bounces_stale_ledger(tmp_path):
+    stale_history = [_action_step(1), _action_step(2), _action_step(3)]
+    node, state, _ = _make_node(tmp_path, PLAN_WITH_LEAF, steps=stale_history)
+    seen = []
+
+    async def ainvoke(messages, *args, **kwargs):
+        seen.append(list(messages))
+        return _click_response()
+
+    with (
+        patch("artemis.agents.operator.operator.get_llm", return_value=_llm(ainvoke)),
+        patch.object(OperatorNode, "_plan_ledger_stale_turns", return_value=4),
+    ):
+        await node(state)
+    assert len(seen) == 2
+    assert any(
+        isinstance(getattr(m, "content", None), str)
+        and "stayed in progress for 4 action turns" in m.content
+        for m in seen[1]
+    )
+
+
+@pytest.mark.asyncio
+async def test_gate_passes_when_plan_changed_in_turn(tmp_path):
+    node, state, plan_path = _make_node(tmp_path, PLAN_NO_LEAF)
+
+    async def ainvoke(messages, *args, **kwargs):
+        plan_path.write_text(PLAN_WITH_LEAF, encoding="utf-8")
+        return _click_response()
+
+    llm = _llm(ainvoke)
+    with patch("artemis.agents.operator.operator.get_llm", return_value=llm):
+        update = await node(state)
+    assert llm.ainvoke.await_count == 1
+    assert update["structured_decisions"] is not None
+
+
+@pytest.mark.asyncio
+async def test_gate_exempts_bursts_and_open_incidents(tmp_path):
+    node, state, _ = _make_node(tmp_path / "burst", PLAN_NO_LEAF)
+    llm = _llm(return_value=_click_response(n_actions=2))
+    with patch("artemis.agents.operator.operator.get_llm", return_value=llm):
+        update = await node(state)
+    assert llm.ainvoke.await_count == 1
+    assert len(json.loads(update["structured_decisions"])) == 2
+
+    node, state, _ = _make_node(tmp_path / "incident", PLAN_NO_LEAF)
+    state.open_incident = {"category": "intercepted"}
+    llm = _llm(return_value=_click_response())
+    with patch("artemis.agents.operator.operator.get_llm", return_value=llm):
+        await node(state)
+    assert llm.ainvoke.await_count == 1
+
+
+@pytest.mark.asyncio
+async def test_gate_inactive_without_plan_or_when_disabled(tmp_path):
+    llm = _llm(return_value=_click_response())
+
+    node, state, _ = _make_node(tmp_path / "a", "No task plan yet.")
+    with patch("artemis.agents.operator.operator.get_llm", return_value=llm):
+        await node(state)
+    assert llm.ainvoke.await_count == 1
+
+    llm.ainvoke.reset_mock()
+    node, state, _ = _make_node(tmp_path / "b", PLAN_NO_LEAF)
+    with (
+        patch("artemis.agents.operator.operator.get_llm", return_value=llm),
+        patch.object(OperatorNode, "_plan_ledger_gate_enabled", return_value=False),
+    ):
+        await node(state)
+    assert llm.ainvoke.await_count == 1

tests/unit/agents/test_operator_transcript.py

@@ -60,10 +60,11 @@ from artemis.memory.transcript import (
 # Incident workflow section and the Phase 3 burst-discipline rule; and again
 # on 2026-09-02 after the prompt consolidation (every rule stated once, the
 # transcript/legacy history-marker split, the recited tool budget, the
-# recommended diagnosis triggers). Any other drift here means the flag-off
-# path is no longer byte-identical.
-GOLDEN_EMPTY_PLAN = "a20046ff46058fa331fa3b8f2d3ffa4a018be567bf264588b9a29bc64e5b11b8"
-GOLDEN_SENTINEL_PLAN = "8f4456524c8a964d1614d26d2fafab809bd5bd3f30334fd40498999e4b27638b"
+# recommended diagnosis triggers); and again on 2026-09-03 for the nested
+# plan-ledger contract and its validation gate. Any other drift means the
+# flag-off path is no longer byte-identical.
+GOLDEN_EMPTY_PLAN = "2ed35be503e3872f008ae5a12616af6f90cde65f23b63adbb6da782079ae4e7c"
+GOLDEN_SENTINEL_PLAN = "8bd6aa60cf3fad8d21cf5c284a01f586c57d209fcd1d2b4a517a791736f9b1de"
 
 SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")