feat: harden agent verification and runtime contracts

Introduce two-gate checker settlement, explicit graph visibility and checkpoints, structured LLM reliability, centralized action specifications, and corresponding tests and design documentation.

artemis/utils/file.py

@@ -13,11 +13,8 @@
 # limitations under the License.
 
 import json
-from pathlib import Path
 import re
-import shutil
 from typing import IO
-from unittest.mock import MagicMock
 
 
 def strip_json_comments(text: str) -> str:
@@ -28,28 +25,3 @@ def strip_json_comments(text: str) -> str:
 
 def load_jsonc(file: IO) -> dict:
     return json.loads(strip_json_comments(file.read()))
-
-
-def create_snapshot(src_dir: Path, snapshot_dir: Path):
-    """Creates a snapshot of a directory by copying it."""
-    if (
-        isinstance(src_dir, MagicMock)
-        or isinstance(snapshot_dir, MagicMock)
-        or "MagicMock" in str(src_dir)
-        or "MagicMock" in str(snapshot_dir)
-    ):
-        return
-    if not src_dir.exists():
-        return
-    if snapshot_dir.exists():
-        shutil.rmtree(snapshot_dir)
-    shutil.copytree(src_dir, snapshot_dir)
-
-
-def restore_snapshot(snapshot_dir: Path, dst_dir: Path):
-    """Restores a directory from a snapshot."""
-    if not snapshot_dir.exists():
-        raise FileNotFoundError(f"Snapshot directory {snapshot_dir} does not exist.")
-    if dst_dir.exists():
-        shutil.rmtree(dst_dir)
-    shutil.copytree(snapshot_dir, dst_dir)

artemis/utils/plan_grammar.py

@@ -0,0 +1,362 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Single source of truth for the task-plan grammar (the "machine channel").
+
+``task_plan.md`` is a dual-channel document:
+
+- **Machine channel**: the checkbox micro-grammar parsed here (status chars,
+  indentation, ``[Loop]`` / ``[Loop:continuous]`` tags). Harness code may only
+  ever read this channel.
+- **Semantic channel**: all remaining free text. Only LLM agents interpret it;
+  harness code must never branch on wording.
+
+Every deterministic decision the harness makes about the plan (termination,
+loop protection, validation/checker triggers, hand-slip detection) derives
+from the :class:`PlanSnapshot` produced by :func:`parse_plan`. The same
+constants render :data:`PLAN_GRAMMAR_SPEC`, the grammar description injected
+into agent prompts via the ``{{ plan_grammar }}`` template variable — so the
+contract the agents are taught and the contract the code enforces cannot
+drift apart.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from collections import Counter
+from dataclasses import dataclass, field
+from typing import Literal
+
+STATUS_PENDING = " "
+STATUS_ACTIVE = "/"
+STATUS_DONE = "x"
+STATUS_BLOCKED = "!"
+STATUS_CHARS = STATUS_PENDING + STATUS_ACTIVE + STATUS_DONE + STATUS_BLOCKED
+
+LOOP_TAG = "[Loop]"
+CONTINUOUS_LOOP_TAG = "[Loop:continuous]"
+
+CHECKBOX_LINE_RE = re.compile(
+    r"^(?P<indent>\s*)-\s*\[(?P<status>[" + re.escape(STATUS_CHARS) + r"])\]\s*(?P<text>.*)$"
+)
+_LOOP_TAG_RE = re.compile(r"\[Loop(?::continuous)?\]")
+_CONTINUOUS_TAG_RE = re.compile(re.escape(CONTINUOUS_LOOP_TAG))
+
+#: Check lines are indented attachments under a top-level subgoal
+#: (``- verify: ...`` / ``- assert: ...``) or task-level free items
+#: (``- assert@end: ...`` at zero indent). The optional ``@end`` suffix moves
+#: the judgment moment from the anchored subgoal's completion to task exit.
+CHECK_LINE_RE = re.compile(
+    r"^(?P<indent>\s*)-\s*(?P<kind>verify|assert)(?P<at_end>@end)?\s*:\s*(?P<text>\S.*?)\s*$"
+)
+
+
+def subgoal_hash(text: str) -> str:
+    """Stable identity hash for a subgoal text (used for per-subgoal artifacts)."""
+    return hashlib.md5(text.strip().encode("utf-8")).hexdigest()
+
+
+@dataclass(frozen=True)
+class PlanItem:
+    """One checkbox line of the plan's machine channel."""
+
+    indent: int
+    status: str
+    text: str
+    line_no: int
+
+    @property
+    def is_top_level(self) -> bool:
+        return self.indent == 0
+
+    @property
+    def is_pending(self) -> bool:
+        return self.status == STATUS_PENDING
+
+    @property
+    def is_active(self) -> bool:
+        return self.status == STATUS_ACTIVE
+
+    @property
+    def is_done(self) -> bool:
+        return self.status == STATUS_DONE
+
+    @property
+    def is_loop(self) -> bool:
+        return bool(_LOOP_TAG_RE.search(self.text))
+
+    @property
+    def is_continuous(self) -> bool:
+        return bool(_CONTINUOUS_TAG_RE.search(self.text))
+
+    @property
+    def key(self) -> str:
+        return subgoal_hash(self.text)
+
+
+@dataclass(frozen=True)
+class CheckItem:
+    """One declared check line of the plan's machine channel.
+
+    ``verify`` marks a completion acceptance criterion (a failure triggers
+    repair); ``assert`` marks a test assertion (a failure is recorded as a
+    legitimate test result, never repaired).
+    """
+
+    kind: Literal["verify", "assert"]
+    when: Literal["on_complete", "at_end"]
+    text: str
+    parent_key: str | None
+
+    @property
+    def key(self) -> str:
+        """Stable identity of the check line (kind + when + text + anchor)."""
+        return subgoal_hash(f"{self.kind}|{self.when}|{self.parent_key}|{self.text}")
+
+
+@dataclass(frozen=True)
+class PlanSnapshot:
+    """Read-only derived view of a plan. Never persisted, never written back."""
+
+    items: tuple[PlanItem, ...]
+    check_items: tuple[CheckItem, ...] = field(default=())
+
+    @property
+    def top_level(self) -> tuple[PlanItem, ...]:
+        return tuple(i for i in self.items if i.is_top_level)
+
+    @property
+    def has_top_level(self) -> bool:
+        return bool(self.top_level)
+
+    @property
+    def all_top_level_done(self) -> bool:
+        top = self.top_level
+        return bool(top) and all(i.is_done for i in top)
+
+    @property
+    def continuous_top_level(self) -> tuple[PlanItem, ...]:
+        return tuple(i for i in self.top_level if i.is_continuous)
+
+    @property
+    def milestone_texts(self) -> tuple[str, ...]:
+        return tuple(i.text.strip() for i in self.top_level)
+
+    def first_active(self) -> PlanItem | None:
+        for item in self.items:
+            if item.is_active:
+                return item
+        return None
+
+    def last_active(self) -> PlanItem | None:
+        for item in reversed(self.items):
+            if item.is_active:
+                return item
+        return None
+
+    def parent_of(self, item: PlanItem) -> PlanItem | None:
+        """Nearest top-level item above an indented item."""
+        parent = None
+        for candidate in self.items:
+            if candidate.line_no >= item.line_no:
+                break
+            if candidate.is_top_level:
+                parent = candidate
+        return parent
+
+    def check_items_of(self, item: PlanItem) -> tuple[CheckItem, ...]:
+        """Check lines anchored to the given top-level subgoal."""
+        return tuple(ci for ci in self.check_items if ci.parent_key == item.key)
+
+    @property
+    def task_level_check_items(self) -> tuple[CheckItem, ...]:
+        return tuple(ci for ci in self.check_items if ci.parent_key is None)
+
+    @property
+    def all_check_items(self) -> tuple[CheckItem, ...]:
+        return self.check_items
+
+
+def parse_plan(content: str | None) -> PlanSnapshot:
+    """Parses plan markdown into its machine channel. Non-checkbox lines are ignored.
+
+    Check lines (``- verify:`` / ``- assert:``) are parsed into
+    :class:`CheckItem` entries; they never contribute to the checkbox item list
+    and therefore never influence milestone hashing, drift detection, or
+    completion triggers.
+    """
+    items: list[PlanItem] = []
+    checks: list[CheckItem] = []
+    current_top: PlanItem | None = None
+    if content:
+        for line_no, line in enumerate(content.splitlines()):
+            match = CHECKBOX_LINE_RE.match(line)
+            if match:
+                item = PlanItem(
+                    indent=len(match.group("indent")),
+                    status=match.group("status"),
+                    text=match.group("text").strip(),
+                    line_no=line_no,
+                )
+                items.append(item)
+                if item.is_top_level:
+                    current_top = item
+                continue
+            check_match = CHECK_LINE_RE.match(line)
+            if check_match:
+                indented = bool(check_match.group("indent"))
+                at_end = bool(check_match.group("at_end"))
+                parent_key = current_top.key if (indented and current_top) else None
+                checks.append(
+                    CheckItem(
+                        kind=check_match.group("kind"),
+                        when="at_end" if at_end else "on_complete",
+                        text=check_match.group("text").strip(),
+                        parent_key=parent_key,
+                    )
+                )
+    return PlanSnapshot(items=tuple(items), check_items=tuple(checks))
+
+
+def milestones_changed(before: PlanSnapshot, after: PlanSnapshot) -> bool:
+    """True if the top-level milestone texts differ (status flips don't count)."""
+    return before.milestone_texts != after.milestone_texts
+
+
+def _check_multiset(snapshot: PlanSnapshot) -> Counter:
+    return Counter((ci.kind, ci.when, ci.text, ci.parent_key) for ci in snapshot.check_items)
+
+
+def check_items_changed(before: PlanSnapshot, after: PlanSnapshot) -> bool:
+    """True if the multiset of check lines differs between the two snapshots."""
+    return _check_multiset(before) != _check_multiset(after)
+
+
+def missing_check_items(before: PlanSnapshot, after: PlanSnapshot) -> list[CheckItem]:
+    """Check items present in ``before`` but absent from ``after`` (multiset diff).
+
+    Additions in ``after`` are ignored — adding check lines is allowed; only
+    deletions/rewrites of existing ones are guarded.
+    """
+    deficit = _check_multiset(before) - _check_multiset(after)
+    missing: list[CheckItem] = []
+    for ci in before.check_items:
+        sig = (ci.kind, ci.when, ci.text, ci.parent_key)
+        if deficit.get(sig, 0) > 0:
+            deficit[sig] -= 1
+            missing.append(ci)
+    return missing
+
+
+def render_check_line(item: CheckItem, indent: int = 2) -> str:
+    suffix = "@end" if item.when == "at_end" else ""
+    prefix = " " * indent if item.parent_key is not None else ""
+    return f"{prefix}- {item.kind}{suffix}: {item.text}"
+
+
+def restore_missing_check_items(content_before: str, content_after: str) -> str | None:
+    """Deterministic guard: merge check lines deleted/rewritten by a plan write
+    back into the new content.
+
+    Returns the merged plan text, or ``None`` when nothing is missing. Lines
+    whose parent subgoal still exists are re-anchored directly under it; lines
+    whose parent subgoal was removed are converted to task-level ``@end`` items
+    appended at the end of the plan.
+    """
+    before = parse_plan(content_before)
+    after = parse_plan(content_after)
+    missing = missing_check_items(before, after)
+    if not missing:
+        return None
+
+    lines = (content_after or "").splitlines()
+    top_line_by_key: dict[str, int] = {}
+    for item in after.items:
+        if item.is_top_level:
+            top_line_by_key.setdefault(item.key, item.line_no)
+
+    inserts: dict[int, list[str]] = {}
+    tail: list[str] = []
+    for ci in missing:
+        if ci.parent_key is not None and ci.parent_key in top_line_by_key:
+            inserts.setdefault(top_line_by_key[ci.parent_key], []).append(render_check_line(ci))
+        else:
+            orphan = CheckItem(kind=ci.kind, when="at_end", text=ci.text, parent_key=None)
+            tail.append(render_check_line(orphan))
+
+    merged: list[str] = []
+    for line_no, line in enumerate(lines):
+        merged.append(line)
+        if line_no in inserts:
+            merged.extend(inserts[line_no])
+    merged.extend(tail)
+    return "\n".join(merged) + ("\n" if (content_after or "").endswith("\n") else "")
+
+
+def new_top_level_completions(before: PlanSnapshot, after: PlanSnapshot) -> list[str]:
+    """Top-level texts newly marked done, in plan order."""
+    before_done = {i.text for i in before.top_level if i.is_done}
+    return [i.text for i in after.top_level if i.is_done and i.text not in before_done]
+
+
+def unintended_milestone_edits(before: PlanSnapshot, after: PlanSnapshot) -> list[tuple[str, str]]:
+    """Hand-slip signature of a full-file rewrite: positional top-level pairs
+    whose status char is unchanged but whose text was reworded.
+
+    Only defined when the milestone count is unchanged; structural changes
+    (add/remove) are a declared replan and are judged by planner validation
+    instead.
+    """
+    top_before = before.top_level
+    top_after = after.top_level
+    if len(top_before) != len(top_after):
+        return []
+    return [
+        (b.text, a.text)
+        for b, a in zip(top_before, top_after)
+        if b.status == a.status and b.text != a.text
+    ]
+
+
+_PLAN_GRAMMAR_BASE = f"""### Task Plan Grammar (machine-enforced contract)
+Every checklist line MUST match `<indent>- [<status>] <text>`:
+- Status characters: `[{STATUS_PENDING}]` pending, `[{STATUS_ACTIVE}]` in progress, `[{STATUS_DONE}]` completed, `[{STATUS_BLOCKED}]` blocked.
+- Top-level lines (no indentation) are strategic milestones; their sub-tasks are indented by 2 spaces. All other text in the note is free-form context.
+- `{LOOP_TAG}` tags a BOUNDED iterative milestone: declare its exit boundary (e.g., `(Exit: <condition>; Interval: <cadence>)`) and mark it `[{STATUS_DONE}]` only once that exit condition is verifiably met.
+- `{CONTINUOUS_LOOP_TAG}` tags an UNBOUNDED continuous-monitoring milestone: it must stay `[{STATUS_ACTIVE}]` and can never be marked `[{STATUS_DONE}]`, deleted, or untagged by you — the system mechanically rejects such edits. Only an explicit external stop signal injected by the user unlocks its completion.
+- The task terminates only when every top-level milestone is `[{STATUS_DONE}]`."""
+
+_CHECK_GRAMMAR_EXTENSION = """### Check Line Grammar (declared verification standards)
+A top-level milestone may carry indented check lines, and the plan may end with task-level check lines at zero indentation:
+- `  - verify: <expected state>` — an acceptance criterion for its parent milestone. It is judged by an independent Checker at the moment the milestone is marked completed; a failed verify triggers a repair loop.
+- `  - assert: <expected observation>` — a test assertion. It is judged by the independent Checker and a failure is recorded verbatim as a legitimate test result; assertions are NEVER repaired, worked around, or satisfied by constructing state.
+- `- verify@end: ...` / `- assert@end: ...` — the `@end` suffix defers judgment to task exit, using the final device state. Items without `@end` are judged with the evidence available when their anchored milestone completes.
+- Capability boundary (declare honestly, never promise more): all checks are POST-HOC audits over recorded evidence. They detect and record violations after the fact; they CANNOT block an action from happening, so "B must not run before A is confirmed" ordering rules are verified retroactively, not enforced. Transient prompts that were never captured in the execution history cannot be recovered — prefer expressing expectations as persistently probeable state."""
+
+
+def render_plan_grammar_spec(include_checks: bool) -> str:
+    """Renders the grammar spec taught to agents.
+
+    ``include_checks=False`` (both check gates disabled) returns exactly the
+    base grammar — zero context pollution from the checking feature.
+    """
+    if include_checks:
+        return _PLAN_GRAMMAR_BASE + "\n\n" + _CHECK_GRAMMAR_EXTENSION
+    return _PLAN_GRAMMAR_BASE
+
+
+#: Backward-compatible module constant: the base grammar without the check
+#: extension. Prompt injection sites use :func:`render_plan_grammar_spec`.
+PLAN_GRAMMAR_SPEC = _PLAN_GRAMMAR_BASE

artemis/utils/task_tree.py

@@ -13,12 +13,13 @@
 # limitations under the License.
 
 import ast
-import hashlib
 import json
 from pathlib import Path
 import re
 from typing import Any
 
+from artemis.utils.plan_grammar import parse_plan
+
 
 def format_action_clean(action_obj) -> str:
     if not action_obj:
@@ -952,53 +953,26 @@ def get_active_subgoal_hashes(task_plan: str) -> tuple[str, str | None]:
         return "default", None
 
     try:
-        lines = task_plan.split("\n")
-        active_line_idx = -1
-        # Search from bottom to top to prioritize the most specific deeply nested active subgoal
-        for i in range(len(lines) - 1, -1, -1):
-            if lines[i].strip().startswith("- [/]"):
-                active_line_idx = i
-                break
+        snapshot = parse_plan(task_plan)
 
-        if active_line_idx != -1:
-            active_line = lines[active_line_idx]
-            active_text = active_line.strip()[5:].strip()
-
-            # Check if it is indented (sub-subgoal)
-            if active_line.startswith(" "):
-                # Find parent (first non-indented line above it)
-                parent_text = ""
-                for i in range(active_line_idx - 1, -1, -1):
-                    if not lines[i].startswith(" ") and lines[i].strip().startswith("- ["):
-                        parent_text = lines[i].strip()[5:].strip()
-                        break
-
-                # Consolidate under the parent Level 1 subgoal
-                if parent_text:
-                    parent_hash = hashlib.md5(parent_text.encode("utf-8")).hexdigest()
-                    return parent_hash, None
-
-            # If not indented, it is a top-level Level 1 subgoal
-            active_hash = hashlib.md5(active_text.encode("utf-8")).hexdigest()
-            return active_hash, None
+        # Bottom-most active item wins (most specific deeply nested subgoal)
+        active_item = snapshot.last_active()
+        if active_item is not None:
+            if not active_item.is_top_level:
+                parent = snapshot.parent_of(active_item)
+                if parent is not None:
+                    return parent.key, None
+            return active_item.key, None
 
         # No active subgoal found. Check if all are completed.
-        top_level_lines = [line for line in lines if line.strip().startswith("- [")]
-        all_done = all(line.strip().startswith("- [x]") for line in top_level_lines)
-        if all_done:
+        if all(item.is_done for item in snapshot.top_level):
             return "default", None
 
-        # SAFE FALLBACK: Check if all subgoals are not started (all are "[ ]")
-        all_pending = all(line.strip().startswith("- [ ]") for line in top_level_lines)
-        if all_pending:
-            # If all are pending, we are at the start of the task.
-            # The active subgoal is guaranteed to be the first one.
-            for line in lines:
-                stripped = line.strip()
-                if stripped.startswith("- [ ]"):
-                    first_pending_text = stripped[5:].strip()
-                    first_hash = hashlib.md5(first_pending_text.encode("utf-8")).hexdigest()
-                    return first_hash, None
+        # SAFE FALLBACK: if every item is untouched, the first pending one is active.
+        if all(item.is_pending for item in snapshot.top_level):
+            for item in snapshot.items:
+                if item.is_pending:
+                    return item.key, None
 
     except Exception:
         pass
@@ -1011,13 +985,7 @@ def get_completed_subgoal_hashes(task_plan: str) -> set[str]:
     if not task_plan:
         return set()
 
-    completed_hashes = set()
-
-    pattern = re.compile(r"^-\s*\[x\]\s*(.*)$", re.MULTILINE)
-    for match in pattern.findall(task_plan):
-        completed_hashes.add(hashlib.md5(match.strip().encode("utf-8")).hexdigest())
-
-    return completed_hashes
+    return {item.key for item in parse_plan(task_plan).top_level if item.is_done}
 
 
 def get_all_subgoal_aliases(target_hash: str, base_dir: str | Path | None = None) -> set[str]:

artemis/utils/verification.py

@@ -1,62 +0,0 @@
-# Copyright 2026 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-from pathlib import Path
-from artemis.utils.logger import get_logger
-from artemis.utils.notes import get_notes_dir
-
-logger = get_logger(__name__)
-
-
-def get_verification_chat_path(base_dir: str | Path, subgoal_hash: str) -> Path:
-    """Gets the path to the verification chat file for a given subgoal."""
-    notes_dir = get_notes_dir(base_dir)
-    return notes_dir / f"verification_chat_{subgoal_hash}.json"
-
-
-def read_verification_chat(chat_path: Path) -> list[dict]:
-    """Reads the verification chat history from the given path."""
-    if not chat_path.exists():
-        return []
-    try:
-        return json.loads(chat_path.read_text(encoding="utf-8"))
-    except Exception as e:
-        logger.error(f"Failed to load verification chat from {chat_path}: {e}")
-        return []
-
-
-def get_verification_chat_rounds(turns: list[dict]) -> tuple[int, int]:
-    """Calculates the maximum rounds for operator and checker from the turns."""
-    max_op = max(
-        [t.get("round", 0) for t in turns if t.get("role") == "operator"],
-        default=0,
-    )
-    max_chk = max(
-        [t.get("round", 0) for t in turns if t.get("role") == "checker"],
-        default=0,
-    )
-    return max_op, max_chk
-
-
-def append_verification_chat(chat_path: Path, role: str, content: str, round_num: int) -> bool:
-    """Appends a new turn to the verification chat file."""
-    turns = read_verification_chat(chat_path)
-    turns.append({"role": role, "round": round_num, "content": content})
-    try:
-        chat_path.write_text(json.dumps(turns, indent=2, ensure_ascii=False), encoding="utf-8")
-        return True
-    except Exception as e:
-        logger.error(f"Failed to save verification chat to {chat_path}: {e}")
-        return False