# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Plan-driven checkpoint scheduling, the append-only verdict ledger, and exit settlement bookkeeping. Checkpoint processing preserves three invariants: 1. **Append-only history**: the Operator's context is never rolled back or switched; verdicts influence the future (status flips forward, findings injected) but never rewrite the past. 2. **Release / verdict / wrap-up separation**: fail-open only affects the release decision — the verdict value itself (``inconclusive``) is always recorded verbatim; assert failures are never repaired but always produce a machine-readable test result. 3. **Zero side effects inside check tasks**: state mutation happens only at harvest points (``execution_check_node``) and at exit settlement. """ from __future__ import annotations import asyncio import json import re import time from dataclasses import dataclass from pathlib import Path from typing import Any, Literal from pydantic import BaseModel, Field from artemis.utils.logger import get_logger from artemis.utils.notes import append_note_content, get_note_file_path from artemis.utils.plan_grammar import ( CHECKBOX_LINE_RE, STATUS_ACTIVE, STATUS_DONE, CheckItem, PlanSnapshot, apply_finding_lines, parse_plan, subgoal_hash, ) logger = get_logger(__name__) LEDGER_FILENAME = "check_ledger.jsonl" RUN_OUTCOME_FILENAME = "run_outcome.json" #: Per-attempt transcript of the Checker's own streamed reasoning (one record #: per attempt, appended by the Checker when its run ends). Companion of the #: ledger for UI replay: the ledger books verdicts, this file keeps what the #: Checker said while reaching them. STREAMS_FILENAME = "check_streams.jsonl" #: Upper bound on the persisted text of one attempt (all segments together); #: the middle is cut out with a marker so both the opening and the conclusion #: survive. STREAM_TEXT_LIMIT = 20_000 #: Event type published on the DataEngine bus (and forwarded over IPC/SSE to the #: admin console) for every visible step of the Checker's process. CHECKER_EVENT = "checker_event" FINAL_CHECKPOINT_ID = "final" FINAL_SUBGOAL_TEXT = "Final review against the user's original goal" # --- Config accessors (all fail-safe against a missing execution_setup) -------------- def _setup(ctx) -> Any: return getattr(ctx, "execution_setup", None) def midway_checks_enabled(ctx) -> bool: setup = _setup(ctx) return bool(setup) and bool(getattr(setup, "midway_checks_enabled", False)) def final_check_enabled(ctx) -> bool: setup = _setup(ctx) return bool(setup) and bool(getattr(setup, "final_check_enabled", False)) def _setting(ctx, name: str, default): setup = _setup(ctx) value = getattr(setup, name, None) if setup else None return default if value is None else value # --- Data contracts ------------------------------------------------------------------ @dataclass(frozen=True) class EvidenceAnchor: """Anchors a checkpoint to the evidence available at completion time.""" anchor_step_id: str | None trigger_ts: float plan_text: str @dataclass(frozen=True) class PendingCheckpoint: """A queued checkpoint: enqueue happens at plan-write time, spawn happens in ``execution_check_node`` after the turn's step is recorded.""" checkpoint_id: str subgoal_text: str check_items: tuple[CheckItem, ...] plan_text: str trigger_ts: float @dataclass(frozen=True) class CheckpointRun: """One in-flight (or finished) check attempt held in ``ctx.checkpoint_tasks``.""" attempt_id: str task: asyncio.Task checkpoint: PendingCheckpoint class TestSummary(BaseModel): passed: int = 0 failed: int = 0 inconclusive: int = 0 unchecked: int = 0 failed_items: list[dict] = Field(default_factory=list) @property def total(self) -> int: return self.passed + self.failed + self.inconclusive + self.unchecked class RunOutcome(BaseModel): """Machine-readable run outcome: goal-achievement axis x test-result axis.""" task_status: Literal["completed", "partial", "blocked"] tests: TestSummary # --- Live process events -------------------------------------------------------------- def publish_checker_event(ctx, payload: dict) -> None: """Publishes one ``checker_event`` on the DataEngine bus. The event is a *projection* for observers (admin console / SSE): the append-only ledger stays the single source of truth, and publishing never influences booking or release. Best-effort: a missing engine or a failing subscriber is logged and ignored. """ engine = getattr(ctx, "data_engine", None) publish = getattr(engine, "_publish", None) if not callable(publish): return now = time.time() try: # ``timestamp`` mirrors the other bus events so the UI orders checker # blocks on the same timeline as steps and traces. publish(CHECKER_EVENT, {"ts": now, "timestamp": now, **payload}) except Exception as e: logger.debug(f"Failed to publish checker event: {e}") def announce_checker_attempt( ctx, *, attempt_id: str | None, phase: str, checkpoint_id: str, subgoal_text: str, anchor_step_id: str | None, check_items, ) -> None: """Publishes ``attempt_started`` from inside the Checker's traced scope. Called at the entry of the checker functions so the event can carry the Checker's own ``trace_id``: the UI uses it to route the Checker's streamed reasoning (``llm_stream``) and tool traces into the attempt's timeline block instead of the Operator step that happens to be current. """ if not attempt_id: return try: from artemis.data_engine.context_vars import CURRENT_TRACE_ID trace_id = CURRENT_TRACE_ID.get() except (ImportError, LookupError): trace_id = None if trace_id: _attempt_traces(ctx)[attempt_id] = str(trace_id) publish_checker_event( ctx, { "event": "attempt_started", "phase": phase, "attempt_id": attempt_id, "checkpoint_id": checkpoint_id, "subgoal_text": subgoal_text, "anchor_step_id": anchor_step_id, "trace_id": str(trace_id) if trace_id else None, "items": check_item_payloads(check_items), }, ) def _attempt_traces(ctx) -> dict: """attempt_id -> the Checker's trace id for that attempt (set at announce time). Persisted on ledger records so the UI backfill can re-attach the attempt's tool traces (which carry that trace as parent) to its block.""" traces = getattr(ctx, "checker_attempt_traces", None) if not isinstance(traces, dict): traces = {} try: ctx.checker_attempt_traces = traces except (AttributeError, TypeError, ValueError): pass return traces def attempt_trace_id(ctx, attempt_id: str | None) -> str | None: if not attempt_id: return None value = _attempt_traces(ctx).get(attempt_id) return str(value) if value else None def verdict_payloads(verdicts) -> list[dict]: """Plain-dict projection of verdict objects (pydantic or attribute bags).""" out: list[dict] = [] for v in verdicts or []: out.append( { "item_text": str(getattr(v, "item_text", "") or ""), "kind": str(getattr(v, "kind", "") or ""), "status": str(getattr(v, "status", "") or ""), "evidence": str(getattr(v, "evidence", "") or ""), "suggestion": str(getattr(v, "suggestion", "") or ""), } ) return out def check_item_payloads(check_items) -> list[dict]: return [ {"kind": ci.kind, "text": ci.text, "when": getattr(ci, "when", "on_complete")} for ci in (check_items or []) ] # --- Ledger (append-only) ------------------------------------------------------------ def ledger_path(base_dir: str | Path) -> Path: return Path(base_dir) / LEDGER_FILENAME def append_ledger_record(base_dir: str | Path, record: dict) -> None: """Appends one verdict record. Records are immutable once written; any later result for the same item can only be appended as a new record.""" record = dict(record) record.setdefault("ts", time.time()) path = ledger_path(base_dir) try: path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as f: f.write(json.dumps(record, ensure_ascii=False) + "\n") except Exception as e: logger.error(f"Failed to append check ledger record: {e}") def read_ledger(base_dir: str | Path) -> list[dict]: path = ledger_path(base_dir) if not path.exists(): return [] records: list[dict] = [] try: for line in path.read_text(encoding="utf-8").splitlines(): line = line.strip() if not line: continue try: records.append(json.loads(line)) except Exception: logger.warning(f"Skipping malformed ledger line: {line[:120]}") except Exception as e: logger.error(f"Failed to read check ledger: {e}") return records def has_ledger_records(base_dir: str | Path) -> bool: path = ledger_path(base_dir) try: return path.exists() and path.stat().st_size > 0 except Exception: return False # --- Attempt stream transcripts (append-only, one record per attempt) ---------------- def streams_path(base_dir: str | Path) -> Path: return Path(base_dir) / STREAMS_FILENAME def clamp_stream_segments( segments: list[dict], limit: int = STREAM_TEXT_LIMIT ) -> tuple[list[dict], int]: """Bounds the total text of an attempt's stream segments to ``limit`` chars. The head and the tail of the attempt (in stream order) are kept and the middle is replaced by one ``…[N chars truncated]…`` marker, attached to the segment where the cut happens. Returns ``(segments, dropped_chars)``; segments left with no text are removed. """ texts = [str(s.get("text") or "") for s in segments] total = sum(len(t) for t in texts) if total <= limit: return [dict(s) for s in segments], 0 head_end = limit // 2 tail_start = total - (limit - head_end) dropped = tail_start - head_end marker = f"\n…[{dropped} chars truncated]…\n" out: list[dict] = [] pos = 0 marker_done = False for seg, text in zip(segments, texts): start, end = pos, pos + len(text) pos = end piece = "" if start < head_end: piece += text[: max(0, min(len(text), head_end - start))] if end > tail_start: if not marker_done: piece += marker marker_done = True piece += text[max(0, tail_start - start) :] elif start < head_end < end and not marker_done: piece += marker marker_done = True if not piece: continue out.append({**seg, "text": piece}) return out, dropped def append_attempt_stream_record( base_dir: str | Path, record: dict, limit: int = STREAM_TEXT_LIMIT ) -> None: """Appends one attempt's streamed-reasoning transcript. ``record["segments"]`` is a list of ``{execution_id, role, when, text}`` dicts (``role`` is ``thought`` for the model's reasoning stream and ``answer`` for its visible text) in the order the turns started. The text is clamped to ``limit`` chars before writing; the record then carries ``truncated`` / ``dropped_chars``. """ record = dict(record) record.setdefault("ts", time.time()) segments, dropped = clamp_stream_segments(list(record.get("segments") or []), limit) record["segments"] = segments record["truncated"] = dropped > 0 record["dropped_chars"] = dropped path = streams_path(base_dir) try: path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as f: f.write(json.dumps(record, ensure_ascii=False) + "\n") except Exception as e: logger.error(f"Failed to append check stream record: {e}") def read_attempt_streams(base_dir: str | Path) -> list[dict]: path = streams_path(base_dir) if not path.exists(): return [] records: list[dict] = [] try: for line in path.read_text(encoding="utf-8").splitlines(): line = line.strip() if not line: continue try: loaded = json.loads(line) except Exception: logger.warning(f"Skipping malformed check stream line: {line[:120]}") continue if isinstance(loaded, dict): records.append(loaded) except Exception as e: logger.error(f"Failed to read check streams: {e}") return records # --- Plan helpers -------------------------------------------------------------------- def _read_plan_text(ctx) -> str: if not getattr(ctx, "data_engine", None): return "" path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") if not path.exists(): return "" try: return path.read_text(encoding="utf-8") except Exception as e: logger.error(f"Failed to read task plan: {e}") return "" def _subgoal_still_current(ctx, checkpoint_id: str) -> bool: """The anchored subgoal's text is unchanged iff an item with the same content hash still exists in the current plan.""" snapshot = parse_plan(_read_plan_text(ctx)) return any(i.is_top_level and i.key == checkpoint_id for i in snapshot.items) def revert_subgoal_status(ctx, subgoal_key: str) -> bool: """Flips the matching top-level subgoal from ``[x]`` back to ``[/]``. This is a forward-looking state change (never a rollback of history) and is written directly to the file, bypassing the wrapped note tools. """ if not getattr(ctx, "data_engine", None): return False path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") if not path.exists(): return False try: lines = path.read_text(encoding="utf-8").split("\n") changed = False for idx, line in enumerate(lines): match = CHECKBOX_LINE_RE.match(line) if not match or match.group("indent"): continue if subgoal_hash(match.group("text").strip()) == subgoal_key: if match.group("status") == STATUS_DONE: lines[idx] = re.sub( r"^(\s*-\s*\[)" + re.escape(STATUS_DONE) + r"(\])", lambda m: m.group(1) + STATUS_ACTIVE + m.group(2), line, ) changed = True break if changed: path.write_text("\n".join(lines), encoding="utf-8") return changed except Exception as e: logger.error(f"Failed to revert subgoal status: {e}") return False # --- Verify-finding persistence (four layers) ------------- # # Layer 1: the append-only ledger above (single source of truth — unchanged). # Layer 2: `- finding:` plan lines — a single-direction projection rendered by # :func:`sync_finding_lines`; harness code never reads them back. # Layer 3: the system-authored `checker-` note — full details on # first failure, then one appended section per attempt verdict. # Layer 4: single-turn ``operator_feedback`` findings (unchanged). #: Reserved note-key prefixes for system-authored checker repair logs. The #: model-side note tool wrappers reject writes to these keys. ``checker-`` is #: the prefix actually used for storage (a colon is not a legal filename #: character on Windows/NTFS); ``checker:`` is reserved alongside it so the #: conceptual spelling can never be squatted either. CHECKER_NOTE_PREFIXES = ("checker-", "checker:") _FINDING_HEADLINE_EVIDENCE_LIMIT = 160 def checker_note_key(checkpoint_id: str) -> str: """Note key of the system-authored detail/repair log for one checkpoint.""" return f"checker-{checkpoint_id}" def is_checker_note_key(key: str) -> bool: """True when a note key is reserved for system-authored checker logs.""" return str(key or "").strip().lower().startswith(CHECKER_NOTE_PREFIXES) def _finding_registry(ctx) -> dict: """checkpoint_id -> rendered headline of the unresolved verify finding. Kept as a dynamic context attribute (``ArtemisContext`` allows extras) so the registry lives exactly as long as the run. """ registry = getattr(ctx, "checker_findings", None) if not isinstance(registry, dict): registry = {} try: ctx.checker_findings = registry except (AttributeError, TypeError, ValueError): pass return registry def _render_finding_headline(checkpoint_id: str, verify_failures: list) -> str: first = verify_failures[0] evidence = str(getattr(first, "evidence", "") or "").strip() if len(evidence) > _FINDING_HEADLINE_EVIDENCE_LIMIT: evidence = evidence[: _FINDING_HEADLINE_EVIDENCE_LIMIT - 1] + "…" detail = f": {evidence}" if evidence else "" return ( f"verify failed — '{first.item_text}'{detail}" f" (details & repair log: note '{checker_note_key(checkpoint_id)}')" ) def sync_finding_lines(ctx) -> None: """Re-projects the unresolved verify findings into ``task_plan.md``. Deterministic merge-back protection for the ``- finding:`` channel: every call strips all finding lines and re-renders one per unresolved checkpoint under its subgoal, so a model deletion grows back and a resolved finding disappears. Pure projection — nothing is ever parsed back out of the plan. """ if not getattr(ctx, "data_engine", None): return path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") if not path.exists(): return try: content = path.read_text(encoding="utf-8") registry = getattr(ctx, "checker_findings", None) findings = dict(registry) if isinstance(registry, dict) else {} projected = apply_finding_lines(content, findings) if projected != content: path.write_text(projected, encoding="utf-8") except Exception as e: logger.error(f"Failed to project finding lines into task plan: {e}") def _log_checker_note(ctx, run: CheckpointRun, verdicts: list) -> None: """Appends this attempt's verdicts to the checkpoint's system note. The note starts at the FIRST failed verify (full criterion/evidence/ suggestion detail) and afterwards accumulates one section per attempt — including the eventually passing one — as a persistent repair log. It is written through the notes storage API directly (never the model tool path), stays on disk after resolution, and is discoverable via read_note / search_history under the key returned by :func:`checker_note_key`. """ try: base_dir = ctx.data_engine.base_dir cid = run.checkpoint.checkpoint_id key = checker_note_key(cid) note_path = get_note_file_path(base_dir, key) verify_failures = [v for v in verdicts if v.kind == "verify" and v.status == "failed"] if not note_path.exists() and not verify_failures: return # the log only starts once a verify criterion has failed blocks: list[str] = [] if not note_path.exists(): blocks.append( f"# Checker findings — subgoal: {run.checkpoint.subgoal_text}\n" "System-authored verification log (append-only). Each section" " records one checkpoint attempt's verdicts." ) outcome = "failed" if verify_failures else "passed" lines = [f"## attempt {run.attempt_id} — verify {outcome}"] for v in verdicts: line = f"- [{v.kind}] '{v.item_text}': {v.status} — {v.evidence}" suggestion = str(getattr(v, "suggestion", "") or "") if suggestion: line += f"\n suggestion: {suggestion}" lines.append(line) blocks.append("\n".join(lines)) append_note_content(base_dir, key, "\n\n".join(blocks)) except Exception as e: logger.error(f"Failed to append checker note for {run.attempt_id}: {e}") # --- Queueing (called from _process_plan_write; never spawns) ------------------------ def queue_checkpoints( ctx, state, after: PlanSnapshot, new_completions: list[str], plan_text: str, ) -> None: """Enqueues a pending checkpoint for every newly completed top-level subgoal that carries ``on_complete`` check items. Spawning is deliberately deferred to ``execution_check_node``: at this moment the turn's step is not yet recorded, so an anchor taken here would point at the previous turn. """ if not midway_checks_enabled(ctx): return if state is not None and getattr(state, "user_stop_requested", False): return items_by_key = {i.key: i for i in after.top_level} for text in new_completions: key = subgoal_hash(text) item = items_by_key.get(key) if item is None: continue check_items = tuple(ci for ci in after.check_items_of(item) if ci.when == "on_complete") if not check_items: continue ctx.pending_checkpoints.append( PendingCheckpoint( checkpoint_id=key, subgoal_text=text, check_items=check_items, plan_text=plan_text, trigger_ts=time.time(), ) ) logger.info(f"Queued checkpoint for completed subgoal: {text}") # --- Harvest ------------------------------------------------------------------ def _record_verdicts( ctx, run: CheckpointRun, verdicts: list, anchor_step_id: str | None, ) -> None: base_dir = ctx.data_engine.base_dir when_by_text = {(ci.kind, ci.text): ci.when for ci in run.checkpoint.check_items} for v in verdicts: append_ledger_record( base_dir, { "attempt_id": run.attempt_id, "checkpoint_id": run.checkpoint.checkpoint_id, "subgoal_text": run.checkpoint.subgoal_text, "trace_id": attempt_trace_id(ctx, run.attempt_id), "item_text": v.item_text, "kind": v.kind, "when": when_by_text.get((v.kind, v.item_text), "on_complete"), "status": v.status, "evidence": v.evidence, "suggestion": getattr(v, "suggestion", ""), "anchor_step_id": anchor_step_id, }, ) def _annotate_history_chunks(ctx, run: CheckpointRun, verdicts) -> None: """Annotate an existing history chunk: a verdict harvested after its subgoal's segment was chunk-compressed is appended to that chunk and bumps its version. Best-effort — history annotation never gates checkpoint booking.""" try: chunker = getattr(getattr(ctx, "transcript_ledger", None), "chunker", None) if chunker is None: return payload = [ { "kind": getattr(v, "kind", "check"), "item_text": getattr(v, "item_text", ""), "status": getattr(v, "status", ""), "evidence": getattr(v, "evidence", ""), } for v in verdicts ] chunker.annotate_from_checkpoint(run.checkpoint.checkpoint_id, payload) except Exception as e: logger.debug(f"History chunk annotation skipped: {e}") def _record_attempt_failure(ctx, run: CheckpointRun, status: str, evidence: str) -> None: base_dir = ctx.data_engine.base_dir for ci in run.checkpoint.check_items: append_ledger_record( base_dir, { "attempt_id": run.attempt_id, "checkpoint_id": run.checkpoint.checkpoint_id, "subgoal_text": run.checkpoint.subgoal_text, "trace_id": attempt_trace_id(ctx, run.attempt_id), "item_text": ci.text, "kind": ci.kind, "when": ci.when, "status": status, "evidence": evidence, "anchor_step_id": None, }, ) def harvest_run( ctx, state, run: CheckpointRun, *, allow_side_effects: bool, cancelled_status: str = "superseded", anchor_step_id: str | None = None, ) -> list[str]: """Books a finished (done/cancelled) attempt into the ledger and, when the verdict is still applicable, applies its execution-state side effects. Returns Operator-facing findings (injected as ``operator_feedback``). The verdict values themselves are never altered by the release decision: an errored verify is *released* (fail-open) but *recorded* inconclusive. """ task = run.task findings: list[str] = [] checkpoint = run.checkpoint def _event(status: str, verdicts_payload: list[dict], **extra) -> None: publish_checker_event( ctx, { "event": "attempt_finished", "phase": "checkpoint", "attempt_id": run.attempt_id, "checkpoint_id": checkpoint.checkpoint_id, "subgoal_text": checkpoint.subgoal_text, "trace_id": attempt_trace_id(ctx, run.attempt_id), "status": status, "verdicts": verdicts_payload, "findings": list(findings), **extra, }, ) def _failure_verdicts(status: str, evidence: str) -> list[dict]: return [ { "item_text": ci.text, "kind": ci.kind, "status": status, "evidence": evidence, "suggestion": "", } for ci in checkpoint.check_items ] if task.cancelled(): evidence = ( "attempt superseded by a newer completion of the same subgoal" if cancelled_status == "superseded" else "cancelled at exit settlement (settlement timeout)" ) _record_attempt_failure(ctx, run, cancelled_status, evidence) _event(cancelled_status, _failure_verdicts(cancelled_status, evidence)) return findings exc = task.exception() if exc is not None: status = "inconclusive" reason = ( f"check timed out after {_setting(ctx, 'checkpoint_timeout', 180.0)}s" if isinstance(exc, (asyncio.TimeoutError, TimeoutError)) else f"check raised an exception: {exc}" ) logger.warning( f"Checkpoint attempt {run.attempt_id} did not produce a verdict" f" ({reason}); releasing (fail-open) but recording inconclusive." ) _record_attempt_failure(ctx, run, status, reason) _event("error", _failure_verdicts(status, reason), error=reason) return findings report = task.result() verdicts = list(getattr(report, "verdicts", []) or []) _record_verdicts(ctx, run, verdicts, anchor_step_id) _annotate_history_chunks(ctx, run, verdicts) # Layer 3: the per-checkpoint repair log accumulates every booked attempt. _log_checker_note(ctx, run, verdicts) # Applicability gate for side effects: the verdict must belong to the # current attempt of its checkpoint AND the anchored subgoal's text must be # unchanged in the current plan. Stale/mismatched verdicts are ledger-only. applicable = allow_side_effects and _subgoal_still_current(ctx, checkpoint.checkpoint_id) user_stopped = bool(state is not None and getattr(state, "user_stop_requested", False)) verify_failures = [v for v in verdicts if v.kind == "verify" and v.status == "failed"] assert_failures = [v for v in verdicts if v.kind == "assert" and v.status == "failed"] registry = _finding_registry(ctx) registry_changed = False reverted = False if verify_failures and applicable and not user_stopped: max_repairs = int(_setting(ctx, "checkpoint_max_repairs", 2)) repairs = ctx.checkpoint_repairs.get(checkpoint.checkpoint_id, 0) if repairs < max_repairs: ctx.checkpoint_repairs[checkpoint.checkpoint_id] = repairs + 1 reverted = revert_subgoal_status(ctx, checkpoint.checkpoint_id) # Layer 2: register the standing plan headline for this unresolved # finding; the projection below renders it under the subgoal. registry[checkpoint.checkpoint_id] = _render_finding_headline( checkpoint.checkpoint_id, verify_failures ) registry_changed = True for v in verify_failures: suggestion = getattr(v, "suggestion", "") or "" findings.append( f"[verify failed] Subgoal '{checkpoint.subgoal_text}' —" f" criterion '{v.item_text}': {v.evidence}" + (f" Suggestion: {suggestion}" if suggestion else "") ) if reverted: findings.append( f"The subgoal '{checkpoint.subgoal_text}' has been set back to" " in-progress ([/]). Address the findings above, then complete" " it again." ) else: # Quota exhausted: the run is settled for this checkpoint — the # standing headline is retired (the ledger and the checker note # keep the full record). registry_changed = registry.pop(checkpoint.checkpoint_id, None) is not None logger.warning( f"Checkpoint {checkpoint.checkpoint_id} exhausted its repair" f" quota ({max_repairs}); verdict stays failed, no further" " repair loop." ) elif not verify_failures: # A booked attempt without failed verify criteria resolves (or # releases) the checkpoint: its standing headline is retired. registry_changed = registry.pop(checkpoint.checkpoint_id, None) is not None if registry_changed: sync_finding_lines(ctx) if assert_failures: policy = str(_setting(ctx, "assert_failure_policy", "continue")) if policy == "halt": ctx.assert_halt = True logger.warning( "Assert failure under 'halt' policy: latching halt flag for exit settlement." ) _event( "done", verdict_payloads(verdicts), applicable=bool(applicable), reverted=bool(reverted), repairs_used=int(ctx.checkpoint_repairs.get(checkpoint.checkpoint_id, 0)), ) return findings def harvest_finished_checkpoints(ctx, state) -> list[str]: """Non-blocking harvest: books every ``done()`` attempt, never awaits a running one. Returns Operator-facing findings.""" if not getattr(ctx, "data_engine", None): return [] findings: list[str] = [] for cid, run in list(ctx.checkpoint_tasks.items()): if not run.task.done(): continue del ctx.checkpoint_tasks[cid] findings.extend(harvest_run(ctx, state, run, allow_side_effects=True)) return findings # --- Spawn after record_step ------------------------------------------------- async def spawn_pending_checkpoints(ctx, state, anchor_step_id: str | None) -> list[str]: """Spawns queued checkpoints against the just-recorded step, honoring the concurrency cap and superseding stale attempts without losing verdicts. Returns findings harvested from superseded-but-finished attempts. """ findings: list[str] = [] if not getattr(ctx, "data_engine", None): return findings if not midway_checks_enabled(ctx): return findings if state is not None and getattr(state, "user_stop_requested", False): return findings from artemis.agents.checker.checker import run_checkpoint_check max_concurrent = int(_setting(ctx, "max_concurrent_checkpoints", 3)) timeout = float(_setting(ctx, "checkpoint_timeout", 180.0)) goal = getattr(state, "initial_goal", "") if state is not None else "" while ctx.pending_checkpoints and len(ctx.checkpoint_tasks) < max_concurrent: pending: PendingCheckpoint = ctx.pending_checkpoints.pop(0) cid = pending.checkpoint_id old_run: CheckpointRun | None = ctx.checkpoint_tasks.pop(cid, None) if old_run is not None: if old_run.task.done(): # A finished-but-unharvested attempt: book it first (verdicts, # including failures, are never dropped), ledger-only. findings.extend(harvest_run(ctx, state, old_run, allow_side_effects=False)) else: old_run.task.cancel() try: await old_run.task except (asyncio.CancelledError, Exception) as exc: logger.debug( f"Superseded checkpoint attempt {old_run.attempt_id} ended with " f"{exc!r}; harvesting anyway", exc_info=True, ) harvest_run(ctx, state, old_run, allow_side_effects=False) seq = ctx.checkpoint_attempt_seq.get(cid, 0) + 1 ctx.checkpoint_attempt_seq[cid] = seq attempt_id = f"{cid}#{seq}" anchor = EvidenceAnchor( anchor_step_id=anchor_step_id, trigger_ts=pending.trigger_ts, plan_text=pending.plan_text, ) task = asyncio.create_task( asyncio.wait_for( run_checkpoint_check( ctx, check_items=pending.check_items, anchor=anchor, goal=goal, subgoal_text=pending.subgoal_text, attempt_id=attempt_id, ), timeout=timeout, ) ) ctx.checkpoint_tasks[cid] = CheckpointRun( attempt_id=attempt_id, task=task, checkpoint=pending ) logger.info(f"Spawned checkpoint attempt {attempt_id} anchored at step {anchor_step_id}") return findings # --- Exit settlement barrier ------------------------------------------ async def settle_all_checkpoints(ctx, state) -> None: """The one intentional blocking wait of the whole flow: collects every outstanding checkpoint before exit, bounded by ``settlement_timeout``. Attempts still running at the deadline are cancelled and recorded ``unchecked`` (settlement timeout). Runs unconditionally — a started check must be booked before exit regardless of the final-check switch.""" if not getattr(ctx, "data_engine", None): return runs = list(ctx.checkpoint_tasks.values()) ctx.checkpoint_tasks.clear() if not runs: return timeout = float(_setting(ctx, "settlement_timeout", 120.0)) tasks = [r.task for r in runs] try: _done, pending = await asyncio.wait(tasks, timeout=timeout) except Exception as e: logger.error(f"Settlement wait failed: {e}") pending = {t for t in tasks if not t.done()} for t in pending: t.cancel() for t in pending: try: await t except (asyncio.CancelledError, Exception) as exc: logger.debug( f"Cancelled checkpoint attempt ended with {exc!r} at settlement; harvesting anyway", exc_info=True, ) for run in runs: # Settlement is bookkeeping, not a repair point: side effects off. harvest_run( ctx, state, run, allow_side_effects=False, cancelled_status="unchecked", ) # --- Run outcome --------------------------------------------------------------------- _RESOLVABLE = ("passed", "failed", "inconclusive") def resolve_item_status(kind: str, records: list[dict]) -> str: """Resolves one check item's final status from its ledger records. Assert: the first failure is permanent — any failed record makes the item failed. Verify: the latest substantive verdict wins (a repaired-then-passed verify is passed). No substantive record at all -> unchecked. """ substantive = [r for r in records if r.get("status") in _RESOLVABLE] if kind == "assert" and any(r.get("status") == "failed" for r in substantive): return "failed" if substantive: return substantive[-1]["status"] return "unchecked" def compute_test_summary(check_items: list[CheckItem], records: list[dict]) -> TestSummary: summary = TestSummary() by_item: dict[tuple[str, str], list[dict]] = {} for r in records: by_item.setdefault((str(r.get("kind")), str(r.get("item_text"))), []).append(r) seen: set[tuple[str, str]] = set() all_items: list[tuple[str, str]] = [] for ci in check_items: sig = (ci.kind, ci.text) if sig not in seen: seen.add(sig) all_items.append(sig) # Items that only ever existed in the ledger (e.g. their plan lines were # legitimately revised by the Planner) still count. for sig in by_item: if sig not in seen: seen.add(sig) all_items.append(sig) for kind, text in all_items: status = resolve_item_status(kind, by_item.get((kind, text), [])) if status == "passed": summary.passed += 1 elif status == "failed": summary.failed += 1 failing = [r for r in by_item.get((kind, text), []) if r.get("status") == "failed"] summary.failed_items.append( { "item_text": text, "kind": kind, "evidence": failing[-1].get("evidence", "") if failing else "", } ) elif status == "inconclusive": summary.inconclusive += 1 else: summary.unchecked += 1 return summary def compute_run_outcome( snapshot: PlanSnapshot, records: list[dict], *, verify_blocked: bool, ) -> RunOutcome: """Assembles the machine-readable run outcome. ``verify_blocked`` marks the "verify unmet and budget exhausted" exit — the goal-achievement axis then reads blocked even though the plan claims completion. Assert failures never change ``task_status``: a finished task with failed assertions is ``completed`` with ``tests.failed > 0``. """ tests = compute_test_summary(list(snapshot.all_check_items), records) if verify_blocked: task_status: Literal["completed", "partial", "blocked"] = "blocked" elif snapshot.all_top_level_done: task_status = "completed" else: task_status = "partial" return RunOutcome(task_status=task_status, tests=tests) def write_run_outcome(base_dir: str | Path, outcome: RunOutcome, extra: dict | None = None) -> None: try: payload = outcome.model_dump() if extra: payload.update(extra) path = Path(base_dir) / RUN_OUTCOME_FILENAME path.parent.mkdir(parents=True, exist_ok=True) path.write_text( json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8", ) except Exception as e: logger.error(f"Failed to write run outcome: {e}") def read_run_outcome(base_dir: str | Path) -> dict | None: path = Path(base_dir) / RUN_OUTCOME_FILENAME if not path.exists(): return None try: return json.loads(path.read_text(encoding="utf-8")) except Exception: return None