feat: harden agent verification and runtime contracts

Introduce two-gate checker settlement, explicit graph visibility and checkpoints, structured LLM reliability, centralized action specifications, and corresponding tests and design documentation.
This commit is contained in:
somew1nd committed 2026-08-31 20:42:24 -07:00
1 parent ddfa420945
commit cbe8f4f0c6
106 files changed
+11542 -4445

No files matched your search

+52 -11
View File
@@ -26,6 +26,7 @@ from artemis.config import OutputConfig
from artemis.context import ArtemisContext
from artemis.data_engine.trace import trace
from artemis.graph.state import State
from artemis.llm.structured import ParseFailure, parse_structured
from artemis.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback
from artemis.tools.scratchpad import (
get_append_note_tool_pure,
@@ -141,6 +142,47 @@ async def outputter(
)
human_message_content = [{"type": "text", "text": human_message}]
# Verification material (only present when the run declared check items):
# the full append-only verdict sequence plus the four-state test summary.
if ctx.data_engine:
try:
from artemis.graph.checkpoints import read_ledger, read_run_outcome
ledger_records = read_ledger(ctx.data_engine.base_dir)
run_outcome = read_run_outcome(ctx.data_engine.base_dir) or getattr(
graph_output, "run_outcome", None
)
if ledger_records or run_outcome:
lines = ["--- Verification Verdict Ledger (append-only) ---"]
for r in ledger_records:
lines.append(
f"- attempt {r.get('attempt_id')} [{r.get('kind')}]"
f" '{r.get('item_text')}' -> {r.get('status')}:"
f" {r.get('evidence', '')}"
)
if run_outcome:
tests = run_outcome.get("tests") or {}
lines.append(
"--- Test Summary ---\n"
f"task_status={run_outcome.get('task_status')},"
f" passed={tests.get('passed', 0)},"
f" failed={tests.get('failed', 0)},"
f" inconclusive={tests.get('inconclusive', 0)},"
f" unchecked={tests.get('unchecked', 0)}"
)
lines.append(
"Report every declared check item with its final state"
" (passed / failed / inconclusive / unchecked) and include"
" the complete verdict sequence above. A failed assertion"
" is a test result to report verbatim, not something to"
" explain away. Items reported unchecked were never"
" judged — never present them as passed."
)
human_message_content.append({"type": "text", "text": "\n".join(lines)})
except Exception as e:
logger.error(f"Failed to attach verdict ledger to outputter: {e}")
if screenshot_b64:
human_message_content.append(
{
@@ -306,16 +348,15 @@ async def outputter(
# Fallback to old behavior if it didn't match the template
if isinstance(raw_answer, str):
raw_answer_stripped = raw_answer.strip()
if raw_answer_stripped.startswith("{") and raw_answer_stripped.endswith("}"):
try:
return json.loads(raw_answer_stripped)
except Exception:
pass
elif raw_answer_stripped.startswith("```json") and raw_answer_stripped.endswith("```"):
try:
content = raw_answer_stripped[7:-3].strip()
return json.loads(content)
except Exception:
pass
if raw_answer_stripped.startswith(("{", "[", "```")):
parsed = parse_structured(raw_answer_stripped)
if not isinstance(parsed, ParseFailure):
return parsed
# The raw text is a legitimate final answer; the miss is only
# noteworthy because the answer *looked* like JSON.
logger.warning(
"Outputter answer looked like JSON but could not be parsed"
f" ({parsed.error}); returning it as plain text."
)
return raw_answer