feat: harden agent verification and runtime contracts
Introduce two-gate checker settlement, explicit graph visibility and checkpoints, structured LLM reliability, centralized action specifications, and corresponding tests and design documentation.
This commit is contained in:
somew1nd committed
1 parent
ddfa420945
commit
cbe8f4f0c6
106 files changed
+11542
-4445
No files matched your search
+218
-191
@@ -14,218 +14,232 @@
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.graph import wrap_note_tool, wrap_update_note_tool
|
||||
import pytest
|
||||
|
||||
ORIGINAL_PLAN = """# Test Plan
|
||||
- [ ] Milestone 1
|
||||
- [ ] Substep 1.1
|
||||
- [ ] Milestone 2
|
||||
"""
|
||||
|
||||
|
||||
def _make_ctx(base_dir, *, disable_checker=True, disable_planner_validation=False):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(base_dir)
|
||||
ctx.execution_setup = MagicMock()
|
||||
ctx.execution_setup.disable_checker = disable_checker
|
||||
ctx.execution_setup.disable_planner_validation = disable_planner_validation
|
||||
ctx.planner_task = None
|
||||
ctx.last_validated_plan = None
|
||||
ctx.pending_checkpoints = []
|
||||
ctx.checkpoint_tasks = {}
|
||||
ctx.pending_validated_plan = None
|
||||
ctx.task_plan_content_before = None
|
||||
return ctx
|
||||
|
||||
|
||||
def _setup_plan(tmp_path, content=ORIGINAL_PLAN):
|
||||
notes_dir = tmp_path / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
task_plan_path.write_text(content, encoding="utf-8")
|
||||
return task_plan_path
|
||||
|
||||
|
||||
def _mock_tool(name):
|
||||
tool = MagicMock()
|
||||
tool.name = name
|
||||
tool.description = f"{name} tool"
|
||||
tool.ainvoke = AsyncMock(return_value="Success")
|
||||
return tool
|
||||
|
||||
|
||||
def _fake_update_invoke(task_plan_path):
|
||||
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
|
||||
content = task_plan_path.read_text(encoding="utf-8")
|
||||
updated = content.replace(args["target"], args["replacement"])
|
||||
task_plan_path.write_text(updated, encoding="utf-8")
|
||||
return "Success"
|
||||
|
||||
return fake_invoke_tool
|
||||
|
||||
|
||||
def _fake_save_invoke(task_plan_path):
|
||||
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
|
||||
task_plan_path.write_text(args["content"], encoding="utf-8")
|
||||
return "Success"
|
||||
|
||||
return fake_invoke_tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_update_note_tool_valid_status_change(tmp_path):
|
||||
base_dir = tmp_path
|
||||
notes_dir = base_dir / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
|
||||
original_plan = """# Test Plan
|
||||
- [ ] Milestone 1
|
||||
- [ ] Substep 1.1
|
||||
- [ ] Milestone 2
|
||||
"""
|
||||
task_plan_path.write_text(original_plan, encoding="utf-8")
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = str(base_dir)
|
||||
mock_ctx.execution_setup = None
|
||||
|
||||
mock_original_tool = MagicMock()
|
||||
mock_original_tool.name = "update_note"
|
||||
mock_original_tool.description = "Update a note"
|
||||
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
|
||||
|
||||
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
|
||||
content = task_plan_path.read_text(encoding="utf-8")
|
||||
updated = content.replace(args["target"], args["replacement"])
|
||||
task_plan_path.write_text(updated, encoding="utf-8")
|
||||
return "Success"
|
||||
|
||||
target = "- [ ] Milestone 1"
|
||||
replacement = "- [x] Milestone 1"
|
||||
task_plan_path = _setup_plan(tmp_path)
|
||||
ctx = _make_ctx(tmp_path)
|
||||
|
||||
with patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=fake_invoke_tool,
|
||||
side_effect=_fake_update_invoke(task_plan_path),
|
||||
):
|
||||
# Instantiate inside patch context to capture the mocked import locally
|
||||
wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool)
|
||||
wrapped_tool = wrap_update_note_tool(ctx, _mock_tool("update_note"))
|
||||
result = await wrapped_tool.ainvoke(
|
||||
{
|
||||
"key": "task_plan",
|
||||
"target": target,
|
||||
"replacement": replacement,
|
||||
"target": "- [ ] Milestone 1",
|
||||
"replacement": "- [x] Milestone 1",
|
||||
"tool_call_id": "test_call_id",
|
||||
}
|
||||
)
|
||||
|
||||
assert result == "Success"
|
||||
updated_content = task_plan_path.read_text(encoding="utf-8")
|
||||
assert "- [x] Milestone 1" in updated_content
|
||||
assert "- [ ] Milestone 2" in updated_content
|
||||
assert result == "Success"
|
||||
updated_content = task_plan_path.read_text(encoding="utf-8")
|
||||
assert "- [x] Milestone 1" in updated_content
|
||||
assert "- [ ] Milestone 2" in updated_content
|
||||
# Status flips never disturb the milestone texts: no validation spawned
|
||||
assert ctx.planner_task is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_update_note_tool_invalid_milestone_text_change(tmp_path):
|
||||
base_dir = tmp_path
|
||||
notes_dir = base_dir / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
async def test_wrap_update_note_tool_intentional_rename_triggers_validation(tmp_path):
|
||||
task_plan_path = _setup_plan(tmp_path)
|
||||
ctx = _make_ctx(tmp_path)
|
||||
|
||||
original_plan = """# Test Plan
|
||||
- [ ] Milestone 1
|
||||
- [ ] Substep 1.1
|
||||
- [ ] Milestone 2
|
||||
"""
|
||||
task_plan_path.write_text(original_plan, encoding="utf-8")
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = str(base_dir)
|
||||
mock_ctx.execution_setup = None
|
||||
|
||||
mock_original_tool = MagicMock()
|
||||
mock_original_tool.name = "update_note"
|
||||
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
|
||||
|
||||
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
|
||||
content = task_plan_path.read_text(encoding="utf-8")
|
||||
updated = content.replace(args["target"], args["replacement"])
|
||||
task_plan_path.write_text(updated, encoding="utf-8")
|
||||
return "Success"
|
||||
|
||||
target = "- [ ] Milestone 1"
|
||||
replacement = "- [ ] Milestone Renamed"
|
||||
|
||||
mock_validation = AsyncMock(return_value={"status": "failed", "feedback": "rejected"})
|
||||
mock_validation = AsyncMock(return_value={"status": "success", "feedback": ""})
|
||||
with (
|
||||
patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=fake_invoke_tool,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.planner.planner.run_async_planner_validation",
|
||||
mock_validation,
|
||||
side_effect=_fake_update_invoke(task_plan_path),
|
||||
),
|
||||
patch("artemis.graph.graph.run_async_planner_validation", mock_validation),
|
||||
):
|
||||
wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool)
|
||||
wrapped_tool = wrap_update_note_tool(ctx, _mock_tool("update_note"))
|
||||
result = await wrapped_tool.ainvoke(
|
||||
{
|
||||
"key": "task_plan",
|
||||
"target": target,
|
||||
"replacement": replacement,
|
||||
"target": "- [ ] Milestone 1",
|
||||
"replacement": "- [ ] Milestone Renamed",
|
||||
"tool_call_id": "test_call_id",
|
||||
}
|
||||
)
|
||||
|
||||
assert result == "Success"
|
||||
assert mock_ctx.planner_task is not None
|
||||
# Declared update_note edit is applied, but ratchet validation fires
|
||||
assert ctx.planner_task is not None
|
||||
await ctx.planner_task
|
||||
|
||||
assert "Milestone Renamed" in task_plan_path.read_text(encoding="utf-8")
|
||||
mock_validation.assert_called_once()
|
||||
# Validation compares against the ratchet baseline (the original plan)
|
||||
assert mock_validation.call_args.args[2] == ORIGINAL_PLAN
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ratchet_baseline_survives_consecutive_small_edits(tmp_path):
|
||||
"""Salami-slicing: every small rename is judged against the ORIGINAL
|
||||
validated baseline, not the immediately preceding write."""
|
||||
import asyncio
|
||||
|
||||
task_plan_path = _setup_plan(tmp_path)
|
||||
ctx = _make_ctx(tmp_path)
|
||||
|
||||
gate = asyncio.Event()
|
||||
|
||||
async def blocking_validation(*args, **kwargs):
|
||||
await gate.wait()
|
||||
return {"status": "success", "feedback": ""}
|
||||
|
||||
mock_validation = AsyncMock(side_effect=blocking_validation)
|
||||
with (
|
||||
patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=_fake_update_invoke(task_plan_path),
|
||||
),
|
||||
patch("artemis.graph.graph.run_async_planner_validation", mock_validation),
|
||||
):
|
||||
wrapped_tool = wrap_update_note_tool(ctx, _mock_tool("update_note"))
|
||||
await wrapped_tool.ainvoke(
|
||||
{
|
||||
"key": "task_plan",
|
||||
"target": "- [ ] Milestone 1",
|
||||
"replacement": "- [ ] Milestone 1a",
|
||||
"tool_call_id": "call_1",
|
||||
}
|
||||
)
|
||||
first_task = ctx.planner_task
|
||||
assert first_task is not None
|
||||
|
||||
await wrapped_tool.ainvoke(
|
||||
{
|
||||
"key": "task_plan",
|
||||
"target": "- [ ] Milestone 1a",
|
||||
"replacement": "- [ ] Milestone 1b",
|
||||
"tool_call_id": "call_2",
|
||||
}
|
||||
)
|
||||
second_task = ctx.planner_task
|
||||
assert second_task is not first_task
|
||||
|
||||
# The superseded in-flight validation is cancelled
|
||||
gate.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await first_task
|
||||
await second_task
|
||||
|
||||
# Both validation runs were anchored to the original baseline
|
||||
for call in mock_validation.call_args_list:
|
||||
assert call.args[2] == ORIGINAL_PLAN
|
||||
# Rollback target preserves the last baseline-consistent content
|
||||
assert ctx.task_plan_content_before == ORIGINAL_PLAN
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_update_note_tool_allow_subgoal_changes(tmp_path):
|
||||
base_dir = tmp_path
|
||||
notes_dir = base_dir / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
|
||||
original_plan = """# Test Plan
|
||||
- [ ] Milestone 1
|
||||
- [ ] Substep 1.1
|
||||
- [ ] Milestone 2
|
||||
"""
|
||||
task_plan_path.write_text(original_plan, encoding="utf-8")
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = str(base_dir)
|
||||
mock_ctx.execution_setup = None
|
||||
|
||||
mock_original_tool = MagicMock()
|
||||
mock_original_tool.name = "update_note"
|
||||
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
|
||||
|
||||
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
|
||||
content = task_plan_path.read_text(encoding="utf-8")
|
||||
updated = content.replace(args["target"], args["replacement"])
|
||||
task_plan_path.write_text(updated, encoding="utf-8")
|
||||
return "Success"
|
||||
|
||||
target = " - [ ] Substep 1.1"
|
||||
replacement = " - [x] Substep 1.1 Completed"
|
||||
task_plan_path = _setup_plan(tmp_path)
|
||||
ctx = _make_ctx(tmp_path)
|
||||
|
||||
with patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=fake_invoke_tool,
|
||||
side_effect=_fake_update_invoke(task_plan_path),
|
||||
):
|
||||
wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool)
|
||||
wrapped_tool = wrap_update_note_tool(ctx, _mock_tool("update_note"))
|
||||
result = await wrapped_tool.ainvoke(
|
||||
{
|
||||
"key": "task_plan",
|
||||
"target": target,
|
||||
"replacement": replacement,
|
||||
"target": " - [ ] Substep 1.1",
|
||||
"replacement": " - [x] Substep 1.1 Completed",
|
||||
"tool_call_id": "test_call_id",
|
||||
}
|
||||
)
|
||||
|
||||
assert result == "Success"
|
||||
updated_content = task_plan_path.read_text(encoding="utf-8")
|
||||
assert "Substep 1.1 Completed" in updated_content
|
||||
assert "- [ ] Milestone 1" in updated_content
|
||||
assert result == "Success"
|
||||
updated_content = task_plan_path.read_text(encoding="utf-8")
|
||||
assert "Substep 1.1 Completed" in updated_content
|
||||
assert "- [ ] Milestone 1" in updated_content
|
||||
# Sub-task edits never touch the top-level milestone texts
|
||||
assert ctx.planner_task is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_note_tool_invalid_rewrite(tmp_path):
|
||||
base_dir = tmp_path
|
||||
notes_dir = base_dir / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
|
||||
original_plan = """# Test Plan
|
||||
- [ ] Milestone 1
|
||||
- [ ] Substep 1.1
|
||||
- [ ] Milestone 2
|
||||
"""
|
||||
task_plan_path.write_text(original_plan, encoding="utf-8")
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = str(base_dir)
|
||||
mock_ctx.execution_setup = None
|
||||
|
||||
mock_original_tool = MagicMock()
|
||||
mock_original_tool.name = "save_note"
|
||||
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
|
||||
|
||||
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
|
||||
task_plan_path.write_text(args["content"], encoding="utf-8")
|
||||
return "Success"
|
||||
async def test_wrap_note_tool_full_rewrite_hand_slip_rejected(tmp_path):
|
||||
"""A save_note full rewrite that rewords status-unchanged milestones is the
|
||||
hand-slip signature: rejected and rolled back, no validation spawned."""
|
||||
task_plan_path = _setup_plan(tmp_path)
|
||||
ctx = _make_ctx(tmp_path)
|
||||
|
||||
corrupted_plan = """# Test Plan
|
||||
- [ ] Milestone 1 (Altered)
|
||||
- [ ] New Milestone Added
|
||||
"""
|
||||
|
||||
mock_validation = AsyncMock(return_value={"status": "failed", "feedback": "rejected"})
|
||||
with (
|
||||
patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=fake_invoke_tool,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.planner.planner.run_async_planner_validation",
|
||||
mock_validation,
|
||||
),
|
||||
with patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=_fake_save_invoke(task_plan_path),
|
||||
):
|
||||
wrapped_tool = wrap_note_tool(mock_ctx, mock_original_tool)
|
||||
wrapped_tool = wrap_note_tool(ctx, _mock_tool("save_note"))
|
||||
result = await wrapped_tool.ainvoke(
|
||||
{
|
||||
"key": "task_plan",
|
||||
@@ -234,53 +248,30 @@ async def test_wrap_note_tool_invalid_rewrite(tmp_path):
|
||||
}
|
||||
)
|
||||
|
||||
assert result == "Success"
|
||||
assert mock_ctx.planner_task is not None
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert task_plan_path.read_text(encoding="utf-8") == ORIGINAL_PLAN
|
||||
assert ctx.planner_task is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_note_tool_valid_lexical_drift(tmp_path):
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
|
||||
base_dir = tmp_path
|
||||
notes_dir = base_dir / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
|
||||
async def test_wrap_note_tool_lexical_drift_rejected(tmp_path):
|
||||
"""Even a tiny same-status rewording in a full rewrite bounces back with
|
||||
guidance — this is exactly the drift that similarity thresholds missed."""
|
||||
original_plan = """# Test Plan
|
||||
- [ ] Open the Chrome browser and navigate to youtube.com
|
||||
"""
|
||||
task_plan_path.write_text(original_plan, encoding="utf-8")
|
||||
task_plan_path = _setup_plan(tmp_path, original_plan)
|
||||
ctx = _make_ctx(tmp_path)
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = str(base_dir)
|
||||
mock_ctx.execution_setup = None
|
||||
|
||||
# Create mock Command result
|
||||
tool_msg = ToolMessage(tool_call_id="test_call_id", content="Success outcome", status="success")
|
||||
mock_command = Command(update={VALIDATOR_MESSAGES_KEY: [tool_msg]})
|
||||
|
||||
mock_original_tool = MagicMock()
|
||||
mock_original_tool.name = "save_note"
|
||||
mock_original_tool.ainvoke = AsyncMock(return_value=mock_command)
|
||||
|
||||
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
|
||||
task_plan_path.write_text(args["content"], encoding="utf-8")
|
||||
return mock_command
|
||||
|
||||
# Minor wording change that is semantically valid (similarity ~92%)
|
||||
minor_drift_plan = """# Test Plan
|
||||
- [ ] Open Chrome browser and navigate to youtube.com
|
||||
"""
|
||||
|
||||
with patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=fake_invoke_tool,
|
||||
side_effect=_fake_save_invoke(task_plan_path),
|
||||
):
|
||||
wrapped_tool = wrap_note_tool(mock_ctx, mock_original_tool)
|
||||
wrapped_tool = wrap_note_tool(ctx, _mock_tool("save_note"))
|
||||
result = await wrapped_tool.ainvoke(
|
||||
{
|
||||
"key": "task_plan",
|
||||
@@ -289,9 +280,45 @@ async def test_wrap_note_tool_valid_lexical_drift(tmp_path):
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert not hasattr(mock_ctx, "planner_task") or mock_ctx.planner_task is None
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert "update_note" in str(result.content)
|
||||
assert task_plan_path.read_text(encoding="utf-8") == original_plan
|
||||
assert ctx.planner_task is None
|
||||
|
||||
# Verify that the drift plan is actually saved
|
||||
saved_content = task_plan_path.read_text(encoding="utf-8")
|
||||
assert "Open Chrome browser" in saved_content
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrap_note_tool_structural_replan_triggers_validation(tmp_path):
|
||||
"""Adding a milestone via full rewrite is a declared replan: applied, then
|
||||
judged asynchronously by the planner against the ratchet baseline."""
|
||||
task_plan_path = _setup_plan(tmp_path)
|
||||
ctx = _make_ctx(tmp_path)
|
||||
|
||||
extended_plan = """# Test Plan
|
||||
- [ ] Milestone 1
|
||||
- [ ] Substep 1.1
|
||||
- [ ] Milestone 2
|
||||
- [ ] Milestone 3 (new phase)
|
||||
"""
|
||||
|
||||
mock_validation = AsyncMock(return_value={"status": "success", "feedback": ""})
|
||||
with (
|
||||
patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=_fake_save_invoke(task_plan_path),
|
||||
),
|
||||
patch("artemis.graph.graph.run_async_planner_validation", mock_validation),
|
||||
):
|
||||
wrapped_tool = wrap_note_tool(ctx, _mock_tool("save_note"))
|
||||
result = await wrapped_tool.ainvoke(
|
||||
{
|
||||
"key": "task_plan",
|
||||
"content": extended_plan,
|
||||
"tool_call_id": "test_call_id",
|
||||
}
|
||||
)
|
||||
assert result == "Success"
|
||||
assert ctx.planner_task is not None
|
||||
await ctx.planner_task
|
||||
|
||||
assert "Milestone 3 (new phase)" in task_plan_path.read_text(encoding="utf-8")
|
||||
mock_validation.assert_called_once()
|
||||
Reference in new issue
Block a user