704 lines
27 KiB
Python
704 lines
27 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Exit settlement tests: gate routing, settlement/final-review decoupling,
|
|
settlement timeout, when-semantics at the exit, run outcome, and the four
|
|
switch-combination matrix."""
|
|
|
|
import asyncio
|
|
import json
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from artemis.agents.checker.checker import CheckReport, CheckVerdict
|
|
from artemis.context import ArtemisContext, ExecutionSetup
|
|
from artemis.graph.checkpoints import (
|
|
CheckpointRun,
|
|
PendingCheckpoint,
|
|
append_ledger_record,
|
|
compute_run_outcome,
|
|
read_ledger,
|
|
resolve_item_status,
|
|
settle_all_checkpoints,
|
|
)
|
|
from artemis.graph.graph import (
|
|
convergence_gate,
|
|
exit_settlement_gate,
|
|
exit_settlement_node,
|
|
)
|
|
from artemis.graph.state import State
|
|
from artemis.sdk.agent import attach_test_summary, resolve_trace_suffix
|
|
from artemis.utils.plan_grammar import CheckItem, parse_plan, subgoal_hash
|
|
|
|
DONE_PLAN_WITH_CHECKS = (
|
|
"- [x] Create alarm\n"
|
|
" - verify: alarm exists\n"
|
|
" - assert: toast appeared\n"
|
|
"- assert@end: no crash dialog\n"
|
|
)
|
|
|
|
DONE_PLAN_NO_CHECKS = "- [x] Create alarm\n"
|
|
|
|
|
|
@pytest.fixture
|
|
def exploration_exit(tmp_path, monkeypatch):
|
|
from apps.admin_console.services.automation_store import AutomationStore
|
|
from artemis.config.exploration import ExplorationConfig
|
|
from artemis.exploration.store import ExplorationStore
|
|
from artemis.runtime import task_control
|
|
|
|
repo = ExplorationStore(AutomationStore(tmp_path / "history.db"))
|
|
session = repo.create("逐步探索，听用户指导", ExplorationConfig(), {}, "create")
|
|
run = repo.reserve_run(session["id"], "run", kind="exploration", goal="探索", task_config={})
|
|
repo.set_run_state(run["id"], "running")
|
|
monkeypatch.setattr("artemis.exploration.runtime.explorations", repo)
|
|
monkeypatch.setattr("artemis.exploration.store.explorations", repo)
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
ctx = _make_ctx(tmp_path)
|
|
ctx.exploration_id = session["id"]
|
|
ctx.exploration_run_id = run["id"]
|
|
ctx.exploration_guidance_count = 0
|
|
ctx.on_execution_review = AsyncMock()
|
|
ctx.on_execution_resume = AsyncMock()
|
|
_write_plan(tmp_path, DONE_PLAN_NO_CHECKS)
|
|
return ctx, repo, session, run
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_exploration_question_prevents_completed_plan_from_starting_final_review(
|
|
exploration_exit,
|
|
):
|
|
ctx, repo, session, run = exploration_exit
|
|
repo.ask(run["id"], "下一步做什么？", "返回主页", "ask")
|
|
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as reviewer:
|
|
assert convergence_gate(_make_state(), ctx) == "continue"
|
|
result = await exit_settlement_node(_make_state(execution_report="本次调查报告"), ctx)
|
|
assert result == {"exit_settlement_route": "continue"}
|
|
reviewer.assert_not_called()
|
|
assert repo.get(session["id"])["pending_question"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_exploration_late_return_home_guidance_invalidates_final_review(exploration_exit):
|
|
ctx, repo, session, run = exploration_exit
|
|
|
|
async def review(*args, **kwargs):
|
|
repo.add_message(
|
|
session["id"], "先回主页，然后结束", "home", metadata={"run_id": run["id"]}
|
|
)
|
|
return CheckReport(verdicts=[], unmet_subgoals=[])
|
|
|
|
with patch("artemis.graph.graph.run_final_check", new=review):
|
|
result = await exit_settlement_node(_make_state(execution_report="本次调查报告"), ctx)
|
|
assert result == {"exit_settlement_route": "continue"}
|
|
assert repo.run(run["id"])["status"] == "running"
|
|
assert repo.run(run["id"])["user_guidance_count"] == 1
|
|
assert not (ctx.data_engine.base_dir / "run_outcome.json").exists()
|
|
ctx.on_execution_review.assert_awaited_once()
|
|
ctx.on_execution_resume.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_review_stops_capture_before_waiting_and_seals_exploration(exploration_exit):
|
|
ctx, repo, session, run = exploration_exit
|
|
|
|
async def review(*args, **kwargs):
|
|
ctx.on_execution_review.assert_awaited_once()
|
|
return CheckReport(verdicts=[], unmet_subgoals=[])
|
|
|
|
with patch("artemis.graph.graph.run_final_check", new=review):
|
|
result = await exit_settlement_node(_make_state(execution_report="本次调查报告"), ctx)
|
|
assert result["exit_settlement_route"] == "end"
|
|
assert repo.get(session["id"])["status"] == "finalizing"
|
|
ctx.on_execution_resume.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_exploration_guidance_can_interrupt_shared_final_review_budget(exploration_exit):
|
|
from types import SimpleNamespace
|
|
|
|
ctx, _, _, _ = exploration_exit
|
|
started = asyncio.Event()
|
|
closed = asyncio.Event()
|
|
|
|
async def review(*args, **kwargs):
|
|
started.set()
|
|
try:
|
|
await asyncio.Event().wait()
|
|
finally:
|
|
closed.set()
|
|
|
|
async def interrupt(operation, **kwargs):
|
|
task = asyncio.create_task(operation)
|
|
await started.wait()
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
return True, None
|
|
|
|
ctx.task_control = SimpleNamespace(
|
|
run_interruptibly=interrupt, directory=ctx.data_engine.base_dir / "control"
|
|
)
|
|
with patch("artemis.graph.graph.run_final_check", new=review):
|
|
result = await exit_settlement_node(_make_state(execution_report="调查报告"), ctx)
|
|
assert result == {"exit_settlement_route": "continue"}
|
|
assert closed.is_set()
|
|
assert ctx.final_check_attempts == 0
|
|
assert not (ctx.data_engine.base_dir / "run_outcome.json").exists()
|
|
diagnostics = [
|
|
call.kwargs["payload"]
|
|
for call in ctx.data_engine.record_trace.call_args_list
|
|
if call.kwargs.get("type") == "final_review_diagnostic"
|
|
]
|
|
assert diagnostics[-1]["status"] == "cancelled"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_completed_exploration_waits_for_report_before_review(exploration_exit):
|
|
ctx, repo, session, run = exploration_exit
|
|
state = State.initial("保存并交付完整报告")
|
|
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as review:
|
|
assert convergence_gate(state, ctx) == "continue"
|
|
assert await exit_settlement_node(state, ctx) == {"exit_settlement_route": "continue"}
|
|
review.assert_not_called()
|
|
ctx.on_execution_review.assert_not_called()
|
|
assert repo.run(run["id"])["status"] == "running"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("verdict_status", ["passed", "failed", "inconclusive"])
|
|
async def test_report_is_saved_and_fully_delivered_before_independent_review(
|
|
exploration_exit, verdict_status
|
|
):
|
|
from artemis.runtime.case_result import CaseTestResult
|
|
|
|
ctx, repo, session, run = exploration_exit
|
|
body = "# 本次调查报告\n" + "实际证据及未测范围。" * 2400
|
|
state = State(initial_goal="交付完整报告", execution_report=body)
|
|
_write_plan(ctx.data_engine.base_dir, "- [x] 调查并交付报告\n - assert: 实际页面结果\n")
|
|
|
|
async def review(*args, **kwargs):
|
|
directory = ctx.data_engine.base_dir
|
|
assert (directory / "notes" / "output.md").read_text(encoding="utf-8") == body
|
|
assert json.loads((directory / "result.json").read_text(encoding="utf-8")) == {
|
|
"result": body
|
|
}
|
|
delivered = [m for m in repo.messages(session["id"]) if m.get("kind") == "execution_report"]
|
|
assert len(delivered) == 3
|
|
assert "".join(m["text"] for m in delivered) == body
|
|
assert kwargs["execution_report"] == body
|
|
assert kwargs["report_delivery"]["exploration_message_sequences"] == [
|
|
m["sequence"] for m in delivered
|
|
]
|
|
return CheckReport(
|
|
verdicts=[
|
|
CheckVerdict(
|
|
item_text="实际页面结果",
|
|
kind="assert",
|
|
status=verdict_status,
|
|
evidence="步骤 1",
|
|
)
|
|
],
|
|
test_result=CaseTestResult(
|
|
status=verdict_status, reason="按本次实际证据裁定", evidence=["步骤 1 实际页面"]
|
|
),
|
|
)
|
|
|
|
with patch("artemis.graph.graph.run_final_check", new=review):
|
|
result = await exit_settlement_node(state, ctx)
|
|
assert result["run_outcome"]["test_result"]["status"] == verdict_status
|
|
assert repo.run(run["id"])["status"] == "finalizing"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_report_delivery_is_idempotent_with_review_disabled(exploration_exit):
|
|
ctx, repo, session, run = exploration_exit
|
|
ctx.execution_setup.disable_checker = True
|
|
state = State(initial_goal="调查", execution_report="完整报告正文")
|
|
assert convergence_gate(state, ctx) == "exit_settlement"
|
|
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as review:
|
|
await exit_settlement_node(state, ctx)
|
|
await exit_settlement_node(state, ctx)
|
|
review.assert_not_called()
|
|
delivered = [m for m in repo.messages(session["id"]) if m.get("kind") == "execution_report"]
|
|
assert len(delivered) == 1
|
|
assert delivered[0]["text"] == "完整报告正文"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_report_write_failure_cannot_release_as_completed(exploration_exit, monkeypatch):
|
|
from pathlib import Path
|
|
|
|
ctx, repo, session, run = exploration_exit
|
|
state = State(initial_goal="调查", execution_report="完整报告正文")
|
|
monkeypatch.setattr(Path, "replace", MagicMock(side_effect=OSError("disk full")))
|
|
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as review:
|
|
with pytest.raises(OSError, match="disk full"):
|
|
await exit_settlement_node(state, ctx)
|
|
review.assert_not_called()
|
|
assert repo.run(run["id"])["status"] == "running"
|
|
assert not list(ctx.data_engine.base_dir.rglob("*.tmp"))
|
|
|
|
|
|
def _make_ctx(tmp_path, **setup_kwargs):
|
|
setup_kwargs.setdefault("disable_checker", False)
|
|
ctx = MagicMock(spec=ArtemisContext)
|
|
ctx.execution_setup = ExecutionSetup(**setup_kwargs)
|
|
ctx.data_engine = MagicMock()
|
|
ctx.data_engine.base_dir = tmp_path
|
|
ctx.pending_checkpoints = []
|
|
ctx.checkpoint_tasks = {}
|
|
ctx.checkpoint_attempt_seq = {}
|
|
ctx.checkpoint_repairs = {}
|
|
ctx.assert_halt = False
|
|
ctx.final_check_attempts = 0
|
|
return ctx
|
|
|
|
|
|
def _make_state(**overrides):
|
|
state = MagicMock(spec=State)
|
|
state.initial_goal = "the user goal"
|
|
state.user_stop_requested = False
|
|
state.checker_success = True
|
|
for k, v in overrides.items():
|
|
setattr(state, k, v)
|
|
return state
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pro_final_review_has_its_own_budget_and_reports_exhaustion(tmp_path, monkeypatch):
|
|
from artemis.llm import final_review
|
|
from artemis.runtime.case_result import CaseTestResult
|
|
|
|
_write_plan(tmp_path, DONE_PLAN_NO_CHECKS)
|
|
ctx = _make_ctx(tmp_path, checkpoint_timeout=0.001)
|
|
|
|
async def slow_review(*args, **kwargs):
|
|
await asyncio.sleep(0.02)
|
|
return CheckReport(
|
|
test_result=CaseTestResult(status="passed", reason="证据齐全", evidence=["步骤 1 截图"])
|
|
)
|
|
|
|
with patch("artemis.graph.graph.run_final_check", new=slow_review):
|
|
result = await exit_settlement_node(_make_state(), ctx)
|
|
assert result["run_outcome"]["test_result"]["status"] == "passed"
|
|
|
|
monkeypatch.setattr(final_review, "MAX_FINAL_REVIEW_SECONDS", 0.001)
|
|
with patch("artemis.graph.graph.run_final_check", new=slow_review):
|
|
result = await exit_settlement_node(_make_state(), ctx)
|
|
assert result["exit_settlement_route"] == "end"
|
|
assert result["run_outcome"]["test_result"]["status"] == "inconclusive"
|
|
assert "总预算已用尽" in result["run_outcome"]["test_result"]["reason"]
|
|
|
|
|
|
def _write_plan(tmp_path, content):
|
|
notes = tmp_path / "notes"
|
|
notes.mkdir(parents=True, exist_ok=True)
|
|
(notes / "task_plan.md").write_text(content, encoding="utf-8")
|
|
|
|
|
|
def _pending_checkpoint(text="Create alarm"):
|
|
return PendingCheckpoint(
|
|
checkpoint_id=subgoal_hash(text),
|
|
subgoal_text=text,
|
|
check_items=(
|
|
CheckItem(
|
|
kind="verify",
|
|
when="on_complete",
|
|
text="alarm exists",
|
|
parent_key=subgoal_hash(text),
|
|
),
|
|
),
|
|
plan_text=f"- [x] {text}\n",
|
|
trigger_ts=0.0,
|
|
)
|
|
|
|
|
|
def _hanging_run(cid="cid"):
|
|
async def hang():
|
|
await asyncio.sleep(3600)
|
|
|
|
task = asyncio.create_task(hang())
|
|
return CheckpointRun(attempt_id=f"{cid}#1", task=task, checkpoint=_pending_checkpoint())
|
|
|
|
|
|
# --- §8.8: gate routing — settlement decoupled from final review ---------------------
|
|
|
|
|
|
def test_gate_routes_settlement_when_tasks_pending_even_with_final_off(tmp_path):
|
|
_write_plan(tmp_path, DONE_PLAN_NO_CHECKS)
|
|
ctx = _make_ctx(tmp_path, disable_final_check=True)
|
|
ctx.checkpoint_tasks["cid"] = MagicMock()
|
|
assert convergence_gate(_make_state(), ctx) == "exit_settlement"
|
|
|
|
|
|
def test_gate_routes_empty_plan_to_final_review_when_enabled(tmp_path):
|
|
_write_plan(tmp_path, "just prose, no checkboxes\n")
|
|
ctx = _make_ctx(tmp_path) # final on (enabled master, default gates)
|
|
assert convergence_gate(_make_state(), ctx) == "exit_settlement"
|
|
|
|
|
|
def test_gate_ends_directly_when_nothing_check_related(tmp_path):
|
|
_write_plan(tmp_path, DONE_PLAN_NO_CHECKS)
|
|
ctx = _make_ctx(tmp_path, disable_checker=True)
|
|
assert convergence_gate(_make_state(), ctx) == "end"
|
|
|
|
|
|
def test_gate_check_items_force_settlement_even_all_switches_off(tmp_path):
|
|
"""Combination row 1 (off/off): declared items still get settled (booked as
|
|
unchecked) instead of silently vanishing."""
|
|
_write_plan(tmp_path, DONE_PLAN_WITH_CHECKS)
|
|
ctx = _make_ctx(tmp_path, disable_checker=True)
|
|
assert convergence_gate(_make_state(), ctx) == "exit_settlement"
|
|
|
|
|
|
def test_gate_incomplete_plan_continues(tmp_path):
|
|
_write_plan(tmp_path, "- [/] Create alarm\n - verify: alarm exists\n")
|
|
ctx = _make_ctx(tmp_path)
|
|
assert convergence_gate(_make_state(), ctx) == "continue"
|
|
|
|
|
|
def test_gate_unevaluated_plan_validation_is_not_a_failure(tmp_path):
|
|
"""First pass after the Planner node: ``checker_success`` is still None
|
|
(never evaluated). That must not short-circuit the gate — a fully done
|
|
plan still routes to the exit, and no "validation failed" log fires."""
|
|
_write_plan(tmp_path, DONE_PLAN_NO_CHECKS)
|
|
ctx = _make_ctx(tmp_path)
|
|
with patch("artemis.graph.graph.logger") as mock_logger:
|
|
assert convergence_gate(_make_state(checker_success=None), ctx) == "exit_settlement"
|
|
logged = " ".join(str(c.args[0]) for c in mock_logger.info.call_args_list if c.args)
|
|
assert "Plan validation failed" not in logged
|
|
|
|
|
|
def test_gate_explicit_validation_failure_still_continues(tmp_path):
|
|
_write_plan(tmp_path, DONE_PLAN_NO_CHECKS)
|
|
ctx = _make_ctx(tmp_path)
|
|
with patch("artemis.graph.graph.logger") as mock_logger:
|
|
assert convergence_gate(_make_state(checker_success=False), ctx) == "continue"
|
|
logged = " ".join(str(c.args[0]) for c in mock_logger.info.call_args_list if c.args)
|
|
assert "Plan validation failed" in logged
|
|
|
|
|
|
def test_gate_assert_halt_terminates_mid_plan(tmp_path):
|
|
_write_plan(tmp_path, "- [/] Create alarm\n - assert: toast appeared\n")
|
|
ctx = _make_ctx(tmp_path)
|
|
ctx.assert_halt = True
|
|
assert convergence_gate(_make_state(), ctx) == "exit_settlement"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_settlement_runs_phase_one_only_when_final_disabled(tmp_path):
|
|
_write_plan(tmp_path, DONE_PLAN_WITH_CHECKS)
|
|
ctx = _make_ctx(tmp_path, disable_final_check=True)
|
|
report = CheckReport(
|
|
verdicts=[
|
|
CheckVerdict(item_text="alarm exists", kind="verify", status="passed", evidence="e")
|
|
]
|
|
)
|
|
fut = asyncio.get_event_loop().create_future()
|
|
fut.set_result(report)
|
|
cid = subgoal_hash("Create alarm")
|
|
ctx.checkpoint_tasks[cid] = CheckpointRun(
|
|
attempt_id=f"{cid}#1", task=fut, checkpoint=_pending_checkpoint()
|
|
)
|
|
|
|
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as final_mock:
|
|
update = await exit_settlement_node(_make_state(), ctx)
|
|
|
|
final_mock.assert_not_awaited()
|
|
assert update["exit_settlement_route"] == "end"
|
|
# Phase one still booked the attempt and produced the summary
|
|
assert any(r["status"] == "passed" for r in read_ledger(tmp_path))
|
|
assert update["run_outcome"]["tests"]["passed"] == 1
|
|
assert ctx.checkpoint_tasks == {}
|
|
|
|
|
|
# --- §8.7: settlement timeout --------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_settlement_timeout_cancels_and_books_unchecked(tmp_path):
|
|
_write_plan(tmp_path, DONE_PLAN_NO_CHECKS)
|
|
ctx = _make_ctx(tmp_path, settlement_timeout=0.05)
|
|
run = _hanging_run()
|
|
ctx.checkpoint_tasks["cid"] = run
|
|
|
|
await settle_all_checkpoints(ctx, _make_state())
|
|
|
|
assert run.task.cancelled()
|
|
records = read_ledger(tmp_path)
|
|
assert records and all(r["status"] == "unchecked" for r in records)
|
|
assert "settlement timeout" in records[0]["evidence"]
|
|
|
|
|
|
# --- §8.9: when-semantics at the exit ------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_final_check_receives_ledger_and_at_end_items(tmp_path):
|
|
_write_plan(tmp_path, DONE_PLAN_WITH_CHECKS)
|
|
ctx = _make_ctx(tmp_path)
|
|
append_ledger_record(
|
|
tmp_path,
|
|
{
|
|
"attempt_id": "x#1",
|
|
"checkpoint_id": "x",
|
|
"item_text": "toast appeared",
|
|
"kind": "assert",
|
|
"when": "on_complete",
|
|
"status": "failed",
|
|
"evidence": "no toast in anchor history",
|
|
},
|
|
)
|
|
|
|
captured = {}
|
|
|
|
async def fake_final(
|
|
ctx_arg,
|
|
goal,
|
|
plan_text,
|
|
ledger,
|
|
check_items,
|
|
attempt_id=None,
|
|
execution_report=None,
|
|
report_delivery=None,
|
|
):
|
|
captured["goal"] = goal
|
|
captured["ledger"] = ledger
|
|
captured["items"] = list(check_items)
|
|
return CheckReport(verdicts=[])
|
|
|
|
with patch("artemis.graph.graph.run_final_check", side_effect=fake_final):
|
|
await exit_settlement_node(_make_state(), ctx)
|
|
|
|
# Final review audits the user's original goal, citing the ledger
|
|
assert captured["goal"] == "the user goal"
|
|
assert any(r["status"] == "failed" for r in captured["ledger"])
|
|
whens = {(ci.text, ci.when) for ci in captured["items"]}
|
|
assert ("no crash dialog", "at_end") in whens
|
|
assert ("alarm exists", "on_complete") in whens
|
|
|
|
|
|
def test_on_complete_without_history_evidence_never_passes():
|
|
"""The 'current state satisfied but historical evidence missing' case: no
|
|
substantive record -> unchecked; a final inconclusive verdict stays
|
|
inconclusive. Neither ever resolves to passed."""
|
|
assert resolve_item_status("assert", []) == "unchecked"
|
|
inconclusive_only = [{"status": "inconclusive", "evidence": "evidence missing for past moment"}]
|
|
assert resolve_item_status("assert", inconclusive_only) == "inconclusive"
|
|
# First assert failure is permanent even if a later record passes
|
|
assert resolve_item_status("assert", [{"status": "failed"}, {"status": "passed"}]) == "failed"
|
|
# Verify: latest substantive verdict wins (repair-then-pass is passed)
|
|
assert resolve_item_status("verify", [{"status": "failed"}, {"status": "passed"}]) == "passed"
|
|
|
|
|
|
# --- §8.14: run outcome + SDK wrap-up ------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_assert_failure_yields_failed_tests_but_completed_status(tmp_path):
|
|
_write_plan(tmp_path, DONE_PLAN_WITH_CHECKS)
|
|
ctx = _make_ctx(tmp_path)
|
|
report = CheckReport(
|
|
verdicts=[
|
|
CheckVerdict(item_text="alarm exists", kind="verify", status="passed", evidence="e"),
|
|
CheckVerdict(item_text="toast appeared", kind="assert", status="failed", evidence="e"),
|
|
CheckVerdict(item_text="no crash dialog", kind="assert", status="passed", evidence="e"),
|
|
]
|
|
)
|
|
|
|
with patch("artemis.graph.graph.run_final_check", new=AsyncMock(return_value=report)):
|
|
update = await exit_settlement_node(_make_state(), ctx)
|
|
|
|
# Assert failures never bounce the task back
|
|
assert update["exit_settlement_route"] == "end"
|
|
outcome = update["run_outcome"]
|
|
assert outcome["task_status"] == "completed"
|
|
assert outcome["tests"]["failed"] == 1
|
|
assert outcome["tests"]["passed"] == 2
|
|
assert outcome["tests"]["failed_items"][0]["item_text"] == "toast appeared"
|
|
|
|
# SDK wrap-up: trace suffix distinguishes assertion failures from _PASS
|
|
assert resolve_trace_suffix("completed", outcome) == "_TESTFAIL"
|
|
assert resolve_trace_suffix("completed", {"tests": {"failed": 0}}) == "_UNVERIFIED"
|
|
assert resolve_trace_suffix("failed", outcome) == "_FAIL"
|
|
|
|
# And the return structure surfaces the machine-readable summary
|
|
wrapped = attach_test_summary("report text", outcome)
|
|
assert wrapped["result"] == "report text"
|
|
assert wrapped["test_summary"]["failed"] == 1
|
|
as_dict = attach_test_summary({"answer": 42}, outcome)
|
|
assert as_dict["answer"] == 42 and as_dict["test_summary"]["failed"] == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_verify_unmet_bounces_back_then_blocks_on_budget_exhaustion(tmp_path):
|
|
_write_plan(tmp_path, DONE_PLAN_WITH_CHECKS)
|
|
ctx = _make_ctx(tmp_path, final_check_max_attempts=2)
|
|
failing = CheckReport(
|
|
verdicts=[
|
|
CheckVerdict(
|
|
item_text="alarm exists",
|
|
kind="verify",
|
|
status="failed",
|
|
evidence="e",
|
|
suggestion="create it",
|
|
)
|
|
],
|
|
unmet_subgoals=["Create alarm"],
|
|
)
|
|
|
|
with patch("artemis.graph.graph.run_final_check", new=AsyncMock(return_value=failing)):
|
|
first = await exit_settlement_node(_make_state(), ctx)
|
|
assert first["exit_settlement_route"] == "continue"
|
|
assert exit_settlement_gate(_make_state(exit_settlement_route="continue")) == "continue"
|
|
# The unmet subgoal was reverted forward to in-progress
|
|
plan = (tmp_path / "notes" / "task_plan.md").read_text(encoding="utf-8")
|
|
assert "- [/] Create alarm" in plan
|
|
assert any("verify failed" in f for f in first["operator_feedback"])
|
|
|
|
# Re-complete and exhaust the budget
|
|
_write_plan(tmp_path, DONE_PLAN_WITH_CHECKS)
|
|
second = await exit_settlement_node(_make_state(), ctx)
|
|
|
|
assert ctx.final_check_attempts == 2
|
|
assert second["exit_settlement_route"] == "end"
|
|
outcome = second["run_outcome"]
|
|
assert outcome["task_status"] == "blocked"
|
|
|
|
|
|
def test_no_run_outcome_without_check_material(tmp_path):
|
|
"""§1.3: the test summary only exists when check items exist."""
|
|
outcome = compute_run_outcome(parse_plan(DONE_PLAN_NO_CHECKS), [], verify_blocked=False)
|
|
assert outcome.tests.passed + outcome.tests.failed == 0
|
|
assert attach_test_summary("plain", outcome.model_dump()) == "plain"
|
|
|
|
|
|
# --- §8.15: continuous + user_stop + FAIL never loops back ---------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_user_stop_final_fail_does_not_reenter_loop(tmp_path):
|
|
plan = "- [x] [Loop:continuous] Monitor inbox\n - verify: monitoring log is complete\n"
|
|
_write_plan(tmp_path, plan)
|
|
ctx = _make_ctx(tmp_path)
|
|
failing = CheckReport(
|
|
verdicts=[
|
|
CheckVerdict(
|
|
item_text="monitoring log is complete",
|
|
kind="verify",
|
|
status="failed",
|
|
evidence="gaps in the log",
|
|
)
|
|
]
|
|
)
|
|
|
|
with patch("artemis.graph.graph.run_final_check", new=AsyncMock(return_value=failing)):
|
|
update = await exit_settlement_node(_make_state(user_stop_requested=True), ctx)
|
|
|
|
assert update["exit_settlement_route"] == "end"
|
|
assert update["run_outcome"]["task_status"] == "blocked"
|
|
|
|
|
|
# --- §8.16: four switch-combination matrix -------------------------------------------
|
|
|
|
|
|
def _fresh_final_report():
|
|
return CheckReport(
|
|
verdicts=[
|
|
CheckVerdict(item_text="no crash dialog", kind="assert", status="passed", evidence="e")
|
|
]
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_matrix_off_off_reports_all_unchecked(tmp_path):
|
|
_write_plan(tmp_path, DONE_PLAN_WITH_CHECKS)
|
|
ctx = _make_ctx(tmp_path, disable_midway_checks=True, disable_final_check=True)
|
|
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as final_mock:
|
|
update = await exit_settlement_node(_make_state(), ctx)
|
|
final_mock.assert_not_awaited()
|
|
outcome = update["run_outcome"]
|
|
assert outcome["tests"]["unchecked"] == 3
|
|
assert outcome["tests"]["passed"] == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_matrix_off_on_final_judges_from_state_and_history(tmp_path):
|
|
_write_plan(tmp_path, DONE_PLAN_WITH_CHECKS)
|
|
ctx = _make_ctx(tmp_path, disable_midway_checks=True)
|
|
report = CheckReport(
|
|
verdicts=[
|
|
CheckVerdict(item_text="no crash dialog", kind="assert", status="passed", evidence="e"),
|
|
CheckVerdict(item_text="alarm exists", kind="verify", status="passed", evidence="e"),
|
|
CheckVerdict(
|
|
item_text="toast appeared",
|
|
kind="assert",
|
|
status="inconclusive",
|
|
evidence="transient evidence was never captured",
|
|
),
|
|
]
|
|
)
|
|
with patch("artemis.graph.graph.run_final_check", new=AsyncMock(return_value=report)):
|
|
update = await exit_settlement_node(_make_state(), ctx)
|
|
outcome = update["run_outcome"]
|
|
assert outcome["tests"]["passed"] == 2
|
|
assert outcome["tests"]["inconclusive"] == 1
|
|
assert outcome["tests"]["failed"] == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_matrix_on_off_settles_without_final(tmp_path):
|
|
# covered structurally by test_settlement_runs_phase_one_only_when_final_disabled;
|
|
# here assert the summary is still produced with unchecked leftovers.
|
|
_write_plan(tmp_path, DONE_PLAN_WITH_CHECKS)
|
|
ctx = _make_ctx(tmp_path, disable_final_check=True)
|
|
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as final_mock:
|
|
update = await exit_settlement_node(_make_state(), ctx)
|
|
final_mock.assert_not_awaited()
|
|
assert update["run_outcome"]["tests"]["unchecked"] == 3
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_matrix_on_on_full_pipeline(tmp_path):
|
|
_write_plan(tmp_path, DONE_PLAN_WITH_CHECKS)
|
|
ctx = _make_ctx(tmp_path)
|
|
# A checkpoint verdict already in the ledger...
|
|
append_ledger_record(
|
|
tmp_path,
|
|
{
|
|
"attempt_id": "c#1",
|
|
"checkpoint_id": "c",
|
|
"item_text": "alarm exists",
|
|
"kind": "verify",
|
|
"when": "on_complete",
|
|
"status": "passed",
|
|
"evidence": "probe output",
|
|
},
|
|
)
|
|
# ...and the final review supplements the rest.
|
|
report = CheckReport(
|
|
verdicts=[
|
|
CheckVerdict(item_text="no crash dialog", kind="assert", status="passed", evidence="e"),
|
|
CheckVerdict(item_text="toast appeared", kind="assert", status="failed", evidence="e"),
|
|
]
|
|
)
|
|
with patch("artemis.graph.graph.run_final_check", new=AsyncMock(return_value=report)):
|
|
update = await exit_settlement_node(_make_state(), ctx)
|
|
outcome = update["run_outcome"]
|
|
assert outcome["tests"]["passed"] == 2
|
|
assert outcome["tests"]["failed"] == 1
|
|
assert outcome["tests"]["unchecked"] == 0
|