fix: give final acceptance a shared retry budget and diagnostics

artemis/agents/checker/checker.py

@@ -51,6 +51,7 @@ from artemis.context import ArtemisContext
 from artemis.data_engine.context_vars import CURRENT_TRACE_ID
 from artemis.data_engine.trace import trace, trace_langchain_tool
 from artemis.llm.parser import extract_thinking_tags
+from artemis.llm.final_review import final_review_stage
 from artemis.llm.structured import content_to_text
 from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message
 from artemis.tools.history import get_history_tools
@@ -499,7 +500,9 @@ def _completed_text_report(response, check_items) -> CheckReport | None:
 
 async def _structured_report(llm, messages, check_items=()) -> CheckReport:
     if messages and (report := _completed_text_report(messages[-1], check_items)) is not None:
+        final_review_stage("validating_verdict")
         return report
+    final_review_stage("structuring_verdict")
     structured_llm = llm.with_structured_output(CheckReport)
     # A conversation must end with a user turn: Gemini rejects requests whose
     # last message is the model's own answer ("Requests ending with a model
@@ -575,6 +578,7 @@ async def _run_check_loop(
 
     report: CheckReport | None = None
     for i in range(max_iterations):
+        final_review_stage("reviewing_evidence", iteration=i + 1)
         if i == max_iterations - 1:
             messages.append(
                 HumanMessage(
@@ -594,6 +598,7 @@ async def _run_check_loop(
             break
 
         messages.append(response)
+        final_review_stage("reading_evidence")
         for tc in response.tool_calls:
             tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
             args = dict(tc["args"])

artemis/agents/checker/flash.py

@@ -1,10 +1,10 @@
 """Flash's exit-only acceptance check uses the same read-only Checker as Pro."""
 
-import asyncio
 from pathlib import Path
 
 from artemis.agents.checker.checker import run_final_check
 from artemis.context import ArtemisContext
+from artemis.llm.diagnostics import final_review_error_text
 from artemis.graph.checkpoints import (
     RunOutcome,
     TestSummary,
@@ -14,6 +14,7 @@ from artemis.graph.checkpoints import (
     write_run_outcome,
 )
 from artemis.runtime.case_result import CaseTestResult, resolve_case_result
+from artemis.services.llm import final_review_budget
 from artemis.utils.logger import get_logger
 from artemis.utils.notes import save_verified_note
 
@@ -50,7 +51,6 @@ async def settle_flash_outcome(
                 logger.warning(f"Flash 最终答复保存失败：{exc}")
                 delivery = {"ok": False, "verified": False, "error": str(exc)}
         if final_check_enabled(ctx):
-            timeout = float(getattr(ctx.execution_setup, "checkpoint_timeout", 180.0))
             publish_checker_event(
                 ctx,
                 {
@@ -63,8 +63,8 @@ async def settle_flash_outcome(
                 },
             )
             try:
-                report = await asyncio.wait_for(
-                    run_final_check(
+                report = await final_review_budget(ctx, attempt_id="final#1").run(
+                    lambda: run_final_check(
                         ctx,
                         goal=goal,
                         plan_text="",
@@ -74,7 +74,6 @@ async def settle_flash_outcome(
                         execution_report=execution_report,
                         report_delivery=delivery,
                     ),
-                    timeout=timeout,
                 )
                 result = report.test_result
                 if report.report_corrections and directory:
@@ -101,11 +100,7 @@ async def settle_flash_outcome(
                     },
                 )
             except Exception as exc:
-                detail = (
-                    f"超过 {timeout:g} 秒"
-                    if isinstance(exc, TimeoutError)
-                    else f"{type(exc).__name__}: {exc}"
-                )
+                detail = final_review_error_text(exc)
                 logger.warning(f"Flash 测试验收未完成：{detail}")
                 result = CaseTestResult(
                     status="inconclusive", reason=f"操作已结束，但测试验收未完成：{detail}。"

artemis/data_engine/storage.py

@@ -993,7 +993,7 @@ class StorageManager:
                         WHEN excluded.status = 'running' AND traces.status IN ('success', 'failed') THEN traces.payload
                         ELSE excluded.payload
                     END
-                WHERE traces.type != 'llm_diagnostic'
+                WHERE traces.type NOT IN ('llm_diagnostic', 'final_review_diagnostic')
                    OR COALESCE(json_extract(excluded.payload, '$.sequence'), 0)
                       > COALESCE(json_extract(traces.payload, '$.sequence'), -1)
                 """,

artemis/graph/graph.py

@@ -62,6 +62,8 @@ from artemis.graph.checkpoints import (
 from artemis.graph.perception import perception_node
 from artemis.graph.state import State
 from artemis.graph.visibility import strict_state
+from artemis.llm.diagnostics import final_review_error_text
+from artemis.services.llm import final_review_budget
 from artemis.tools.command_tool import (
     HANG_GUIDANCE,
     manage_task_wrapper,
@@ -325,7 +327,6 @@ async def _exit_settlement_node(state: State, ctx: ArtemisContext):
         attempt_no = ctx.final_check_attempts + 1
         ctx.final_check_attempts = attempt_no
         final_attempt_id = f"final#{attempt_no}"
-        checkpoint_timeout = float(getattr(setup, "checkpoint_timeout", 180.0) or 180.0)
         ledger = read_ledger(base_dir)
 
         def _final_event(status: str, verdicts: list[dict], **extra) -> None:
@@ -345,8 +346,8 @@ async def _exit_settlement_node(state: State, ctx: ArtemisContext):
             )
 
         try:
-            review = asyncio.wait_for(
-                run_final_check(
+            review = final_review_budget(ctx, attempt_id=final_attempt_id).run(
+                lambda: run_final_check(
                     ctx,
                     goal=getattr(state, "initial_goal", ""),
                     plan_text=plan_text,
@@ -356,7 +357,6 @@ async def _exit_settlement_node(state: State, ctx: ArtemisContext):
                     execution_report=execution_report,
                     report_delivery=report_delivery,
                 ),
-                timeout=checkpoint_timeout,
             )
             control = getattr(ctx, "task_control", None)
             if getattr(ctx, "exploration_id", None) and control:
@@ -379,11 +379,7 @@ async def _exit_settlement_node(state: State, ctx: ArtemisContext):
             logger.warning(f"Final check errored ({e}); releasing fail-open.")
             from artemis.runtime.case_result import CaseTestResult
 
-            detail = (
-                f"超过 {checkpoint_timeout:g} 秒"
-                if isinstance(e, TimeoutError)
-                else f"{type(e).__name__}: {e}"
-            )
+            detail = final_review_error_text(e)
             case_result = CaseTestResult(
                 status="inconclusive", reason=f"最终验收未完成：{detail}。"
             )
@@ -413,7 +409,7 @@ async def _exit_settlement_node(state: State, ctx: ArtemisContext):
                         "suggestion": "",
                     }
                 )
-            _final_event("error", error_verdicts, error=str(e), route="end")
+            _final_event("error", error_verdicts, error=detail, route="end")
             report = None
 
         if report is not None:

artemis/llm/diagnostics.py

@@ -13,6 +13,8 @@ from uuid import uuid4
 from PIL import Image
 
 from artemis.data_engine.context_vars import CURRENT_NODE_NAME, CURRENT_TRACE_ID
+from artemis.llm.final_review import ACTIVE_FINAL_REVIEW, FinalReviewBudgetExceeded
+from artemis.llm.reliability import LLMExhaustedError
 from artemis.llm.usage import usage_from_message
 from artemis.llm.stream_guard import ToolArgumentOverflow
 
@@ -21,6 +23,8 @@ ACTIVE_ATTEMPT: ContextVar["RequestAttempt | None"] = ContextVar("llm_attempt",
 
 
 def error_text(error: BaseException, timeout: float | None = None) -> str:
+    if isinstance(error, FinalReviewBudgetExceeded):
+        return str(error)
     if isinstance(error, TimeoutError):
         return f"模型请求在 {timeout:g} 秒内未完成" if timeout else "模型请求超时"
     text = str(error).strip() or type(error).__name__
@@ -34,6 +38,14 @@ def error_text(error: BaseException, timeout: float | None = None) -> str:
     return text[:1000]
 
 
+def final_review_error_text(error: Exception) -> str:
+    if isinstance(error, FinalReviewBudgetExceeded):
+        return str(error)
+    if isinstance(error, LLMExhaustedError):
+        return f"验收模型请求重试已用尽（{error.failure.category.value}）：{error_text(error)}"
+    return f"{type(error).__name__}: {error_text(error)}"
+
+
 def input_images(args: tuple, kwargs: dict) -> list[dict]:
     """Inspect only inline image headers; never fetch URLs or retain image data."""
     images = []
@@ -76,6 +88,9 @@ class RequestAttempt:
         self.clock = time.monotonic()
         self.last_publish = self.clock
         self.tools: dict[int | str, dict] = {}
+        self.review = ACTIVE_FINAL_REVIEW.get()
+        if self.review:
+            self.review.request_started(request_id, attempt)
         self.data = {
             "images": [],
             **metadata,
@@ -102,6 +117,8 @@ class RequestAttempt:
         self.data["updated_at"] = time.time()
         self.data["elapsed_seconds"] = round(time.monotonic() - self.clock, 3)
         self.last_publish = time.monotonic()
+        if self.review:
+            self.data["final_review"] = self.review.snapshot()
         if not self.engine or not getattr(self.engine, "current_session_id", None):
             return
         try:
@@ -203,14 +220,28 @@ class RequestAttempt:
             if (usage := usage_from_message(response)) is not None:
                 self.data["usage"] = usage
         self.data.update(phase=phase, ended_at=time.time())
+        if self.review:
+            self.review.request_finished(phase, category)
         self.publish()
 
     def retry(self, delay):
         self.data.update(
             phase="retry_wait", retry_at=time.time() + delay, retry_delay_seconds=delay
         )
+        if self.review:
+            self.review.request_finished("retry_wait")
         self.publish()
 
+    def cancel(self, error: BaseException | None = None):
+        if self.review and self.review.expired and not isinstance(error, KeyboardInterrupt):
+            self.finish(
+                "failed",
+                error=FinalReviewBudgetExceeded(self.review.total_seconds),
+                category="final_review_budget_exhausted",
+            )
+        else:
+            self.finish("cancelled", error=error, category="cancelled")
+
 
 async def observe_request_headers(request):
     """Correlate a wire request and record only hashes of routing headers."""

artemis/llm/final_review.py

@@ -0,0 +1,171 @@
+"""One bounded, observable budget for a read-only final acceptance attempt."""
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from contextvars import ContextVar
+import logging
+import sys
+import time
+from uuid import uuid4
+
+from artemis.data_engine.context_vars import CURRENT_TRACE_ID
+from artemis.llm.reliability import LLMCallError, LLMExhaustedError
+
+logger = logging.getLogger(__name__)
+
+# Allow evidence selection, a verdict, and (when needed) structured formatting,
+# plus one shared retry allowance. Do not multiply this by the checker's loop
+# limit: every request, tool read, backoff and breaker wait spends the same pool.
+FINAL_REVIEW_REQUEST_SLOTS = 3
+FINAL_REVIEW_TOOL_ALLOWANCE_SECONDS = 60.0
+MAX_FINAL_REVIEW_SECONDS = 3600.0
+
+ACTIVE_FINAL_REVIEW: ContextVar["FinalReviewBudget | None"] = ContextVar(
+    "final_review_budget", default=None
+)
+
+
+class FinalReviewBudgetExceeded(TimeoutError):
+    def __init__(self, seconds: float):
+        super().__init__(f"最终验收总预算已用尽（{seconds:g} 秒），尚未取得完整结论")
+
+
+class FinalReviewBudget:
+    def __init__(
+        self,
+        engine,
+        *,
+        attempt_id: str,
+        request_timeout: float,
+        max_retries: int,
+        retry_delay: float,
+    ):
+        self.engine = engine
+        self.trace_id = uuid4()
+        self.parent_id = CURRENT_TRACE_ID.get()
+        self.total_seconds = min(
+            MAX_FINAL_REVIEW_SECONDS,
+            request_timeout * (FINAL_REVIEW_REQUEST_SLOTS + max_retries)
+            + retry_delay * max_retries
+            + FINAL_REVIEW_TOOL_ALLOWANCE_SECONDS,
+        )
+        self.clock = time.monotonic()
+        self.timer: asyncio.Timeout | None = None
+        self.data = {
+            "review_id": str(self.trace_id),
+            "attempt_id": attempt_id,
+            "status": "running",
+            "stage": "preparing_evidence",
+            "iteration": 0,
+            "request_count": 0,
+            "retry_count": 0,
+            "sequence": 0,
+            "total_seconds": self.total_seconds,
+            "request_timeout_seconds": request_timeout,
+            "max_retries": max_retries,
+            "retry_allowance_seconds": retry_delay * max_retries,
+        }
+
+    @property
+    def expired(self) -> bool:
+        """Only our timer, never a user/guidance cancellation, exhausts the budget."""
+        return self.timer is not None and self.timer.expired()
+
+    def snapshot(self) -> dict:
+        elapsed = self.data.get("elapsed_seconds", time.monotonic() - self.clock)
+        if self.data["status"] == "running":
+            elapsed = time.monotonic() - self.clock
+        return {
+            **self.data,
+            "elapsed_seconds": round(elapsed, 3),
+            "remaining_seconds": round(max(0.0, self.total_seconds - elapsed), 3),
+        }
+
+    def publish(self) -> None:
+        self.data["sequence"] += 1
+        self.data["updated_at"] = time.time()
+        if not self.engine or not getattr(self.engine, "current_session_id", None):
+            return
+        try:
+            self.engine.record_trace(
+                type="final_review_diagnostic",
+                name="final_review",
+                trace_id=self.trace_id,
+                step_id=getattr(self.engine, "current_step_id", None),
+                parent_trace_id=self.parent_id,
+                payload=self.snapshot(),
+                status="running"
+                if self.data["status"] == "running"
+                else "success"
+                if self.data["status"] == "completed"
+                else "failed",
+                duration=self.snapshot()["elapsed_seconds"],
+            )
+        except (OSError, RuntimeError, ValueError, TypeError, AttributeError):
+            logger.warning("Could not persist final acceptance diagnostics", exc_info=True)
+
+    def stage(self, stage: str, *, iteration: int | None = None) -> None:
+        self.data["stage"] = stage
+        if iteration is not None:
+            self.data["iteration"] = iteration
+        self.publish()
+
+    def request_started(self, request_id: str, attempt: int) -> None:
+        self.data["request_count" if attempt == 1 else "retry_count"] += 1
+        self.data.update(
+            request_id=request_id, request_attempt=attempt, request_phase="waiting_response"
+        )
+        self.publish()
+
+    def request_finished(self, phase: str, category: str | None = None) -> None:
+        self.data["request_phase"] = phase
+        if category:
+            self.data["last_failure_category"] = category
+        self.publish()
+
+    def finish(self, status: str, *, error: BaseException | None = None) -> None:
+        self.data.update(
+            status=status,
+            ended_at=time.time(),
+            elapsed_seconds=time.monotonic() - self.clock,
+        )
+        if error is not None:
+            self.data["error_type"] = type(error).__name__
+            if isinstance(error, LLMCallError):
+                self.data["last_failure_category"] = error.failure.category.value
+        self.publish()
+
+    async def run[T](self, operation: Callable[[], Awaitable[T]]) -> T:
+        self.clock = time.monotonic()
+        self.data["started_at"] = time.time()
+        self.data["deadline_at"] = self.data["started_at"] + self.total_seconds
+        token = ACTIVE_FINAL_REVIEW.set(self)
+        self.timer = asyncio.timeout(self.total_seconds)
+        self.publish()
+        try:
+            async with self.timer:
+                result = await operation()
+        except (KeyboardInterrupt, asyncio.CancelledError) as exc:
+            self.finish("cancelled", error=exc)
+            raise
+        except TimeoutError as exc:
+            if self.expired:
+                error = FinalReviewBudgetExceeded(self.total_seconds)
+                self.finish("budget_exhausted", error=error)
+                raise error from exc
+            raise
+        except LLMExhaustedError as exc:
+            self.finish("request_retries_exhausted", error=exc)
+            raise
+        else:
+            self.finish("completed")
+            return result
+        finally:
+            if self.data["status"] == "running":
+                self.finish("failed", error=sys.exception())
+            ACTIVE_FINAL_REVIEW.reset(token)
+
+
+def final_review_stage(stage: str, *, iteration: int | None = None) -> None:
+    if review := ACTIVE_FINAL_REVIEW.get():
+        review.stage(stage, iteration=iteration)

artemis/services/llm.py

@@ -47,6 +47,10 @@ from artemis.config import (
 from artemis.context import ArtemisContext
 from artemis.data_engine.trace import CURRENT_TRACE_ID, DataEngineCallbackHandler
 from artemis.llm.diagnostics import ACTIVE_ATTEMPT, RequestAttempt, error_text, input_images
+from artemis.llm.final_review import (
+    ACTIVE_FINAL_REVIEW,
+    FinalReviewBudget,
+)
 from artemis.llm.cache_affinity import cache_affinity_options
 from artemis.llm.reliability import (
     CircuitBreaker,
@@ -353,8 +357,9 @@ async def _run_with_recovery[T](
       immediately: retrying cannot help and pausing would hang the task.
     - When retryable attempts are exhausted: if a fallback model is waiting
       (with_fallback), raise LLMExhaustedError so it takes over immediately;
-      otherwise pause the task (bounded by settings.LLM_PAUSE_TIMEOUT_SECONDS)
-      and retry from scratch on resume.
+      final acceptance also raises instead of pausing an already-finished task.
+      Other calls pause (bounded by settings.LLM_PAUSE_TIMEOUT_SECONDS) and
+      retry the same request on resume.
     """
     while True:
         request_token = _begin_llm_request(provider)
@@ -394,7 +399,7 @@ async def _run_with_recovery[T](
                     async with asyncio.timeout(request_timeout or settings.OPENAI_TIMEOUT_SECONDS):
                         result = await call_once()
                 except (KeyboardInterrupt, asyncio.CancelledError) as e:
-                    observation.finish("cancelled", error=e, category="cancelled")
+                    observation.cancel(e)
                     raise
                 except LLMCallError as e:
                     observation.finish("failed", error=e, category=e.failure.category.value)
@@ -458,7 +463,7 @@ async def _run_with_recovery[T](
                     except asyncio.CancelledError:
                         # Guidance may cancel a retry wait while the task continues.
                         # Close this attempt so it cannot look active next to the new request.
-                        observation.finish("cancelled")
+                        observation.cancel()
                         raise
                 else:
                     observation.finish("completed", response=result)
@@ -469,7 +474,7 @@ async def _run_with_recovery[T](
                     ACTIVE_ATTEMPT.reset(observation_token)
 
             # Retryable attempts exhausted.
-            if _FALLBACK_AVAILABLE.get():
+            if _FALLBACK_AVAILABLE.get() or ACTIVE_FINAL_REVIEW.get() is not None:
                 if failed_endpoints is not None and endpoint_key:
                     failed_endpoints[endpoint_key] = last_error
                 _record_llm_event(
@@ -478,7 +483,7 @@ async def _run_with_recovery[T](
                         "error": str(last_error)[:1000],
                         "category": last_failure.category.value,
                         "retryable": True,
-                        "handover": "fallback",
+                        "handover": "fallback" if _FALLBACK_AVAILABLE.get() else "final_review_end",
                     },
                     status="failed",
                 )
@@ -799,6 +804,16 @@ class RobustChatModelWrapper:
                     if emit_deltas:
                         self._emit_stream_delta(stream_exec_id, chunk)
         except (KeyboardInterrupt, asyncio.CancelledError):
+            if full_response is not None:
+                _record_llm_event(
+                    "llm_stream_reset",
+                    {
+                        "stream_exec_id": str(stream_exec_id),
+                        "action": "discard",
+                        "reason": "incomplete_response",
+                        "message": "本次输出未完成，已丢弃半截内容。",
+                    },
+                )
             raise
         except Exception as stream_error:
             if full_response is not None or isinstance(stream_error, ToolArgumentOverflow):
@@ -1085,6 +1100,20 @@ def _resolve_endpoint(
     )
 
 
+def final_review_budget(ctx: ArtemisContext, *, attempt_id: str) -> FinalReviewBudget:
+    """Use the same effective Checker endpoint as get_llm, without creating a model."""
+    endpoint = _resolve_endpoint(ctx, "checker")
+    return FinalReviewBudget(
+        ctx.data_engine,
+        attempt_id=attempt_id,
+        request_timeout=endpoint.timeout_seconds,
+        max_retries=endpoint.max_retries,
+        retry_delay=max(
+            settings.OPENAI_RETRY_INTERVAL_SECONDS, settings.OPENAI_RATE_LIMIT_RETRY_SECONDS
+        ),
+    )
+
+
 @overload
 def get_llm(
     ctx: ArtemisContext,