fix: give final acceptance a shared retry budget and diagnostics

This commit is contained in:
chenmingxuan committed 2026-10-18 20:51:19 +08:00
1 parent 50638d4130
commit 32441e79c4
18 files changed
+779 -40

No files matched your search

+35 -6
View File
@@ -47,6 +47,10 @@ from artemis.config import (
from artemis.context import ArtemisContext
from artemis.data_engine.trace import CURRENT_TRACE_ID, DataEngineCallbackHandler
from artemis.llm.diagnostics import ACTIVE_ATTEMPT, RequestAttempt, error_text, input_images
from artemis.llm.final_review import (
ACTIVE_FINAL_REVIEW,
FinalReviewBudget,
)
from artemis.llm.cache_affinity import cache_affinity_options
from artemis.llm.reliability import (
CircuitBreaker,
@@ -353,8 +357,9 @@ async def _run_with_recovery[T](
immediately: retrying cannot help and pausing would hang the task.
- When retryable attempts are exhausted: if a fallback model is waiting
(with_fallback), raise LLMExhaustedError so it takes over immediately;
otherwise pause the task (bounded by settings.LLM_PAUSE_TIMEOUT_SECONDS)
and retry from scratch on resume.
final acceptance also raises instead of pausing an already-finished task.
Other calls pause (bounded by settings.LLM_PAUSE_TIMEOUT_SECONDS) and
retry the same request on resume.
"""
while True:
request_token = _begin_llm_request(provider)
@@ -394,7 +399,7 @@ async def _run_with_recovery[T](
async with asyncio.timeout(request_timeout or settings.OPENAI_TIMEOUT_SECONDS):
result = await call_once()
except (KeyboardInterrupt, asyncio.CancelledError) as e:
observation.finish("cancelled", error=e, category="cancelled")
observation.cancel(e)
raise
except LLMCallError as e:
observation.finish("failed", error=e, category=e.failure.category.value)
@@ -458,7 +463,7 @@ async def _run_with_recovery[T](
except asyncio.CancelledError:
# Guidance may cancel a retry wait while the task continues.
# Close this attempt so it cannot look active next to the new request.
observation.finish("cancelled")
observation.cancel()
raise
else:
observation.finish("completed", response=result)
@@ -469,7 +474,7 @@ async def _run_with_recovery[T](
ACTIVE_ATTEMPT.reset(observation_token)
# Retryable attempts exhausted.
if _FALLBACK_AVAILABLE.get():
if _FALLBACK_AVAILABLE.get() or ACTIVE_FINAL_REVIEW.get() is not None:
if failed_endpoints is not None and endpoint_key:
failed_endpoints[endpoint_key] = last_error
_record_llm_event(
@@ -478,7 +483,7 @@ async def _run_with_recovery[T](
"error": str(last_error)[:1000],
"category": last_failure.category.value,
"retryable": True,
"handover": "fallback",
"handover": "fallback" if _FALLBACK_AVAILABLE.get() else "final_review_end",
},
status="failed",
)
@@ -799,6 +804,16 @@ class RobustChatModelWrapper:
if emit_deltas:
self._emit_stream_delta(stream_exec_id, chunk)
except (KeyboardInterrupt, asyncio.CancelledError):
if full_response is not None:
_record_llm_event(
"llm_stream_reset",
{
"stream_exec_id": str(stream_exec_id),
"action": "discard",
"reason": "incomplete_response",
"message": "本次输出未完成,已丢弃半截内容。",
},
)
raise
except Exception as stream_error:
if full_response is not None or isinstance(stream_error, ToolArgumentOverflow):
@@ -1085,6 +1100,20 @@ def _resolve_endpoint(
)
def final_review_budget(ctx: ArtemisContext, *, attempt_id: str) -> FinalReviewBudget:
"""Use the same effective Checker endpoint as get_llm, without creating a model."""
endpoint = _resolve_endpoint(ctx, "checker")
return FinalReviewBudget(
ctx.data_engine,
attempt_id=attempt_id,
request_timeout=endpoint.timeout_seconds,
max_retries=endpoint.max_retries,
retry_delay=max(
settings.OPENAI_RETRY_INTERVAL_SECONDS, settings.OPENAI_RATE_LIMIT_RETRY_SECONDS
),
)
@overload
def get_llm(
ctx: ArtemisContext,