fix: give final acceptance a shared retry budget and diagnostics
This commit is contained in:
1 parent
50638d4130
commit
32441e79c4
18 files changed
+779
-40
No files matched your search
+35
-6
@@ -47,6 +47,10 @@ from artemis.config import (
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import CURRENT_TRACE_ID, DataEngineCallbackHandler
|
||||
from artemis.llm.diagnostics import ACTIVE_ATTEMPT, RequestAttempt, error_text, input_images
|
||||
from artemis.llm.final_review import (
|
||||
ACTIVE_FINAL_REVIEW,
|
||||
FinalReviewBudget,
|
||||
)
|
||||
from artemis.llm.cache_affinity import cache_affinity_options
|
||||
from artemis.llm.reliability import (
|
||||
CircuitBreaker,
|
||||
@@ -353,8 +357,9 @@ async def _run_with_recovery[T](
|
||||
immediately: retrying cannot help and pausing would hang the task.
|
||||
- When retryable attempts are exhausted: if a fallback model is waiting
|
||||
(with_fallback), raise LLMExhaustedError so it takes over immediately;
|
||||
otherwise pause the task (bounded by settings.LLM_PAUSE_TIMEOUT_SECONDS)
|
||||
and retry from scratch on resume.
|
||||
final acceptance also raises instead of pausing an already-finished task.
|
||||
Other calls pause (bounded by settings.LLM_PAUSE_TIMEOUT_SECONDS) and
|
||||
retry the same request on resume.
|
||||
"""
|
||||
while True:
|
||||
request_token = _begin_llm_request(provider)
|
||||
@@ -394,7 +399,7 @@ async def _run_with_recovery[T](
|
||||
async with asyncio.timeout(request_timeout or settings.OPENAI_TIMEOUT_SECONDS):
|
||||
result = await call_once()
|
||||
except (KeyboardInterrupt, asyncio.CancelledError) as e:
|
||||
observation.finish("cancelled", error=e, category="cancelled")
|
||||
observation.cancel(e)
|
||||
raise
|
||||
except LLMCallError as e:
|
||||
observation.finish("failed", error=e, category=e.failure.category.value)
|
||||
@@ -458,7 +463,7 @@ async def _run_with_recovery[T](
|
||||
except asyncio.CancelledError:
|
||||
# Guidance may cancel a retry wait while the task continues.
|
||||
# Close this attempt so it cannot look active next to the new request.
|
||||
observation.finish("cancelled")
|
||||
observation.cancel()
|
||||
raise
|
||||
else:
|
||||
observation.finish("completed", response=result)
|
||||
@@ -469,7 +474,7 @@ async def _run_with_recovery[T](
|
||||
ACTIVE_ATTEMPT.reset(observation_token)
|
||||
|
||||
# Retryable attempts exhausted.
|
||||
if _FALLBACK_AVAILABLE.get():
|
||||
if _FALLBACK_AVAILABLE.get() or ACTIVE_FINAL_REVIEW.get() is not None:
|
||||
if failed_endpoints is not None and endpoint_key:
|
||||
failed_endpoints[endpoint_key] = last_error
|
||||
_record_llm_event(
|
||||
@@ -478,7 +483,7 @@ async def _run_with_recovery[T](
|
||||
"error": str(last_error)[:1000],
|
||||
"category": last_failure.category.value,
|
||||
"retryable": True,
|
||||
"handover": "fallback",
|
||||
"handover": "fallback" if _FALLBACK_AVAILABLE.get() else "final_review_end",
|
||||
},
|
||||
status="failed",
|
||||
)
|
||||
@@ -799,6 +804,16 @@ class RobustChatModelWrapper:
|
||||
if emit_deltas:
|
||||
self._emit_stream_delta(stream_exec_id, chunk)
|
||||
except (KeyboardInterrupt, asyncio.CancelledError):
|
||||
if full_response is not None:
|
||||
_record_llm_event(
|
||||
"llm_stream_reset",
|
||||
{
|
||||
"stream_exec_id": str(stream_exec_id),
|
||||
"action": "discard",
|
||||
"reason": "incomplete_response",
|
||||
"message": "本次输出未完成,已丢弃半截内容。",
|
||||
},
|
||||
)
|
||||
raise
|
||||
except Exception as stream_error:
|
||||
if full_response is not None or isinstance(stream_error, ToolArgumentOverflow):
|
||||
@@ -1085,6 +1100,20 @@ def _resolve_endpoint(
|
||||
)
|
||||
|
||||
|
||||
def final_review_budget(ctx: ArtemisContext, *, attempt_id: str) -> FinalReviewBudget:
|
||||
"""Use the same effective Checker endpoint as get_llm, without creating a model."""
|
||||
endpoint = _resolve_endpoint(ctx, "checker")
|
||||
return FinalReviewBudget(
|
||||
ctx.data_engine,
|
||||
attempt_id=attempt_id,
|
||||
request_timeout=endpoint.timeout_seconds,
|
||||
max_retries=endpoint.max_retries,
|
||||
retry_delay=max(
|
||||
settings.OPENAI_RETRY_INTERVAL_SECONDS, settings.OPENAI_RATE_LIMIT_RETRY_SECONDS
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@overload
|
||||
def get_llm(
|
||||
ctx: ArtemisContext,
|
||||
|
||||
Reference in new issue
Block a user