feat(reliability): centralize retry failure classification
This commit is contained in:
1 parent
cbe8f4f0c6
commit
2acdeba8c5
6 files changed
+459
-113
No files matched your search
@@ -19,7 +19,6 @@ import hashlib
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import random
|
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
from typing import Any, Literal
|
from typing import Any, Literal
|
||||||
@@ -27,7 +26,6 @@ from typing import Any, Literal
|
|||||||
import cv2
|
import cv2
|
||||||
from google import genai
|
from google import genai
|
||||||
from google.genai import types
|
from google.genai import types
|
||||||
from google.genai.errors import APIError
|
|
||||||
import httpx
|
import httpx
|
||||||
from langchain_core.messages import (
|
from langchain_core.messages import (
|
||||||
BaseMessage,
|
BaseMessage,
|
||||||
@@ -44,7 +42,13 @@ from artemis.context import ArtemisContext
|
|||||||
from artemis.data_engine.storage import StorageManager
|
from artemis.data_engine.storage import StorageManager
|
||||||
from artemis.data_engine.trace import TraceSpan, trace
|
from artemis.data_engine.trace import TraceSpan, trace
|
||||||
from artemis.graph.state import State
|
from artemis.graph.state import State
|
||||||
from artemis.services.llm import get_llm
|
from artemis.llm.reliability import (
|
||||||
|
LLMExhaustedError,
|
||||||
|
LLMPermanentError,
|
||||||
|
classify_failure,
|
||||||
|
retry_policy_for,
|
||||||
|
)
|
||||||
|
from artemis.services.llm import _record_llm_event, _record_llm_retry, get_llm
|
||||||
|
|
||||||
# Import diagnostic functions directly
|
# Import diagnostic functions directly
|
||||||
from artemis.agents.explorer.constants import EXPLORE_DESCRIPTIONS
|
from artemis.agents.explorer.constants import EXPLORE_DESCRIPTIONS
|
||||||
@@ -59,6 +63,84 @@ from artemis.utils.visualization import draw_dots, format_minimal_list_with_poin
|
|||||||
|
|
||||||
logger = get_logger(__name__)
|
logger = get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
async def _generate_content_with_reliability(operation, *, label: str = "Explorer model call"):
|
||||||
|
"""Run one native google-genai model call under the shared reliability layer.
|
||||||
|
|
||||||
|
Retry decisions are owned by ``classify_failure``/``retry_policy_for``
|
||||||
|
(artemis.llm.reliability) instead of a blanket exponential-backoff loop:
|
||||||
|
non-retryable categories (auth, bad request) are raised immediately as
|
||||||
|
``LLMPermanentError`` and exhausted retryable categories surface as
|
||||||
|
``LLMExhaustedError``. Only the retry/error-classification policy is
|
||||||
|
centralized here; the google-genai transport itself stays untouched.
|
||||||
|
``operation`` must be a zero-argument callable returning a fresh awaitable
|
||||||
|
per attempt.
|
||||||
|
"""
|
||||||
|
attempt = 0
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
return await operation()
|
||||||
|
except Exception as call_err:
|
||||||
|
attempt += 1
|
||||||
|
failure = classify_failure(call_err)
|
||||||
|
if not failure.retryable:
|
||||||
|
logger.error(
|
||||||
|
f"{label} permanently failed [{failure.category.value}]: {call_err}"
|
||||||
|
)
|
||||||
|
_record_llm_event(
|
||||||
|
"llm_gave_up",
|
||||||
|
{
|
||||||
|
"source": "explorer",
|
||||||
|
"error": str(call_err)[:1000],
|
||||||
|
"category": failure.category.value,
|
||||||
|
"retryable": False,
|
||||||
|
},
|
||||||
|
status="failed",
|
||||||
|
)
|
||||||
|
raise LLMPermanentError(
|
||||||
|
f"{label} failed [{failure.category.value}]: {call_err}",
|
||||||
|
failure=failure,
|
||||||
|
cause=call_err,
|
||||||
|
) from call_err
|
||||||
|
policy = retry_policy_for(failure.category)
|
||||||
|
if attempt >= policy.max_attempts:
|
||||||
|
logger.error(
|
||||||
|
f"{label} exhausted {attempt} attempt(s)"
|
||||||
|
f" [{failure.category.value}]: {call_err}"
|
||||||
|
)
|
||||||
|
_record_llm_event(
|
||||||
|
"llm_gave_up",
|
||||||
|
{
|
||||||
|
"source": "explorer",
|
||||||
|
"error": str(call_err)[:1000],
|
||||||
|
"category": failure.category.value,
|
||||||
|
"retryable": True,
|
||||||
|
"attempts": attempt,
|
||||||
|
},
|
||||||
|
status="failed",
|
||||||
|
)
|
||||||
|
raise LLMExhaustedError(
|
||||||
|
f"{label} exhausted {attempt} attempt(s)"
|
||||||
|
f" [{failure.category.value}]: {call_err}",
|
||||||
|
failure=failure,
|
||||||
|
cause=call_err,
|
||||||
|
) from call_err
|
||||||
|
delay = policy.delay_for(attempt)
|
||||||
|
logger.warning(
|
||||||
|
f"{label} failed [{failure.category.value}] on attempt"
|
||||||
|
f" {attempt}/{policy.max_attempts}: {call_err}."
|
||||||
|
f" Retrying in {delay:.2f}s..."
|
||||||
|
)
|
||||||
|
_record_llm_retry(
|
||||||
|
str(call_err),
|
||||||
|
delay,
|
||||||
|
attempt=attempt,
|
||||||
|
max_retries=policy.max_attempts,
|
||||||
|
source="explorer",
|
||||||
|
)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
|
||||||
|
|
||||||
UNIVERSAL_EXPLORER_TOOLS = [
|
UNIVERSAL_EXPLORER_TOOLS = [
|
||||||
{
|
{
|
||||||
"type": "function",
|
"type": "function",
|
||||||
@@ -1802,41 +1884,16 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
max_call_retries = 5
|
response = await _generate_content_with_reliability(
|
||||||
for attempt in range(max_call_retries + 1):
|
lambda: asyncio.wait_for(
|
||||||
try:
|
client.aio.models.generate_content(
|
||||||
response = await asyncio.wait_for(
|
model=model_name,
|
||||||
client.aio.models.generate_content(
|
contents=contents,
|
||||||
model=model_name,
|
config=generate_config,
|
||||||
contents=contents,
|
),
|
||||||
config=generate_config,
|
timeout=180,
|
||||||
),
|
),
|
||||||
timeout=180,
|
)
|
||||||
)
|
|
||||||
break
|
|
||||||
except APIError as api_err:
|
|
||||||
code = getattr(api_err, "code", None)
|
|
||||||
if code in [429, 503]:
|
|
||||||
if attempt >= max_call_retries:
|
|
||||||
raise api_err
|
|
||||||
backoff = (2.0**attempt) + random.uniform(0.1, 1.0)
|
|
||||||
logger.warning(
|
|
||||||
f"APIError {code} (overload/rate limit) on"
|
|
||||||
f" attempt {attempt + 1}. Retrying in"
|
|
||||||
f" {backoff:.2f}s..."
|
|
||||||
)
|
|
||||||
await asyncio.sleep(backoff)
|
|
||||||
else:
|
|
||||||
raise api_err
|
|
||||||
except Exception as e:
|
|
||||||
if attempt >= max_call_retries:
|
|
||||||
raise e
|
|
||||||
backoff = (2.0**attempt) + random.uniform(0.1, 1.0)
|
|
||||||
logger.warning(
|
|
||||||
f"Unexpected error on attempt {attempt + 1}:"
|
|
||||||
f" {e}. Retrying in {backoff:.2f}s..."
|
|
||||||
)
|
|
||||||
await asyncio.sleep(backoff)
|
|
||||||
end_turn = time.perf_counter()
|
end_turn = time.perf_counter()
|
||||||
turn_dur = end_turn - start_turn
|
turn_dur = end_turn - start_turn
|
||||||
self.turn_latencies.append(turn_dur)
|
self.turn_latencies.append(turn_dur)
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ import glob
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import random
|
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import tempfile
|
import tempfile
|
||||||
@@ -28,7 +27,6 @@ from typing import Any
|
|||||||
|
|
||||||
from google import genai
|
from google import genai
|
||||||
from google.genai import types
|
from google.genai import types
|
||||||
from google.genai.errors import APIError
|
|
||||||
from langchain_core.messages import (
|
from langchain_core.messages import (
|
||||||
BaseMessage,
|
BaseMessage,
|
||||||
HumanMessage,
|
HumanMessage,
|
||||||
@@ -49,7 +47,8 @@ from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
|
|||||||
from artemis.context import ArtemisContext
|
from artemis.context import ArtemisContext
|
||||||
from artemis.controllers.controller_factory import get_controller
|
from artemis.controllers.controller_factory import get_controller
|
||||||
from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace
|
from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace
|
||||||
from artemis.services.llm import get_llm
|
from artemis.llm.reliability import retry_policy_for
|
||||||
|
from artemis.services.llm import _record_llm_event, _record_llm_retry, get_llm
|
||||||
from artemis.utils.logger import get_logger
|
from artemis.utils.logger import get_logger
|
||||||
from artemis.utils.video import (
|
from artemis.utils.video import (
|
||||||
compress_video_for_api,
|
compress_video_for_api,
|
||||||
@@ -76,27 +75,50 @@ _CLEANUP_LOCK = asyncio.Lock()
|
|||||||
|
|
||||||
|
|
||||||
async def _invoke_with_retry(operation, label: str, max_attempts: int = 3):
|
async def _invoke_with_retry(operation, label: str, max_attempts: int = 3):
|
||||||
"""Retry side-effect-free model coordination calls on transient failures."""
|
"""Retry side-effect-free model coordination calls using shared classification.
|
||||||
for attempt in range(1, max_attempts + 1):
|
|
||||||
|
Retry decisions come from ``classify_video_failure`` plus the shared
|
||||||
|
per-category policies in :mod:`artemis.llm.reliability` instead of ad-hoc
|
||||||
|
status-code sets. ``operation`` must be restartable from scratch: each
|
||||||
|
retry re-invokes it and any partially accumulated result (e.g. consumed
|
||||||
|
stream chunks) from the failed attempt is discarded, never surfaced as a
|
||||||
|
complete output. The original exception is re-raised on giving up so
|
||||||
|
downstream ``classify_video_failure`` consumers keep seeing the provider's
|
||||||
|
native error shape.
|
||||||
|
"""
|
||||||
|
attempt = 0
|
||||||
|
while True:
|
||||||
|
attempt += 1
|
||||||
try:
|
try:
|
||||||
return await operation()
|
return await operation()
|
||||||
except APIError as exc:
|
except Exception as exc:
|
||||||
code = getattr(exc, "code", None)
|
failure = classify_video_failure(exc)
|
||||||
if code in {400, 401, 403, 404} or attempt >= max_attempts:
|
if not failure.retryable:
|
||||||
|
logger.warning(
|
||||||
|
f"{label} non-retryable failure [{failure.category.value}] "
|
||||||
|
f"on attempt {attempt}: {exc}"
|
||||||
|
)
|
||||||
raise
|
raise
|
||||||
delay = (2.0 ** (attempt - 1)) + random.uniform(0.1, 0.8)
|
policy = retry_policy_for(failure.category)
|
||||||
|
allowed_attempts = min(max_attempts, policy.max_attempts)
|
||||||
|
if attempt >= allowed_attempts:
|
||||||
|
logger.warning(
|
||||||
|
f"{label} exhausted {attempt} attempt(s) "
|
||||||
|
f"[{failure.category.value}]: {exc}"
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
delay = policy.delay_for(attempt)
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f"{label} transient API failure ({attempt}/{max_attempts}): {exc}; "
|
f"{label} retryable failure [{failure.category.value}] "
|
||||||
f"retrying in {delay:.2f}s"
|
f"({attempt}/{allowed_attempts}): {exc}; retrying from scratch "
|
||||||
|
f"in {delay:.2f}s (partial output from the failed attempt is discarded)"
|
||||||
)
|
)
|
||||||
await asyncio.sleep(delay)
|
_record_llm_retry(
|
||||||
except (TimeoutError, ConnectionError, OSError) as exc:
|
str(exc),
|
||||||
if attempt >= max_attempts:
|
delay,
|
||||||
raise
|
attempt=attempt,
|
||||||
delay = (2.0 ** (attempt - 1)) + random.uniform(0.1, 0.8)
|
max_retries=allowed_attempts,
|
||||||
logger.warning(
|
source=label,
|
||||||
f"{label} transient failure ({attempt}/{max_attempts}): {exc}; "
|
|
||||||
f"retrying in {delay:.2f}s"
|
|
||||||
)
|
)
|
||||||
await asyncio.sleep(delay)
|
await asyncio.sleep(delay)
|
||||||
|
|
||||||
@@ -1094,6 +1116,14 @@ class VideoAnalyzer:
|
|||||||
logger.warning(
|
logger.warning(
|
||||||
"Native video chunk exhausted retries; using universal fallback"
|
"Native video chunk exhausted retries; using universal fallback"
|
||||||
)
|
)
|
||||||
|
_record_llm_event(
|
||||||
|
"llm_fallback",
|
||||||
|
{
|
||||||
|
"reason": "native_video_chunk_failed",
|
||||||
|
"category": failure.category.value,
|
||||||
|
"error": str(error)[:500],
|
||||||
|
},
|
||||||
|
)
|
||||||
fallback_result = await self._exec_single_chunk_universal(
|
fallback_result = await self._exec_single_chunk_universal(
|
||||||
compressed_path=compressed_path,
|
compressed_path=compressed_path,
|
||||||
raw_path=path,
|
raw_path=path,
|
||||||
@@ -1697,35 +1727,43 @@ class VideoAnalyzer:
|
|||||||
except Exception as ce:
|
except Exception as ce:
|
||||||
logger.error(f"Failed to delete cloud file {file.name}: {ce}")
|
logger.error(f"Failed to delete cloud file {file.name}: {ce}")
|
||||||
|
|
||||||
except APIError as api_err:
|
|
||||||
logger.warning(f"Sub-agent APIError attempt {attempt + 1} failed: {api_err}")
|
|
||||||
code = getattr(api_err, "code", None)
|
|
||||||
if code == 429:
|
|
||||||
if attempt >= max_retries:
|
|
||||||
return await finish_failed(api_err)
|
|
||||||
|
|
||||||
backoff = (2.0**attempt) + random.uniform(0.1, 1.0)
|
|
||||||
logger.info(
|
|
||||||
f"Rate limited (429). Backing off for {backoff:.2f}s"
|
|
||||||
" without expanding boundaries..."
|
|
||||||
)
|
|
||||||
await asyncio.sleep(backoff)
|
|
||||||
continue
|
|
||||||
elif code in [400, 401, 403, 404]:
|
|
||||||
return await finish_failed(api_err, retryable=False)
|
|
||||||
|
|
||||||
if attempt >= max_retries:
|
|
||||||
return await finish_failed(api_err)
|
|
||||||
backoff = 2.0**attempt
|
|
||||||
await asyncio.sleep(backoff)
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Sub-agent attempt {attempt + 1} failed: {e}")
|
# Retry decisions are owned by the shared classification layer.
|
||||||
if attempt >= max_retries:
|
# Every attempt restarts from scratch (segment re-extracted,
|
||||||
|
# file re-uploaded, stream re-consumed); partially accumulated
|
||||||
|
# stream output from the failed attempt is discarded, never
|
||||||
|
# surfaced as a complete answer.
|
||||||
|
failure = classify_video_failure(e)
|
||||||
|
attempts_so_far = attempt + 1
|
||||||
|
if not failure.retryable:
|
||||||
|
logger.warning(
|
||||||
|
f"Sub-agent non-retryable failure [{failure.category.value}]"
|
||||||
|
f" on attempt {attempts_so_far}: {e}"
|
||||||
|
)
|
||||||
|
return await finish_failed(e, retryable=False)
|
||||||
|
policy = retry_policy_for(failure.category)
|
||||||
|
allowed_attempts = min(max_retries + 1, policy.max_attempts)
|
||||||
|
if attempts_so_far >= allowed_attempts:
|
||||||
|
logger.warning(
|
||||||
|
f"Sub-agent exhausted {attempts_so_far} attempt(s)"
|
||||||
|
f" [{failure.category.value}]: {e}"
|
||||||
|
)
|
||||||
return await finish_failed(e)
|
return await finish_failed(e)
|
||||||
|
delay = policy.delay_for(attempts_so_far)
|
||||||
backoff = 2.0**attempt
|
logger.warning(
|
||||||
logger.info(f"Backing off for {backoff}s before retry...")
|
f"Sub-agent retryable failure [{failure.category.value}]"
|
||||||
await asyncio.sleep(backoff)
|
f" ({attempts_so_far}/{allowed_attempts}): {e}; retrying"
|
||||||
|
f" from scratch in {delay:.2f}s (partial stream output"
|
||||||
|
" from the failed attempt is discarded)"
|
||||||
|
)
|
||||||
|
_record_llm_retry(
|
||||||
|
str(e),
|
||||||
|
delay,
|
||||||
|
attempt=attempts_so_far,
|
||||||
|
max_retries=allowed_attempts,
|
||||||
|
source="video_sub_agent",
|
||||||
|
)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
f"Video sub-agent exhausted retries for {current_start:.1f}s-{current_end:.1f}s"
|
f"Video sub-agent exhausted retries for {current_start:.1f}s-{current_end:.1f}s"
|
||||||
@@ -1803,6 +1841,18 @@ class VideoAnalyzer:
|
|||||||
and prompt_with_context is not None
|
and prompt_with_context is not None
|
||||||
):
|
):
|
||||||
try:
|
try:
|
||||||
|
logger.warning(
|
||||||
|
"Native audio analysis failed"
|
||||||
|
f" [{failure.category.value}]; using universal fallback"
|
||||||
|
)
|
||||||
|
_record_llm_event(
|
||||||
|
"llm_fallback",
|
||||||
|
{
|
||||||
|
"reason": "native_audio_analysis_failed",
|
||||||
|
"category": failure.category.value,
|
||||||
|
"error": str(error)[:500],
|
||||||
|
},
|
||||||
|
)
|
||||||
fallback_result = await self._exec_analyze_audio_universal(
|
fallback_result = await self._exec_analyze_audio_universal(
|
||||||
audio_path=audio_path,
|
audio_path=audio_path,
|
||||||
start_time=current_start,
|
start_time=current_start,
|
||||||
@@ -2135,35 +2185,40 @@ class VideoAnalyzer:
|
|||||||
except Exception as ce:
|
except Exception as ce:
|
||||||
logger.error(f"Failed to delete cloud file {file.name}: {ce}")
|
logger.error(f"Failed to delete cloud file {file.name}: {ce}")
|
||||||
|
|
||||||
except APIError as api_err:
|
|
||||||
logger.warning(f"Audio APIError attempt {attempt + 1} failed: {api_err}")
|
|
||||||
code = getattr(api_err, "code", None)
|
|
||||||
if code == 429:
|
|
||||||
if attempt >= max_retries:
|
|
||||||
return await finish_failed_audio(api_err)
|
|
||||||
|
|
||||||
backoff = (2.0**attempt) + random.uniform(0.1, 1.0)
|
|
||||||
logger.info(
|
|
||||||
f"Rate limited (429). Backing off for {backoff:.2f}s"
|
|
||||||
" without expanding boundaries..."
|
|
||||||
)
|
|
||||||
await asyncio.sleep(backoff)
|
|
||||||
continue
|
|
||||||
elif code in [400, 401, 403, 404]:
|
|
||||||
return await finish_failed_audio(api_err, retryable=False)
|
|
||||||
|
|
||||||
if attempt >= max_retries:
|
|
||||||
return await finish_failed_audio(api_err)
|
|
||||||
backoff = 2.0**attempt
|
|
||||||
await asyncio.sleep(backoff)
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Audio attempt {attempt + 1} failed: {e}")
|
# Same contract as the video sub-agent loop: classification
|
||||||
if attempt >= max_retries:
|
# owns the retry decision and every retry restarts the attempt
|
||||||
|
# from scratch.
|
||||||
|
failure = classify_video_failure(e)
|
||||||
|
attempts_so_far = attempt + 1
|
||||||
|
if not failure.retryable:
|
||||||
|
logger.warning(
|
||||||
|
f"Audio non-retryable failure [{failure.category.value}]"
|
||||||
|
f" on attempt {attempts_so_far}: {e}"
|
||||||
|
)
|
||||||
|
return await finish_failed_audio(e, retryable=False)
|
||||||
|
policy = retry_policy_for(failure.category)
|
||||||
|
allowed_attempts = min(max_retries + 1, policy.max_attempts)
|
||||||
|
if attempts_so_far >= allowed_attempts:
|
||||||
|
logger.warning(
|
||||||
|
f"Audio analysis exhausted {attempts_so_far} attempt(s)"
|
||||||
|
f" [{failure.category.value}]: {e}"
|
||||||
|
)
|
||||||
return await finish_failed_audio(e)
|
return await finish_failed_audio(e)
|
||||||
|
delay = policy.delay_for(attempts_so_far)
|
||||||
backoff = 2.0**attempt
|
logger.warning(
|
||||||
logger.info(f"Backing off for {backoff}s before retry...")
|
f"Audio retryable failure [{failure.category.value}]"
|
||||||
await asyncio.sleep(backoff)
|
f" ({attempts_so_far}/{allowed_attempts}): {e}; retrying"
|
||||||
|
f" from scratch in {delay:.2f}s"
|
||||||
|
)
|
||||||
|
_record_llm_retry(
|
||||||
|
str(e),
|
||||||
|
delay,
|
||||||
|
attempt=attempts_so_far,
|
||||||
|
max_retries=allowed_attempts,
|
||||||
|
source="video_audio_agent",
|
||||||
|
)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
|
||||||
async def _exec_single_chunk_universal(
|
async def _exec_single_chunk_universal(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -77,6 +77,9 @@ def extract_status_code(error: BaseException) -> int | None:
|
|||||||
_RATE_LIMIT_MARKERS = ("rate limit", "quota", "resource_exhausted")
|
_RATE_LIMIT_MARKERS = ("rate limit", "quota", "resource_exhausted")
|
||||||
_UNAVAILABLE_MARKERS = ("unavailable", "overloaded", "high demand", "service unavailable")
|
_UNAVAILABLE_MARKERS = ("unavailable", "overloaded", "high demand", "service unavailable")
|
||||||
_AUTH_MARKERS = ("unauthorized", "forbidden", "api key", "credential")
|
_AUTH_MARKERS = ("unauthorized", "forbidden", "api key", "credential")
|
||||||
|
# "deadline exceeded"/"deadline_exceeded" cover google-genai SDK timeouts that
|
||||||
|
# surface without an HTTP status code attached.
|
||||||
|
_TIMEOUT_MARKERS = ("timed out", "timeout", "deadline exceeded", "deadline_exceeded")
|
||||||
|
|
||||||
|
|
||||||
def classify_failure(error: BaseException) -> Failure:
|
def classify_failure(error: BaseException) -> Failure:
|
||||||
@@ -91,7 +94,7 @@ def classify_failure(error: BaseException) -> Failure:
|
|||||||
return Failure(FailureCategory.RATE_LIMIT, True, True)
|
return Failure(FailureCategory.RATE_LIMIT, True, True)
|
||||||
if code in {500, 502, 503, 504} or any(marker in message for marker in _UNAVAILABLE_MARKERS):
|
if code in {500, 502, 503, 504} or any(marker in message for marker in _UNAVAILABLE_MARKERS):
|
||||||
return Failure(FailureCategory.PROVIDER_UNAVAILABLE, True, True)
|
return Failure(FailureCategory.PROVIDER_UNAVAILABLE, True, True)
|
||||||
if isinstance(error, TimeoutError) or "timed out" in message or "timeout" in message:
|
if isinstance(error, TimeoutError) or any(marker in message for marker in _TIMEOUT_MARKERS):
|
||||||
return Failure(FailureCategory.TIMEOUT, True, True)
|
return Failure(FailureCategory.TIMEOUT, True, True)
|
||||||
if isinstance(error, (ConnectionError, OSError)):
|
if isinstance(error, (ConnectionError, OSError)):
|
||||||
return Failure(FailureCategory.CONNECTION, True, True)
|
return Failure(FailureCategory.CONNECTION, True, True)
|
||||||
@@ -163,7 +166,19 @@ _DEFAULT_RETRY_POLICIES: dict[FailureCategory, RetryPolicy] = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def retry_policy_for(category: FailureCategory) -> RetryPolicy:
|
def retry_policy_for(category: FailureCategory | str) -> RetryPolicy:
|
||||||
|
"""Look up the retry policy for a failure category.
|
||||||
|
|
||||||
|
Accepts the generic ``FailureCategory`` or any string-valued category from
|
||||||
|
a domain extension (e.g. the video pipeline's ``VideoFailureCategory``).
|
||||||
|
Extension-only categories fall back to the UNKNOWN policy, so callers must
|
||||||
|
gate on ``failure.retryable`` before consulting attempt counts.
|
||||||
|
"""
|
||||||
|
if not isinstance(category, FailureCategory):
|
||||||
|
try:
|
||||||
|
category = FailureCategory(str(category))
|
||||||
|
except ValueError:
|
||||||
|
category = FailureCategory.UNKNOWN
|
||||||
return _DEFAULT_RETRY_POLICIES.get(category, _DEFAULT_RETRY_POLICIES[FailureCategory.UNKNOWN])
|
return _DEFAULT_RETRY_POLICIES.get(category, _DEFAULT_RETRY_POLICIES[FailureCategory.UNKNOWN])
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1272,3 +1272,93 @@ def test_explorer_prune_historical_images():
|
|||||||
assert contents[0].parts[1].text == "Some text"
|
assert contents[0].parts[1].text == "Some text"
|
||||||
# Second (last) image part should be kept intact
|
# Second (last) image part should be kept intact
|
||||||
assert contents[1].parts[0] is part2
|
assert contents[1].parts[0] is part2
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeGenaiError(Exception):
|
||||||
|
"""Mimics google.genai.errors.APIError's shape (a ``code`` attribute)."""
|
||||||
|
|
||||||
|
def __init__(self, message: str, code: int | None = None):
|
||||||
|
super().__init__(message)
|
||||||
|
self.code = code
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_generate_content_reliability_permanent_error_is_not_retried():
|
||||||
|
from artemis.agents.explorer.explorer import _generate_content_with_reliability
|
||||||
|
from artemis.llm.reliability import LLMPermanentError
|
||||||
|
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
async def op():
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
raise _FakeGenaiError("PERMISSION_DENIED: forbidden", code=403)
|
||||||
|
|
||||||
|
with pytest.raises(LLMPermanentError) as exc_info:
|
||||||
|
await _generate_content_with_reliability(op)
|
||||||
|
assert calls == 1
|
||||||
|
assert exc_info.value.failure.category.value == "authentication"
|
||||||
|
assert isinstance(exc_info.value.__cause__, _FakeGenaiError)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_generate_content_reliability_bad_request_is_not_retried():
|
||||||
|
from artemis.agents.explorer.explorer import _generate_content_with_reliability
|
||||||
|
from artemis.llm.reliability import LLMPermanentError
|
||||||
|
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
async def op():
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
raise _FakeGenaiError("invalid request payload", code=400)
|
||||||
|
|
||||||
|
with pytest.raises(LLMPermanentError) as exc_info:
|
||||||
|
await _generate_content_with_reliability(op)
|
||||||
|
assert calls == 1
|
||||||
|
assert exc_info.value.failure.category.value == "bad_request"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_generate_content_reliability_retries_transient_then_succeeds():
|
||||||
|
from artemis.agents.explorer import explorer as explorer_module
|
||||||
|
|
||||||
|
calls = 0
|
||||||
|
sentinel = object()
|
||||||
|
|
||||||
|
async def op():
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
if calls < 2:
|
||||||
|
raise _FakeGenaiError("429 rate limit exceeded", code=429)
|
||||||
|
return sentinel
|
||||||
|
|
||||||
|
with patch.object(explorer_module.asyncio, "sleep", new=AsyncMock()) as sleep_mock:
|
||||||
|
result = await explorer_module._generate_content_with_reliability(op)
|
||||||
|
assert result is sentinel
|
||||||
|
assert calls == 2
|
||||||
|
assert sleep_mock.await_count == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_generate_content_reliability_exhausts_to_typed_error():
|
||||||
|
from artemis.agents.explorer import explorer as explorer_module
|
||||||
|
from artemis.llm.reliability import (
|
||||||
|
FailureCategory,
|
||||||
|
LLMExhaustedError,
|
||||||
|
retry_policy_for,
|
||||||
|
)
|
||||||
|
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
async def op():
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
raise _FakeGenaiError("503 service unavailable", code=503)
|
||||||
|
|
||||||
|
with patch.object(explorer_module.asyncio, "sleep", new=AsyncMock()):
|
||||||
|
with pytest.raises(LLMExhaustedError) as exc_info:
|
||||||
|
await explorer_module._generate_content_with_reliability(op)
|
||||||
|
assert calls == retry_policy_for(FailureCategory.PROVIDER_UNAVAILABLE).max_attempts
|
||||||
|
assert exc_info.value.failure.category.value == "provider_unavailable"
|
||||||
|
assert isinstance(exc_info.value.__cause__, _FakeGenaiError)
|
||||||
@@ -191,3 +191,104 @@ def test_action_timestamps_drive_dense_sampling():
|
|||||||
assert offsets[0] == 2.0
|
assert offsets[0] == 2.0
|
||||||
assert 4.0 in offsets
|
assert 4.0 in offsets
|
||||||
assert offsets[-1] == 6.0
|
assert offsets[-1] == 6.0
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeGenaiError(Exception):
|
||||||
|
"""Mimics google.genai.errors.APIError's shape (a ``code`` attribute)."""
|
||||||
|
|
||||||
|
def __init__(self, message: str, code: int | None = None):
|
||||||
|
super().__init__(message)
|
||||||
|
self.code = code
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_invoke_with_retry_raises_non_retryable_immediately():
|
||||||
|
from artemis.agents.video_analyzer.video_analyzer import _invoke_with_retry
|
||||||
|
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
async def op():
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
raise _FakeGenaiError("invalid api key", code=401)
|
||||||
|
|
||||||
|
with pytest.raises(_FakeGenaiError):
|
||||||
|
await _invoke_with_retry(op, "test-op")
|
||||||
|
assert calls == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_invoke_with_retry_bad_request_never_retried():
|
||||||
|
from artemis.agents.video_analyzer.video_analyzer import _invoke_with_retry
|
||||||
|
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
async def op():
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
raise _FakeGenaiError("invalid request payload", code=400)
|
||||||
|
|
||||||
|
with pytest.raises(_FakeGenaiError):
|
||||||
|
await _invoke_with_retry(op, "test-op")
|
||||||
|
assert calls == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_invoke_with_retry_restarts_from_scratch_on_transient_failure():
|
||||||
|
from artemis.agents.video_analyzer import video_analyzer as va
|
||||||
|
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
async def op():
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
accumulated = "partial-" # per-attempt accumulator, like a stream loop
|
||||||
|
if calls < 2:
|
||||||
|
raise _FakeGenaiError("503 service unavailable", code=503)
|
||||||
|
return accumulated + "complete"
|
||||||
|
|
||||||
|
with patch.object(va.asyncio, "sleep", new=AsyncMock()) as sleep_mock:
|
||||||
|
result = await va._invoke_with_retry(op, "test-op")
|
||||||
|
|
||||||
|
# The successful attempt produced the whole output; the failed attempt's
|
||||||
|
# partial accumulation never leaked into the result.
|
||||||
|
assert result == "partial-complete"
|
||||||
|
assert calls == 2
|
||||||
|
assert sleep_mock.await_count == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_invoke_with_retry_exhaustion_reraises_original_error():
|
||||||
|
from artemis.agents.video_analyzer import video_analyzer as va
|
||||||
|
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
async def op():
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
raise _FakeGenaiError("503 service unavailable", code=503)
|
||||||
|
|
||||||
|
with patch.object(va.asyncio, "sleep", new=AsyncMock()):
|
||||||
|
with pytest.raises(_FakeGenaiError):
|
||||||
|
await va._invoke_with_retry(op, "test-op", max_attempts=2)
|
||||||
|
assert calls == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_invoke_with_retry_caps_unknown_failures_by_policy():
|
||||||
|
from artemis.llm.reliability import FailureCategory, retry_policy_for
|
||||||
|
|
||||||
|
from artemis.agents.video_analyzer import video_analyzer as va
|
||||||
|
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
async def op():
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
raise RuntimeError("something odd")
|
||||||
|
|
||||||
|
with patch.object(va.asyncio, "sleep", new=AsyncMock()):
|
||||||
|
with pytest.raises(RuntimeError):
|
||||||
|
await va._invoke_with_retry(op, "test-op", max_attempts=5)
|
||||||
|
# UNKNOWN failures follow the shared policy, not the caller's outer cap.
|
||||||
|
assert calls == retry_policy_for(FailureCategory.UNKNOWN).max_attempts
|
||||||
@@ -66,3 +66,31 @@ def test_breaker_only_trips_on_transient_categories():
|
|||||||
assert not breaker.allow("m", now=13.6)
|
assert not breaker.allow("m", now=13.6)
|
||||||
breaker.record_success("m")
|
breaker.record_success("m")
|
||||||
assert breaker.allow("m", now=13.7)
|
assert breaker.allow("m", now=13.7)
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_policy_accepts_extension_category_values():
|
||||||
|
from enum import StrEnum
|
||||||
|
|
||||||
|
class ExtCategory(StrEnum):
|
||||||
|
RATE_LIMIT = "rate_limit"
|
||||||
|
MEDIA_PROCESSING = "media_processing"
|
||||||
|
|
||||||
|
assert retry_policy_for(ExtCategory.RATE_LIMIT) is retry_policy_for(
|
||||||
|
FailureCategory.RATE_LIMIT
|
||||||
|
)
|
||||||
|
assert retry_policy_for("provider_unavailable") is retry_policy_for(
|
||||||
|
FailureCategory.PROVIDER_UNAVAILABLE
|
||||||
|
)
|
||||||
|
# Extension-only categories fall back to the UNKNOWN policy; callers gate
|
||||||
|
# on failure.retryable before consulting attempt counts.
|
||||||
|
assert retry_policy_for(ExtCategory.MEDIA_PROCESSING) is retry_policy_for(
|
||||||
|
FailureCategory.UNKNOWN
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_classifier_recognizes_genai_deadline_exceeded_shapes():
|
||||||
|
# google-genai SDK timeouts can surface without a status code attached.
|
||||||
|
for message in ("DEADLINE_EXCEEDED", "deadline exceeded while awaiting response"):
|
||||||
|
failure = classify_failure(RuntimeError(message))
|
||||||
|
assert failure.category is FailureCategory.TIMEOUT
|
||||||
|
assert failure.retryable
|
||||||
Reference in new issue
Block a user