feat(reliability): centralize retry failure classification

artemis/agents/explorer/explorer.py

@@ -19,7 +19,6 @@ import hashlib
 import json
 import os
 from pathlib import Path
-import random
 import re
 import time
 from typing import Any, Literal
@@ -27,7 +26,6 @@ from typing import Any, Literal
 import cv2
 from google import genai
 from google.genai import types
-from google.genai.errors import APIError
 import httpx
 from langchain_core.messages import (
     BaseMessage,
@@ -44,7 +42,13 @@ from artemis.context import ArtemisContext
 from artemis.data_engine.storage import StorageManager
 from artemis.data_engine.trace import TraceSpan, trace
 from artemis.graph.state import State
-from artemis.services.llm import get_llm
+from artemis.llm.reliability import (
+    LLMExhaustedError,
+    LLMPermanentError,
+    classify_failure,
+    retry_policy_for,
+)
+from artemis.services.llm import _record_llm_event, _record_llm_retry, get_llm
 
 # Import diagnostic functions directly
 from artemis.agents.explorer.constants import EXPLORE_DESCRIPTIONS
@@ -59,6 +63,84 @@ from artemis.utils.visualization import draw_dots, format_minimal_list_with_poin
 
 logger = get_logger(__name__)
 
+
+async def _generate_content_with_reliability(operation, *, label: str = "Explorer model call"):
+    """Run one native google-genai model call under the shared reliability layer.
+
+    Retry decisions are owned by ``classify_failure``/``retry_policy_for``
+    (artemis.llm.reliability) instead of a blanket exponential-backoff loop:
+    non-retryable categories (auth, bad request) are raised immediately as
+    ``LLMPermanentError`` and exhausted retryable categories surface as
+    ``LLMExhaustedError``.  Only the retry/error-classification policy is
+    centralized here; the google-genai transport itself stays untouched.
+    ``operation`` must be a zero-argument callable returning a fresh awaitable
+    per attempt.
+    """
+    attempt = 0
+    while True:
+        try:
+            return await operation()
+        except Exception as call_err:
+            attempt += 1
+            failure = classify_failure(call_err)
+            if not failure.retryable:
+                logger.error(
+                    f"{label} permanently failed [{failure.category.value}]: {call_err}"
+                )
+                _record_llm_event(
+                    "llm_gave_up",
+                    {
+                        "source": "explorer",
+                        "error": str(call_err)[:1000],
+                        "category": failure.category.value,
+                        "retryable": False,
+                    },
+                    status="failed",
+                )
+                raise LLMPermanentError(
+                    f"{label} failed [{failure.category.value}]: {call_err}",
+                    failure=failure,
+                    cause=call_err,
+                ) from call_err
+            policy = retry_policy_for(failure.category)
+            if attempt >= policy.max_attempts:
+                logger.error(
+                    f"{label} exhausted {attempt} attempt(s)"
+                    f" [{failure.category.value}]: {call_err}"
+                )
+                _record_llm_event(
+                    "llm_gave_up",
+                    {
+                        "source": "explorer",
+                        "error": str(call_err)[:1000],
+                        "category": failure.category.value,
+                        "retryable": True,
+                        "attempts": attempt,
+                    },
+                    status="failed",
+                )
+                raise LLMExhaustedError(
+                    f"{label} exhausted {attempt} attempt(s)"
+                    f" [{failure.category.value}]: {call_err}",
+                    failure=failure,
+                    cause=call_err,
+                ) from call_err
+            delay = policy.delay_for(attempt)
+            logger.warning(
+                f"{label} failed [{failure.category.value}] on attempt"
+                f" {attempt}/{policy.max_attempts}: {call_err}."
+                f" Retrying in {delay:.2f}s..."
+            )
+            _record_llm_retry(
+                str(call_err),
+                delay,
+                attempt=attempt,
+                max_retries=policy.max_attempts,
+                source="explorer",
+            )
+            await asyncio.sleep(delay)
+
+
 UNIVERSAL_EXPLORER_TOOLS = [
     {
         "type": "function",
@@ -1802,41 +1884,16 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
                             ),
                         )
 
-                    max_call_retries = 5
-                    for attempt in range(max_call_retries + 1):
-                        try:
-                            response = await asyncio.wait_for(
-                                client.aio.models.generate_content(
-                                    model=model_name,
-                                    contents=contents,
-                                    config=generate_config,
-                                ),
-                                timeout=180,
-                            )
-                            break
-                        except APIError as api_err:
-                            code = getattr(api_err, "code", None)
-                            if code in [429, 503]:
-                                if attempt >= max_call_retries:
-                                    raise api_err
-                                backoff = (2.0**attempt) + random.uniform(0.1, 1.0)
-                                logger.warning(
-                                    f"APIError {code} (overload/rate limit) on"
-                                    f" attempt {attempt + 1}. Retrying in"
-                                    f" {backoff:.2f}s..."
-                                )
-                                await asyncio.sleep(backoff)
-                            else:
-                                raise api_err
-                        except Exception as e:
-                            if attempt >= max_call_retries:
-                                raise e
-                            backoff = (2.0**attempt) + random.uniform(0.1, 1.0)
-                            logger.warning(
-                                f"Unexpected error on attempt {attempt + 1}:"
-                                f" {e}. Retrying in {backoff:.2f}s..."
-                            )
-                            await asyncio.sleep(backoff)
+                    response = await _generate_content_with_reliability(
+                        lambda: asyncio.wait_for(
+                            client.aio.models.generate_content(
+                                model=model_name,
+                                contents=contents,
+                                config=generate_config,
+                            ),
+                            timeout=180,
+                        ),
+                    )
                     end_turn = time.perf_counter()
                     turn_dur = end_turn - start_turn
                     self.turn_latencies.append(turn_dur)

artemis/agents/video_analyzer/video_analyzer.py

@@ -19,7 +19,6 @@ import glob
 import json
 import os
 from pathlib import Path
-import random
 import re
 import shutil
 import tempfile
@@ -28,7 +27,6 @@ from typing import Any
 
 from google import genai
 from google.genai import types
-from google.genai.errors import APIError
 from langchain_core.messages import (
     BaseMessage,
     HumanMessage,
@@ -49,7 +47,8 @@ from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
 from artemis.context import ArtemisContext
 from artemis.controllers.controller_factory import get_controller
 from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace
-from artemis.services.llm import get_llm
+from artemis.llm.reliability import retry_policy_for
+from artemis.services.llm import _record_llm_event, _record_llm_retry, get_llm
 from artemis.utils.logger import get_logger
 from artemis.utils.video import (
     compress_video_for_api,
@@ -76,27 +75,50 @@ _CLEANUP_LOCK = asyncio.Lock()
 
 
 async def _invoke_with_retry(operation, label: str, max_attempts: int = 3):
-    """Retry side-effect-free model coordination calls on transient failures."""
-    for attempt in range(1, max_attempts + 1):
+    """Retry side-effect-free model coordination calls using shared classification.
+
+    Retry decisions come from ``classify_video_failure`` plus the shared
+    per-category policies in :mod:`artemis.llm.reliability` instead of ad-hoc
+    status-code sets.  ``operation`` must be restartable from scratch: each
+    retry re-invokes it and any partially accumulated result (e.g. consumed
+    stream chunks) from the failed attempt is discarded, never surfaced as a
+    complete output.  The original exception is re-raised on giving up so
+    downstream ``classify_video_failure`` consumers keep seeing the provider's
+    native error shape.
+    """
+    attempt = 0
+    while True:
+        attempt += 1
         try:
             return await operation()
-        except APIError as exc:
-            code = getattr(exc, "code", None)
-            if code in {400, 401, 403, 404} or attempt >= max_attempts:
+        except Exception as exc:
+            failure = classify_video_failure(exc)
+            if not failure.retryable:
+                logger.warning(
+                    f"{label} non-retryable failure [{failure.category.value}] "
+                    f"on attempt {attempt}: {exc}"
+                )
                 raise
-            delay = (2.0 ** (attempt - 1)) + random.uniform(0.1, 0.8)
+            policy = retry_policy_for(failure.category)
+            allowed_attempts = min(max_attempts, policy.max_attempts)
+            if attempt >= allowed_attempts:
+                logger.warning(
+                    f"{label} exhausted {attempt} attempt(s) "
+                    f"[{failure.category.value}]: {exc}"
+                )
+                raise
+            delay = policy.delay_for(attempt)
             logger.warning(
-                f"{label} transient API failure ({attempt}/{max_attempts}): {exc}; "
-                f"retrying in {delay:.2f}s"
+                f"{label} retryable failure [{failure.category.value}] "
+                f"({attempt}/{allowed_attempts}): {exc}; retrying from scratch "
+                f"in {delay:.2f}s (partial output from the failed attempt is discarded)"
             )
-            await asyncio.sleep(delay)
-        except (TimeoutError, ConnectionError, OSError) as exc:
-            if attempt >= max_attempts:
-                raise
-            delay = (2.0 ** (attempt - 1)) + random.uniform(0.1, 0.8)
-            logger.warning(
-                f"{label} transient failure ({attempt}/{max_attempts}): {exc}; "
-                f"retrying in {delay:.2f}s"
+            _record_llm_retry(
+                str(exc),
+                delay,
+                attempt=attempt,
+                max_retries=allowed_attempts,
+                source=label,
             )
             await asyncio.sleep(delay)
 
@@ -1094,6 +1116,14 @@ class VideoAnalyzer:
                     logger.warning(
                         "Native video chunk exhausted retries; using universal fallback"
                     )
+                    _record_llm_event(
+                        "llm_fallback",
+                        {
+                            "reason": "native_video_chunk_failed",
+                            "category": failure.category.value,
+                            "error": str(error)[:500],
+                        },
+                    )
                     fallback_result = await self._exec_single_chunk_universal(
                         compressed_path=compressed_path,
                         raw_path=path,
@@ -1697,35 +1727,43 @@ class VideoAnalyzer:
                         except Exception as ce:
                             logger.error(f"Failed to delete cloud file {file.name}: {ce}")
 
-            except APIError as api_err:
-                logger.warning(f"Sub-agent APIError attempt {attempt + 1} failed: {api_err}")
-                code = getattr(api_err, "code", None)
-                if code == 429:
-                    if attempt >= max_retries:
-                        return await finish_failed(api_err)
-
-                    backoff = (2.0**attempt) + random.uniform(0.1, 1.0)
-                    logger.info(
-                        f"Rate limited (429). Backing off for {backoff:.2f}s"
-                        " without expanding boundaries..."
-                    )
-                    await asyncio.sleep(backoff)
-                    continue
-                elif code in [400, 401, 403, 404]:
-                    return await finish_failed(api_err, retryable=False)
-
-                if attempt >= max_retries:
-                    return await finish_failed(api_err)
-                backoff = 2.0**attempt
-                await asyncio.sleep(backoff)
             except Exception as e:
-                logger.warning(f"Sub-agent attempt {attempt + 1} failed: {e}")
-                if attempt >= max_retries:
+                # Retry decisions are owned by the shared classification layer.
+                # Every attempt restarts from scratch (segment re-extracted,
+                # file re-uploaded, stream re-consumed); partially accumulated
+                # stream output from the failed attempt is discarded, never
+                # surfaced as a complete answer.
+                failure = classify_video_failure(e)
+                attempts_so_far = attempt + 1
+                if not failure.retryable:
+                    logger.warning(
+                        f"Sub-agent non-retryable failure [{failure.category.value}]"
+                        f" on attempt {attempts_so_far}: {e}"
+                    )
+                    return await finish_failed(e, retryable=False)
+                policy = retry_policy_for(failure.category)
+                allowed_attempts = min(max_retries + 1, policy.max_attempts)
+                if attempts_so_far >= allowed_attempts:
+                    logger.warning(
+                        f"Sub-agent exhausted {attempts_so_far} attempt(s)"
+                        f" [{failure.category.value}]: {e}"
+                    )
                     return await finish_failed(e)
-
-                backoff = 2.0**attempt
-                logger.info(f"Backing off for {backoff}s before retry...")
-                await asyncio.sleep(backoff)
+                delay = policy.delay_for(attempts_so_far)
+                logger.warning(
+                    f"Sub-agent retryable failure [{failure.category.value}]"
+                    f" ({attempts_so_far}/{allowed_attempts}): {e}; retrying"
+                    f" from scratch in {delay:.2f}s (partial stream output"
+                    " from the failed attempt is discarded)"
+                )
+                _record_llm_retry(
+                    str(e),
+                    delay,
+                    attempt=attempts_so_far,
+                    max_retries=allowed_attempts,
+                    source="video_sub_agent",
+                )
+                await asyncio.sleep(delay)
 
         raise RuntimeError(
             f"Video sub-agent exhausted retries for {current_start:.1f}s-{current_end:.1f}s"
@@ -1803,6 +1841,18 @@ class VideoAnalyzer:
                 and prompt_with_context is not None
             ):
                 try:
+                    logger.warning(
+                        "Native audio analysis failed"
+                        f" [{failure.category.value}]; using universal fallback"
+                    )
+                    _record_llm_event(
+                        "llm_fallback",
+                        {
+                            "reason": "native_audio_analysis_failed",
+                            "category": failure.category.value,
+                            "error": str(error)[:500],
+                        },
+                    )
                     fallback_result = await self._exec_analyze_audio_universal(
                         audio_path=audio_path,
                         start_time=current_start,
@@ -2135,35 +2185,40 @@ class VideoAnalyzer:
                         except Exception as ce:
                             logger.error(f"Failed to delete cloud file {file.name}: {ce}")
 
-            except APIError as api_err:
-                logger.warning(f"Audio APIError attempt {attempt + 1} failed: {api_err}")
-                code = getattr(api_err, "code", None)
-                if code == 429:
-                    if attempt >= max_retries:
-                        return await finish_failed_audio(api_err)
-
-                    backoff = (2.0**attempt) + random.uniform(0.1, 1.0)
-                    logger.info(
-                        f"Rate limited (429). Backing off for {backoff:.2f}s"
-                        " without expanding boundaries..."
-                    )
-                    await asyncio.sleep(backoff)
-                    continue
-                elif code in [400, 401, 403, 404]:
-                    return await finish_failed_audio(api_err, retryable=False)
-
-                if attempt >= max_retries:
-                    return await finish_failed_audio(api_err)
-                backoff = 2.0**attempt
-                await asyncio.sleep(backoff)
             except Exception as e:
-                logger.warning(f"Audio attempt {attempt + 1} failed: {e}")
-                if attempt >= max_retries:
+                # Same contract as the video sub-agent loop: classification
+                # owns the retry decision and every retry restarts the attempt
+                # from scratch.
+                failure = classify_video_failure(e)
+                attempts_so_far = attempt + 1
+                if not failure.retryable:
+                    logger.warning(
+                        f"Audio non-retryable failure [{failure.category.value}]"
+                        f" on attempt {attempts_so_far}: {e}"
+                    )
+                    return await finish_failed_audio(e, retryable=False)
+                policy = retry_policy_for(failure.category)
+                allowed_attempts = min(max_retries + 1, policy.max_attempts)
+                if attempts_so_far >= allowed_attempts:
+                    logger.warning(
+                        f"Audio analysis exhausted {attempts_so_far} attempt(s)"
+                        f" [{failure.category.value}]: {e}"
+                    )
                     return await finish_failed_audio(e)
-
-                backoff = 2.0**attempt
-                logger.info(f"Backing off for {backoff}s before retry...")
-                await asyncio.sleep(backoff)
+                delay = policy.delay_for(attempts_so_far)
+                logger.warning(
+                    f"Audio retryable failure [{failure.category.value}]"
+                    f" ({attempts_so_far}/{allowed_attempts}): {e}; retrying"
+                    f" from scratch in {delay:.2f}s"
+                )
+                _record_llm_retry(
+                    str(e),
+                    delay,
+                    attempt=attempts_so_far,
+                    max_retries=allowed_attempts,
+                    source="video_audio_agent",
+                )
+                await asyncio.sleep(delay)
 
     async def _exec_single_chunk_universal(
         self,

artemis/llm/reliability.py

@@ -77,6 +77,9 @@ def extract_status_code(error: BaseException) -> int | None:
 _RATE_LIMIT_MARKERS = ("rate limit", "quota", "resource_exhausted")
 _UNAVAILABLE_MARKERS = ("unavailable", "overloaded", "high demand", "service unavailable")
 _AUTH_MARKERS = ("unauthorized", "forbidden", "api key", "credential")
+# "deadline exceeded"/"deadline_exceeded" cover google-genai SDK timeouts that
+# surface without an HTTP status code attached.
+_TIMEOUT_MARKERS = ("timed out", "timeout", "deadline exceeded", "deadline_exceeded")
 
 
 def classify_failure(error: BaseException) -> Failure:
@@ -91,7 +94,7 @@ def classify_failure(error: BaseException) -> Failure:
         return Failure(FailureCategory.RATE_LIMIT, True, True)
     if code in {500, 502, 503, 504} or any(marker in message for marker in _UNAVAILABLE_MARKERS):
         return Failure(FailureCategory.PROVIDER_UNAVAILABLE, True, True)
-    if isinstance(error, TimeoutError) or "timed out" in message or "timeout" in message:
+    if isinstance(error, TimeoutError) or any(marker in message for marker in _TIMEOUT_MARKERS):
         return Failure(FailureCategory.TIMEOUT, True, True)
     if isinstance(error, (ConnectionError, OSError)):
         return Failure(FailureCategory.CONNECTION, True, True)
@@ -163,7 +166,19 @@ _DEFAULT_RETRY_POLICIES: dict[FailureCategory, RetryPolicy] = {
 }
 
 
-def retry_policy_for(category: FailureCategory) -> RetryPolicy:
+def retry_policy_for(category: FailureCategory | str) -> RetryPolicy:
+    """Look up the retry policy for a failure category.
+
+    Accepts the generic ``FailureCategory`` or any string-valued category from
+    a domain extension (e.g. the video pipeline's ``VideoFailureCategory``).
+    Extension-only categories fall back to the UNKNOWN policy, so callers must
+    gate on ``failure.retryable`` before consulting attempt counts.
+    """
+    if not isinstance(category, FailureCategory):
+        try:
+            category = FailureCategory(str(category))
+        except ValueError:
+            category = FailureCategory.UNKNOWN
     return _DEFAULT_RETRY_POLICIES.get(category, _DEFAULT_RETRY_POLICIES[FailureCategory.UNKNOWN])