merge(reliability): integrate classified retry handling

This commit is contained in:
somew1nd committed 2026-09-01 11:52:29 -07:00
commit 53e84a5145
6 files changed
+459 -113

No files matched your search

+90
View File
@@ -1272,3 +1272,93 @@ def test_explorer_prune_historical_images():
assert contents[0].parts[1].text == "Some text"
# Second (last) image part should be kept intact
assert contents[1].parts[0] is part2
class _FakeGenaiError(Exception):
"""Mimics google.genai.errors.APIError's shape (a ``code`` attribute)."""
def __init__(self, message: str, code: int | None = None):
super().__init__(message)
self.code = code
@pytest.mark.asyncio
async def test_generate_content_reliability_permanent_error_is_not_retried():
from artemis.agents.explorer.explorer import _generate_content_with_reliability
from artemis.llm.reliability import LLMPermanentError
calls = 0
async def op():
nonlocal calls
calls += 1
raise _FakeGenaiError("PERMISSION_DENIED: forbidden", code=403)
with pytest.raises(LLMPermanentError) as exc_info:
await _generate_content_with_reliability(op)
assert calls == 1
assert exc_info.value.failure.category.value == "authentication"
assert isinstance(exc_info.value.__cause__, _FakeGenaiError)
@pytest.mark.asyncio
async def test_generate_content_reliability_bad_request_is_not_retried():
from artemis.agents.explorer.explorer import _generate_content_with_reliability
from artemis.llm.reliability import LLMPermanentError
calls = 0
async def op():
nonlocal calls
calls += 1
raise _FakeGenaiError("invalid request payload", code=400)
with pytest.raises(LLMPermanentError) as exc_info:
await _generate_content_with_reliability(op)
assert calls == 1
assert exc_info.value.failure.category.value == "bad_request"
@pytest.mark.asyncio
async def test_generate_content_reliability_retries_transient_then_succeeds():
from artemis.agents.explorer import explorer as explorer_module
calls = 0
sentinel = object()
async def op():
nonlocal calls
calls += 1
if calls < 2:
raise _FakeGenaiError("429 rate limit exceeded", code=429)
return sentinel
with patch.object(explorer_module.asyncio, "sleep", new=AsyncMock()) as sleep_mock:
result = await explorer_module._generate_content_with_reliability(op)
assert result is sentinel
assert calls == 2
assert sleep_mock.await_count == 1
@pytest.mark.asyncio
async def test_generate_content_reliability_exhausts_to_typed_error():
from artemis.agents.explorer import explorer as explorer_module
from artemis.llm.reliability import (
FailureCategory,
LLMExhaustedError,
retry_policy_for,
)
calls = 0
async def op():
nonlocal calls
calls += 1
raise _FakeGenaiError("503 service unavailable", code=503)
with patch.object(explorer_module.asyncio, "sleep", new=AsyncMock()):
with pytest.raises(LLMExhaustedError) as exc_info:
await explorer_module._generate_content_with_reliability(op)
assert calls == retry_policy_for(FailureCategory.PROVIDER_UNAVAILABLE).max_attempts
assert exc_info.value.failure.category.value == "provider_unavailable"
assert isinstance(exc_info.value.__cause__, _FakeGenaiError)
+101
View File
@@ -191,3 +191,104 @@ def test_action_timestamps_drive_dense_sampling():
assert offsets[0] == 2.0
assert 4.0 in offsets
assert offsets[-1] == 6.0
class _FakeGenaiError(Exception):
"""Mimics google.genai.errors.APIError's shape (a ``code`` attribute)."""
def __init__(self, message: str, code: int | None = None):
super().__init__(message)
self.code = code
@pytest.mark.asyncio
async def test_invoke_with_retry_raises_non_retryable_immediately():
from artemis.agents.video_analyzer.video_analyzer import _invoke_with_retry
calls = 0
async def op():
nonlocal calls
calls += 1
raise _FakeGenaiError("invalid api key", code=401)
with pytest.raises(_FakeGenaiError):
await _invoke_with_retry(op, "test-op")
assert calls == 1
@pytest.mark.asyncio
async def test_invoke_with_retry_bad_request_never_retried():
from artemis.agents.video_analyzer.video_analyzer import _invoke_with_retry
calls = 0
async def op():
nonlocal calls
calls += 1
raise _FakeGenaiError("invalid request payload", code=400)
with pytest.raises(_FakeGenaiError):
await _invoke_with_retry(op, "test-op")
assert calls == 1
@pytest.mark.asyncio
async def test_invoke_with_retry_restarts_from_scratch_on_transient_failure():
from artemis.agents.video_analyzer import video_analyzer as va
calls = 0
async def op():
nonlocal calls
calls += 1
accumulated = "partial-" # per-attempt accumulator, like a stream loop
if calls < 2:
raise _FakeGenaiError("503 service unavailable", code=503)
return accumulated + "complete"
with patch.object(va.asyncio, "sleep", new=AsyncMock()) as sleep_mock:
result = await va._invoke_with_retry(op, "test-op")
# The successful attempt produced the whole output; the failed attempt's
# partial accumulation never leaked into the result.
assert result == "partial-complete"
assert calls == 2
assert sleep_mock.await_count == 1
@pytest.mark.asyncio
async def test_invoke_with_retry_exhaustion_reraises_original_error():
from artemis.agents.video_analyzer import video_analyzer as va
calls = 0
async def op():
nonlocal calls
calls += 1
raise _FakeGenaiError("503 service unavailable", code=503)
with patch.object(va.asyncio, "sleep", new=AsyncMock()):
with pytest.raises(_FakeGenaiError):
await va._invoke_with_retry(op, "test-op", max_attempts=2)
assert calls == 2
@pytest.mark.asyncio
async def test_invoke_with_retry_caps_unknown_failures_by_policy():
from artemis.llm.reliability import FailureCategory, retry_policy_for
from artemis.agents.video_analyzer import video_analyzer as va
calls = 0
async def op():
nonlocal calls
calls += 1
raise RuntimeError("something odd")
with patch.object(va.asyncio, "sleep", new=AsyncMock()):
with pytest.raises(RuntimeError):
await va._invoke_with_retry(op, "test-op", max_attempts=5)
# UNKNOWN failures follow the shared policy, not the caller's outer cap.
assert calls == retry_policy_for(FailureCategory.UNKNOWN).max_attempts
+28
View File
@@ -66,3 +66,31 @@ def test_breaker_only_trips_on_transient_categories():
assert not breaker.allow("m", now=13.6)
breaker.record_success("m")
assert breaker.allow("m", now=13.7)
def test_retry_policy_accepts_extension_category_values():
from enum import StrEnum
class ExtCategory(StrEnum):
RATE_LIMIT = "rate_limit"
MEDIA_PROCESSING = "media_processing"
assert retry_policy_for(ExtCategory.RATE_LIMIT) is retry_policy_for(
FailureCategory.RATE_LIMIT
)
assert retry_policy_for("provider_unavailable") is retry_policy_for(
FailureCategory.PROVIDER_UNAVAILABLE
)
# Extension-only categories fall back to the UNKNOWN policy; callers gate
# on failure.retryable before consulting attempt counts.
assert retry_policy_for(ExtCategory.MEDIA_PROCESSING) is retry_policy_for(
FailureCategory.UNKNOWN
)
def test_classifier_recognizes_genai_deadline_exceeded_shapes():
# google-genai SDK timeouts can surface without a status code attached.
for message in ("DEADLINE_EXCEEDED", "deadline exceeded while awaiting response"):
failure = classify_failure(RuntimeError(message))
assert failure.category is FailureCategory.TIMEOUT
assert failure.retryable