merge(reliability): integrate classified retry handling
This commit is contained in:
commit
53e84a5145
6 files changed
+459
-113
No files matched your search
@@ -1272,3 +1272,93 @@ def test_explorer_prune_historical_images():
|
||||
assert contents[0].parts[1].text == "Some text"
|
||||
# Second (last) image part should be kept intact
|
||||
assert contents[1].parts[0] is part2
|
||||
|
||||
|
||||
class _FakeGenaiError(Exception):
|
||||
"""Mimics google.genai.errors.APIError's shape (a ``code`` attribute)."""
|
||||
|
||||
def __init__(self, message: str, code: int | None = None):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_content_reliability_permanent_error_is_not_retried():
|
||||
from artemis.agents.explorer.explorer import _generate_content_with_reliability
|
||||
from artemis.llm.reliability import LLMPermanentError
|
||||
|
||||
calls = 0
|
||||
|
||||
async def op():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise _FakeGenaiError("PERMISSION_DENIED: forbidden", code=403)
|
||||
|
||||
with pytest.raises(LLMPermanentError) as exc_info:
|
||||
await _generate_content_with_reliability(op)
|
||||
assert calls == 1
|
||||
assert exc_info.value.failure.category.value == "authentication"
|
||||
assert isinstance(exc_info.value.__cause__, _FakeGenaiError)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_content_reliability_bad_request_is_not_retried():
|
||||
from artemis.agents.explorer.explorer import _generate_content_with_reliability
|
||||
from artemis.llm.reliability import LLMPermanentError
|
||||
|
||||
calls = 0
|
||||
|
||||
async def op():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise _FakeGenaiError("invalid request payload", code=400)
|
||||
|
||||
with pytest.raises(LLMPermanentError) as exc_info:
|
||||
await _generate_content_with_reliability(op)
|
||||
assert calls == 1
|
||||
assert exc_info.value.failure.category.value == "bad_request"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_content_reliability_retries_transient_then_succeeds():
|
||||
from artemis.agents.explorer import explorer as explorer_module
|
||||
|
||||
calls = 0
|
||||
sentinel = object()
|
||||
|
||||
async def op():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls < 2:
|
||||
raise _FakeGenaiError("429 rate limit exceeded", code=429)
|
||||
return sentinel
|
||||
|
||||
with patch.object(explorer_module.asyncio, "sleep", new=AsyncMock()) as sleep_mock:
|
||||
result = await explorer_module._generate_content_with_reliability(op)
|
||||
assert result is sentinel
|
||||
assert calls == 2
|
||||
assert sleep_mock.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_content_reliability_exhausts_to_typed_error():
|
||||
from artemis.agents.explorer import explorer as explorer_module
|
||||
from artemis.llm.reliability import (
|
||||
FailureCategory,
|
||||
LLMExhaustedError,
|
||||
retry_policy_for,
|
||||
)
|
||||
|
||||
calls = 0
|
||||
|
||||
async def op():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise _FakeGenaiError("503 service unavailable", code=503)
|
||||
|
||||
with patch.object(explorer_module.asyncio, "sleep", new=AsyncMock()):
|
||||
with pytest.raises(LLMExhaustedError) as exc_info:
|
||||
await explorer_module._generate_content_with_reliability(op)
|
||||
assert calls == retry_policy_for(FailureCategory.PROVIDER_UNAVAILABLE).max_attempts
|
||||
assert exc_info.value.failure.category.value == "provider_unavailable"
|
||||
assert isinstance(exc_info.value.__cause__, _FakeGenaiError)
|
||||
@@ -191,3 +191,104 @@ def test_action_timestamps_drive_dense_sampling():
|
||||
assert offsets[0] == 2.0
|
||||
assert 4.0 in offsets
|
||||
assert offsets[-1] == 6.0
|
||||
|
||||
|
||||
class _FakeGenaiError(Exception):
|
||||
"""Mimics google.genai.errors.APIError's shape (a ``code`` attribute)."""
|
||||
|
||||
def __init__(self, message: str, code: int | None = None):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoke_with_retry_raises_non_retryable_immediately():
|
||||
from artemis.agents.video_analyzer.video_analyzer import _invoke_with_retry
|
||||
|
||||
calls = 0
|
||||
|
||||
async def op():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise _FakeGenaiError("invalid api key", code=401)
|
||||
|
||||
with pytest.raises(_FakeGenaiError):
|
||||
await _invoke_with_retry(op, "test-op")
|
||||
assert calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoke_with_retry_bad_request_never_retried():
|
||||
from artemis.agents.video_analyzer.video_analyzer import _invoke_with_retry
|
||||
|
||||
calls = 0
|
||||
|
||||
async def op():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise _FakeGenaiError("invalid request payload", code=400)
|
||||
|
||||
with pytest.raises(_FakeGenaiError):
|
||||
await _invoke_with_retry(op, "test-op")
|
||||
assert calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoke_with_retry_restarts_from_scratch_on_transient_failure():
|
||||
from artemis.agents.video_analyzer import video_analyzer as va
|
||||
|
||||
calls = 0
|
||||
|
||||
async def op():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
accumulated = "partial-" # per-attempt accumulator, like a stream loop
|
||||
if calls < 2:
|
||||
raise _FakeGenaiError("503 service unavailable", code=503)
|
||||
return accumulated + "complete"
|
||||
|
||||
with patch.object(va.asyncio, "sleep", new=AsyncMock()) as sleep_mock:
|
||||
result = await va._invoke_with_retry(op, "test-op")
|
||||
|
||||
# The successful attempt produced the whole output; the failed attempt's
|
||||
# partial accumulation never leaked into the result.
|
||||
assert result == "partial-complete"
|
||||
assert calls == 2
|
||||
assert sleep_mock.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoke_with_retry_exhaustion_reraises_original_error():
|
||||
from artemis.agents.video_analyzer import video_analyzer as va
|
||||
|
||||
calls = 0
|
||||
|
||||
async def op():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise _FakeGenaiError("503 service unavailable", code=503)
|
||||
|
||||
with patch.object(va.asyncio, "sleep", new=AsyncMock()):
|
||||
with pytest.raises(_FakeGenaiError):
|
||||
await va._invoke_with_retry(op, "test-op", max_attempts=2)
|
||||
assert calls == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invoke_with_retry_caps_unknown_failures_by_policy():
|
||||
from artemis.llm.reliability import FailureCategory, retry_policy_for
|
||||
|
||||
from artemis.agents.video_analyzer import video_analyzer as va
|
||||
|
||||
calls = 0
|
||||
|
||||
async def op():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise RuntimeError("something odd")
|
||||
|
||||
with patch.object(va.asyncio, "sleep", new=AsyncMock()):
|
||||
with pytest.raises(RuntimeError):
|
||||
await va._invoke_with_retry(op, "test-op", max_attempts=5)
|
||||
# UNKNOWN failures follow the shared policy, not the caller's outer cap.
|
||||
assert calls == retry_policy_for(FailureCategory.UNKNOWN).max_attempts
|
||||
@@ -66,3 +66,31 @@ def test_breaker_only_trips_on_transient_categories():
|
||||
assert not breaker.allow("m", now=13.6)
|
||||
breaker.record_success("m")
|
||||
assert breaker.allow("m", now=13.7)
|
||||
|
||||
|
||||
def test_retry_policy_accepts_extension_category_values():
|
||||
from enum import StrEnum
|
||||
|
||||
class ExtCategory(StrEnum):
|
||||
RATE_LIMIT = "rate_limit"
|
||||
MEDIA_PROCESSING = "media_processing"
|
||||
|
||||
assert retry_policy_for(ExtCategory.RATE_LIMIT) is retry_policy_for(
|
||||
FailureCategory.RATE_LIMIT
|
||||
)
|
||||
assert retry_policy_for("provider_unavailable") is retry_policy_for(
|
||||
FailureCategory.PROVIDER_UNAVAILABLE
|
||||
)
|
||||
# Extension-only categories fall back to the UNKNOWN policy; callers gate
|
||||
# on failure.retryable before consulting attempt counts.
|
||||
assert retry_policy_for(ExtCategory.MEDIA_PROCESSING) is retry_policy_for(
|
||||
FailureCategory.UNKNOWN
|
||||
)
|
||||
|
||||
|
||||
def test_classifier_recognizes_genai_deadline_exceeded_shapes():
|
||||
# google-genai SDK timeouts can surface without a status code attached.
|
||||
for message in ("DEADLINE_EXCEEDED", "deadline exceeded while awaiting response"):
|
||||
failure = classify_failure(RuntimeError(message))
|
||||
assert failure.category is FailureCategory.TIMEOUT
|
||||
assert failure.retryable
|
||||
Reference in new issue
Block a user