feat: harden agent verification and runtime contracts
Introduce two-gate checker settlement, explicit graph visibility and checkpoints, structured LLM reliability, centralized action specifications, and corresponding tests and design documentation.
This commit is contained in:
somew1nd committed
1 parent
ddfa420945
commit
cbe8f4f0c6
106 files changed
+11542
-4445
No files matched your search
@@ -12,15 +12,24 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Failure classification and circuit breaking for video model calls."""
|
||||
"""Failure classification and circuit breaking for video model calls.
|
||||
|
||||
Thin media-aware extension of :mod:`artemis.llm.reliability`, which owns the
|
||||
generic taxonomy and circuit breaker. This module adds the media-processing
|
||||
category and the ``should_split`` recovery decision (bisecting long clips).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from artemis.llm.reliability import (
|
||||
CircuitBreaker,
|
||||
FailureCategory,
|
||||
classify_failure,
|
||||
extract_status_code,
|
||||
)
|
||||
|
||||
|
||||
class VideoFailureCategory(StrEnum):
|
||||
@@ -43,115 +52,37 @@ class VideoFailure:
|
||||
should_fallback: bool
|
||||
|
||||
|
||||
_MEDIA_MARKERS = ("ffmpeg", "codec", "corrupt", "invalid video", "invalid audio", "media")
|
||||
|
||||
|
||||
def classify_video_failure(error: BaseException) -> VideoFailure:
|
||||
"""Convert provider/transport/media exceptions into recovery decisions."""
|
||||
|
||||
if isinstance(error, KeyboardInterrupt) or type(error).__name__ == "CancelledError":
|
||||
return VideoFailure(VideoFailureCategory.CANCELLED, False, False, False)
|
||||
|
||||
code = getattr(error, "code", None) or getattr(error, "status_code", None)
|
||||
try:
|
||||
code = int(code) if code is not None else None
|
||||
except (TypeError, ValueError):
|
||||
code = None
|
||||
|
||||
generic = classify_failure(error)
|
||||
message = str(error).lower()
|
||||
if code == 429 or any(marker in message for marker in ("rate limit", "quota", "resource_exhausted")):
|
||||
return VideoFailure(VideoFailureCategory.RATE_LIMIT, True, False, True)
|
||||
if code in {500, 502, 503, 504} or any(
|
||||
marker in message
|
||||
for marker in ("unavailable", "overloaded", "high demand", "service unavailable")
|
||||
):
|
||||
return VideoFailure(VideoFailureCategory.PROVIDER_UNAVAILABLE, True, False, True)
|
||||
if isinstance(error, TimeoutError) or "timed out" in message or "timeout" in message:
|
||||
return VideoFailure(VideoFailureCategory.TIMEOUT, True, True, True)
|
||||
if isinstance(error, (ConnectionError, OSError)) and not any(
|
||||
marker in message for marker in ("ffmpeg", "video", "audio", "codec", "media")
|
||||
):
|
||||
return VideoFailure(VideoFailureCategory.CONNECTION, True, False, True)
|
||||
if code in {401, 403} or any(
|
||||
marker in message for marker in ("unauthorized", "forbidden", "api key", "credential")
|
||||
):
|
||||
return VideoFailure(VideoFailureCategory.AUTHENTICATION, False, False, True)
|
||||
if any(
|
||||
marker in message
|
||||
for marker in ("ffmpeg", "codec", "corrupt", "invalid video", "invalid audio", "media")
|
||||
):
|
||||
has_media_marker = any(marker in message for marker in _MEDIA_MARKERS)
|
||||
|
||||
# Media markers take precedence over generic connection / bad-request /
|
||||
# unknown classification (rate-limit, unavailable, timeout, and auth
|
||||
# signals still win, matching the original ordering of checks).
|
||||
if has_media_marker and generic.category in {
|
||||
FailureCategory.CONNECTION,
|
||||
FailureCategory.BAD_REQUEST,
|
||||
FailureCategory.UNKNOWN,
|
||||
}:
|
||||
return VideoFailure(VideoFailureCategory.MEDIA_PROCESSING, False, True, True)
|
||||
if code in {400, 404, 413, 415, 422}:
|
||||
return VideoFailure(VideoFailureCategory.BAD_REQUEST, False, code == 413, True)
|
||||
return VideoFailure(VideoFailureCategory.UNKNOWN, True, True, True)
|
||||
|
||||
category = VideoFailureCategory(generic.category.value)
|
||||
should_split = category in {VideoFailureCategory.TIMEOUT, VideoFailureCategory.UNKNOWN} or (
|
||||
category is VideoFailureCategory.BAD_REQUEST and extract_status_code(error) == 413
|
||||
)
|
||||
# The video pipeline keeps its historical behavior of falling back to the
|
||||
# universal analyzer even for bad requests (a different provider may accept
|
||||
# the same media payload).
|
||||
should_fallback = generic.should_fallback or category is VideoFailureCategory.BAD_REQUEST
|
||||
if category is VideoFailureCategory.CANCELLED:
|
||||
should_fallback = False
|
||||
return VideoFailure(category, generic.retryable, should_split, should_fallback)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _CircuitState:
|
||||
consecutive_failures: int = 0
|
||||
open_until: float = 0.0
|
||||
half_open_in_flight: bool = False
|
||||
|
||||
|
||||
class VideoCircuitBreaker:
|
||||
"""Small context-scoped circuit breaker keyed by provider/model identity."""
|
||||
|
||||
def __init__(self, threshold: int = 3, cooldown_seconds: float = 60.0) -> None:
|
||||
self.threshold = max(1, int(threshold))
|
||||
self.cooldown_seconds = max(0.1, float(cooldown_seconds))
|
||||
self._states: dict[str, _CircuitState] = {}
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def allow(self, key: str, *, now: float | None = None) -> bool:
|
||||
current = time.time() if now is None else now
|
||||
with self._lock:
|
||||
state = self._states.get(key)
|
||||
if state is None:
|
||||
return True
|
||||
if state.open_until:
|
||||
if current < state.open_until or state.half_open_in_flight:
|
||||
return False
|
||||
# Half-open: permit exactly one trial. Other concurrent calls
|
||||
# continue to bypass the primary until it succeeds or fails.
|
||||
state.half_open_in_flight = True
|
||||
state.consecutive_failures = max(0, self.threshold - 1)
|
||||
return True
|
||||
return not state.half_open_in_flight
|
||||
|
||||
def record_success(self, key: str) -> None:
|
||||
with self._lock:
|
||||
self._states.pop(key, None)
|
||||
|
||||
def record_failure(
|
||||
self,
|
||||
key: str,
|
||||
failure: VideoFailure,
|
||||
*,
|
||||
now: float | None = None,
|
||||
) -> None:
|
||||
if failure.category not in {
|
||||
VideoFailureCategory.RATE_LIMIT,
|
||||
VideoFailureCategory.PROVIDER_UNAVAILABLE,
|
||||
VideoFailureCategory.TIMEOUT,
|
||||
VideoFailureCategory.CONNECTION,
|
||||
}:
|
||||
with self._lock:
|
||||
state = self._states.get(key)
|
||||
if state is not None and state.half_open_in_flight:
|
||||
self._states.pop(key, None)
|
||||
return
|
||||
current = time.time() if now is None else now
|
||||
with self._lock:
|
||||
state = self._states.setdefault(key, _CircuitState())
|
||||
state.half_open_in_flight = False
|
||||
state.consecutive_failures += 1
|
||||
if state.consecutive_failures >= self.threshold:
|
||||
state.open_until = current + self.cooldown_seconds
|
||||
|
||||
def snapshot(self) -> dict[str, dict[str, Any]]:
|
||||
with self._lock:
|
||||
return {
|
||||
key: {
|
||||
"consecutive_failures": state.consecutive_failures,
|
||||
"open_until": state.open_until,
|
||||
"half_open_in_flight": state.half_open_in_flight,
|
||||
}
|
||||
for key, state in self._states.items()
|
||||
}
|
||||
class VideoCircuitBreaker(CircuitBreaker):
|
||||
"""Context-scoped circuit breaker keyed by provider/model identity."""
|
||||
Reference in new issue
Block a user