feat: enable full AI automation and safe startup updates

artemis/services/llm.py

@@ -99,6 +99,9 @@ def _begin_llm_request(provider: str | None) -> Token:
 # behind this call (set by with_fallback).  When a fallback exists, exhausting
 # retries hands over to it immediately instead of pausing the whole task.
 _FALLBACK_AVAILABLE: ContextVar[bool] = ContextVar("llm_fallback_available", default=False)
+_FAILED_ENDPOINTS: ContextVar[dict[str, Exception] | None] = ContextVar(
+    "llm_failed_endpoints", default=None
+)
 
 # Shared circuit breaker keyed by "provider:model".  Transient failures open
 # it; while open, concurrent calls wait out the cooldown instead of hammering
@@ -109,9 +112,6 @@ _ENDPOINT_BREAKER = CircuitBreaker(threshold=3, cooldown_seconds=30.0)
 # remembered so subsequent calls go straight to non-streaming invocation.
 _NON_STREAMING_ENDPOINTS: set[str] = set()
 
-# Total retry limit when failures switch categories within one recovery cycle.
-_MAX_TOTAL_RETRY_ATTEMPTS = 8
-
 
 def _get_current_data_engine():
     if _CURRENT_DATA_ENGINE is not None:
@@ -331,6 +331,8 @@ async def _run_with_recovery[T](
     *,
     provider: str | None = None,
     endpoint_key: str | None = None,
+    request_timeout: float | None = None,
+    max_retries: int | None = None,
 ) -> T:
     """Execute one LLM call under the classified retry / pause / breaker policy.
 
@@ -353,10 +355,22 @@ async def _run_with_recovery[T](
         pause_file: Path | None = None
         try:
             while True:
-                if endpoint_key:
+                failed_endpoints = _FAILED_ENDPOINTS.get()
+                previous = (
+                    failed_endpoints.get(endpoint_key)
+                    if failed_endpoints is not None and endpoint_key
+                    else None
+                )
+                if previous is not None:
+                    last_error, last_failure = previous, classify_failure(previous)
+                    if not last_failure.retryable:
+                        raise LLMPermanentError(str(previous), failure=last_failure, cause=previous)
+                    break
+                if endpoint_key and not attempts:
                     await _wait_for_breaker(endpoint_key)
                 try:
-                    result = await call_once()
+                    async with asyncio.timeout(request_timeout or settings.OPENAI_TIMEOUT_SECONDS):
+                        result = await call_once()
                 except (KeyboardInterrupt, asyncio.CancelledError):
                     raise
                 except LLMCallError:
@@ -369,6 +383,8 @@ async def _run_with_recovery[T](
                     if endpoint_key:
                         _ENDPOINT_BREAKER.record_failure(endpoint_key, failure)
                     if not failure.retryable:
+                        if failed_endpoints is not None and endpoint_key:
+                            failed_endpoints[endpoint_key] = e
                         llm_logger.warning(
                             f"LLM call failed permanently ({failure.category.value}): {e}"
                         )
@@ -386,21 +402,18 @@ async def _run_with_recovery[T](
                     attempt = attempts.get(failure.category, 0) + 1
                     attempts[failure.category] = attempt
                     policy = retry_policy_for(failure.category)
-                    if attempt >= policy.max_attempts:
-                        break
                     total_attempts = sum(attempts.values())
-                    if total_attempts >= _MAX_TOTAL_RETRY_ATTEMPTS:
-                        llm_logger.warning(
-                            "LLM retry total across failure categories reached"
-                            f" {total_attempts}; treating as exhausted."
-                        )
+                    max_attempts = (
+                        max_retries + 1 if max_retries is not None else policy.max_attempts
+                    )
+                    if total_attempts >= max_attempts:
                         break
                     delay = policy.delay_for(attempt)
                     _record_llm_retry(
                         str(e),
                         delay,
-                        attempt=attempt,
-                        max_retries=policy.max_attempts,
+                        attempt=total_attempts,
+                        max_retries=max_attempts - 1,
                         provider=provider,
                     )
                     llm_logger.warning(
@@ -414,6 +427,8 @@ async def _run_with_recovery[T](
 
             # Retryable attempts exhausted.
             if _FALLBACK_AVAILABLE.get():
+                if failed_endpoints is not None and endpoint_key:
+                    failed_endpoints[endpoint_key] = last_error
                 _record_llm_event(
                     "llm_gave_up",
                     {
@@ -445,6 +460,8 @@ async def _run_with_recovery[T](
             ) from last_error
 
         llm_logger.info("Resume signal received, retrying LLM call...")
+        if failed_endpoints is not None and endpoint_key:
+            failed_endpoints.pop(endpoint_key, None)
         current_engine = _get_current_data_engine()
         if current_engine:
             current_engine._publish("task_resumed", {})
@@ -540,8 +557,8 @@ class RobustChatModelWrapper:
 
     def _endpoint_key(self) -> str:
         if self.endpoint is not None:
-            return f"{self._provider_value()}:{self.endpoint.model_name}"
-        return type(self.base_model).__name__
+            return f"{self._provider_value()}:{self.endpoint.model_name}:{self.endpoint.api_base or settings.OPENAI_BASE_URL}:{self.endpoint.reasoning_effort}"
+        return f"{type(self.base_model).__name__}:{id(self.base_model)}"
 
     def _traced_call(self, args: tuple, kwargs: dict) -> tuple[tuple, dict, Any]:
         trace_id = None
@@ -564,6 +581,8 @@ class RobustChatModelWrapper:
             functools.partial(self.base_model.ainvoke, *args, **kwargs),
             provider=self._provider_value(),
             endpoint_key=self._endpoint_key(),
+            request_timeout=self.endpoint.timeout_seconds if self.endpoint else None,
+            max_retries=self.endpoint.max_retries if self.endpoint else None,
         )
         self._meter_usage(response)
         return response
@@ -584,6 +603,8 @@ class RobustChatModelWrapper:
             functools.partial(self._complete_attempt, emit_deltas, *args, **kwargs),
             provider=self._provider_value(),
             endpoint_key=self._endpoint_key(),
+            request_timeout=self.endpoint.timeout_seconds if self.endpoint else None,
+            max_retries=self.endpoint.max_retries if self.endpoint else None,
         )
         self._meter_usage(response)
         return response
@@ -783,7 +804,7 @@ async def acomplete_structured(
 async def invoke_llm_with_timeout_message[T](
     llm_call: Coroutine[Any, Any, T],
     timeout_seconds: int = 10,
-    hard_timeout: int = 180,
+    hard_timeout: int | None = None,
 ) -> T:
     """Send an LLM call and display a countdown / timeout message if delayed."""
     llm_task = asyncio.create_task(llm_call)
@@ -810,7 +831,7 @@ async def invoke_llm_with_timeout_message[T](
                     continue
 
                 elapsed = asyncio.get_event_loop().time() - start_time
-                if elapsed > max(0, hard_timeout - timeout_seconds):
+                if hard_timeout is not None and elapsed > max(0, hard_timeout - timeout_seconds):
                     user_messages_logger.error(f"LLM call timed out after {hard_timeout} seconds.")
                     raise TimeoutError(f"LLM call timed out after {hard_timeout} seconds.")
     except BaseException:
@@ -855,7 +876,9 @@ def get_cached_raw_model(
         model_name=model_name or settings.OPENAI_MODEL,
         temperature=temperature,
         timeout_seconds=timeout if timeout is not None else settings.OPENAI_TIMEOUT_SECONDS,
-        reasoning_effort=reasoning_effort,
+        reasoning_effort=reasoning_effort
+        if reasoning_effort is not None
+        else settings.OPENAI_REASONING_EFFORT,
         enable_grounding=enable_grounding,
     )
     return ModelFactory.get_model(ep)
@@ -963,6 +986,19 @@ async def with_fallback[T](
     main_call: Callable[[], Awaitable[T]],
     fallback_call: Callable[[], Awaitable[T]],
     none_should_fallback: bool = True,
+) -> T:
+    """Share failed endpoint identities so an identical fallback cannot repeat retries."""
+    token = _FAILED_ENDPOINTS.set({})
+    try:
+        return await _with_fallback(main_call, fallback_call, none_should_fallback)
+    finally:
+        _FAILED_ENDPOINTS.reset(token)
+
+
+async def _with_fallback[T](
+    main_call: Callable[[], Awaitable[T]],
+    fallback_call: Callable[[], Awaitable[T]],
+    none_should_fallback: bool = True,
 ) -> T:
     """Run main_call, switching to fallback_call only when it can actually help.
 
@@ -995,6 +1031,7 @@ async def with_fallback[T](
     try:
         result = await main_call()
     except (KeyboardInterrupt, asyncio.CancelledError):
+        _FALLBACK_AVAILABLE.reset(fallback_token)
         raise
     except LLMCallError as e:
         _FALLBACK_AVAILABLE.reset(fallback_token)