runtime: harden LLM retry handling and operator workflows

tests/unit/test_llm_pause.py

@@ -57,24 +57,98 @@ def test_provider_sdk_retry_is_persisted_and_published(monkeypatch):
         exc_info=None,
     )
 
-    llm._ProviderRetryTelemetryHandler().emit(record)
-
-    assert engine.trace_kwargs == {
-        "type": "llm_call",
-        "name": "llm_retry",
-        "payload": {
-            "error": "ServerError: 503 UNAVAILABLE: high demand",
-            "delay": 1.18,
+    request_token = llm._ACTIVE_LLM_REQUEST.set(
+        {
+            "request_id": "request-1",
             "provider": "google",
-            "source": "provider_sdk",
-            "recoverable": True,
-        },
-        "step_id": "step-7",
-        "parent_trace_id": None,
-        "status": "retrying",
+            "started_at": 100.0,
+            "retries": [],
+        }
+    )
+    try:
+        llm._ProviderRetryTelemetryHandler().emit(record)
+    finally:
+        llm._ACTIVE_LLM_REQUEST.reset(request_token)
+
+    assert engine.trace_kwargs["type"] == "llm_call"
+    assert engine.trace_kwargs["name"] == "llm_retry"
+    assert engine.trace_kwargs["status"] == "retrying"
+    assert engine.trace_kwargs["payload"] == {
+        "error": "ServerError: 503 UNAVAILABLE: high demand",
+        "delay": 1.18,
+        "provider": "google",
+        "source": "provider_sdk",
+        "recoverable": True,
+        "request_id": "request-1",
+        "scheduled_at": engine.trace_kwargs["payload"]["scheduled_at"],
     }
     event_type, payload = engine.published
     assert event_type == "llm_retrying"
     assert payload["trace_id"] == "trace-1"
     assert payload["delay"] == 1.18
     assert payload["error"].startswith("ServerError: 503")
+
+
+def test_provider_sdk_retry_is_hidden_for_non_gemini_request(monkeypatch):
+    engine = _RecordingEngine()
+    monkeypatch.setattr(llm, "_CURRENT_DATA_ENGINE", engine)
+    record = logging.LogRecord(
+        name="google_genai._api_client",
+        level=logging.INFO,
+        pathname=__file__,
+        lineno=1,
+        msg="Retrying request in 2 seconds as it raised 503 UNAVAILABLE",
+        args=(),
+        exc_info=None,
+    )
+    request_token = llm._ACTIVE_LLM_REQUEST.set(
+        {
+            "request_id": "request-openai",
+            "provider": "openai",
+            "started_at": 100.0,
+            "retries": [],
+        }
+    )
+    try:
+        llm._ProviderRetryTelemetryHandler().emit(record)
+    finally:
+        llm._ACTIVE_LLM_REQUEST.reset(request_token)
+
+    assert engine.trace_kwargs is None
+    assert engine.published is None
+
+
+def test_terminal_failure_aggregates_observed_retries_and_total_wait(monkeypatch, tmp_path):
+    engine = _RecordingEngine()
+    monkeypatch.setattr(llm, "_CURRENT_DATA_ENGINE", engine)
+    monkeypatch.setattr(llm.settings, "TRACES_PATH", str(tmp_path / "traces"))
+    monkeypatch.setattr(llm.time, "time", lambda: 120.0)
+    retries = [
+        {
+            "error": "503 high demand",
+            "delay": 10.0,
+            "provider": "google",
+            "source": "provider_sdk",
+            "request_id": "request-2",
+            "scheduled_at": 101.0,
+            "timestamp": 101.0,
+        }
+    ]
+    request_token = llm._ACTIVE_LLM_REQUEST.set(
+        {
+            "request_id": "request-2",
+            "provider": "google",
+            "started_at": 100.0,
+            "retries": retries,
+        }
+    )
+    try:
+        llm._handle_llm_pause_and_resume(RuntimeError("503 high demand"))
+    finally:
+        llm._ACTIVE_LLM_REQUEST.reset(request_token)
+
+    payload = engine.trace_kwargs["payload"]
+    assert payload["request_id"] == "request-2"
+    assert payload["provider"] == "google"
+    assert payload["waited_seconds"] == 20.0
+    assert payload["retries"] == retries