runtime: harden LLM retry handling and operator workflows

2026-08-21 00:34:44 -07:00
1 parent 0f5fb1d32c
commit 71e5bba551
@@ -528,50 +528,36 @@

                    <!-- Generic Tools & LLM Calls (when item is a tool) -->
                    <ng-container *ngIf="item.type === 'tool'">
-                      <!-- Recoverable provider/LLM retries remain visible after the task continues. -->
-                      <div class="step-android-action-card tool-action-card is-retrying"
-                           [class.is-expanded]="isActionExpanded('retry-' + block.id + '-' + itemIdx)"
-                           *ngIf="isLLMRetry(item.data)">
-                        <div class="action-card-header llm-retry-header"
-                             (click)="toggleActionExpanded('retry-' + block.id + '-' + itemIdx, $event)">
-                          <span class="material-symbols-outlined action-icon is-retrying">cloud_sync</span>
-                          <span class="action-title">AI service retry</span>
-                          <span class="action-status-badge is-retrying">
-                            {{ getLLMRetryCount(item.data) }}× retried
-                          </span>
-                          <span class="action-expand-icon material-symbols-outlined">
-                            {{ isActionExpanded('retry-' + block.id + '-' + itemIdx) ? 'expand_less' : 'expand_more' }}
-                          </span>
-                        </div>
-                        <div class="action-card-body">
-                          <div class="retry-summary-grid">
-                            <div class="retry-summary-item">
-                              <span class="retry-summary-label">Attempts</span>
-                              <strong>{{ getLLMRetryCount(item.data) }}</strong>
-                            </div>
-                            <div class="retry-summary-item" *ngIf="getLLMRetryTotalDelay(item.data) as totalDelay">
-                              <span class="retry-summary-label">Extra wait</span>
-                              <strong>{{ totalDelay }}</strong>
-                            </div>
-                            <div class="retry-summary-item" *ngIf="getLLMRetryProvider(item.data) as retryProvider">
-                              <span class="retry-summary-label">Provider</span>
-                              <strong>{{ retryProvider }}</strong>
-                            </div>
-                          </div>
-                          <div class="retry-attempt-list" *ngIf="isActionExpanded('retry-' + block.id + '-' + itemIdx)">
-                            <div class="retry-attempt-item" *ngFor="let retry of getLLMRetryEntries(item.data); let retryIndex = index">
-                              <div class="retry-attempt-heading">
-                                <span>Retry {{ retryIndex + 1 }}</span>
-                                <span class="retry-wait-pill" *ngIf="getLLMRetryEntryDelay(retry) as retryDelay">+{{ retryDelay }}</span>
-                              </div>
-                              <div class="retry-attempt-error">{{ getLLMRetryEntryError(retry) }}</div>
-                            </div>
+                      <!-- Only SDK-observed Gemini retries are shown as quiet stream notes. -->
+                      <div class="llm-retry-notes" *ngIf="isLLMRetry(item.data)">
+                        <div class="llm-retry-note" *ngFor="let retry of getLLMRetryEntries(item.data); let retryIndex = index">
+                          <span class="stream-status-dot"
+                                [class.streaming]="retryIndex === getLLMRetryEntries(item.data).length - 1 && isLLMRetryEntryWaiting(retry)"></span>
+                          <div class="llm-retry-note-text">
+                            <ng-container *ngIf="isLLMRetryEntryWaiting(retry)">
+                              <span class="llm-retry-status-line">
+                                LLM temporarily unavailable
+                                <ng-container *ngIf="getLLMRetryEntryDelay(retry) as retryDelay">
+                                  · Retrying in {{ retryDelay }}…
+                                </ng-container>
+                              </span>
+                              <span class="llm-retry-guidance">
+                                Frequent retries may indicate a low API rate limit. Consider increasing the limit or reducing request concurrency.
+                              </span>
+                            </ng-container>
+                            <span class="llm-retry-status-line" *ngIf="!isLLMRetryEntryWaiting(retry)">
+                              Retrying LLM request · Waited {{ getLLMRetryEntryWaited(retry) }}
+                            </span>
                          </div>
                        </div>
                      </div>

                      <!-- Failed LLM Calls (Permanently maintain Action Card format in red) -->
-                      <div class="step-android-action-card tool-action-card is-failed" *ngIf="item.data.type === 'llm_call' && item.data.status === 'failed'">
+                      <div class="step-android-action-card tool-action-card is-failed"
+                           [class.has-details]="hasLLMFailureRetryEntries(item.data)"
+                           [class.is-expanded]="isActionExpanded('llm-failure-' + block.id + '-' + itemIdx)"
+                           (click)="hasLLMFailureRetryEntries(item.data) && toggleActionExpanded('llm-failure-' + block.id + '-' + itemIdx, $event)"
+                           *ngIf="isDisplayableLLMFailure(item.data)">
                        <div class="action-card-header llm-failure-header">
                          <span class="material-symbols-outlined action-icon is-failed">error</span>
                          <span class="action-title">LLM Request Failed</span>
@@ -596,12 +582,29 @@
                            <span class="material-symbols-outlined">play_arrow</span>
                            Resume
                          </button>
+                          <span class="action-expand-icon material-symbols-outlined" *ngIf="hasLLMFailureRetryEntries(item.data)">
+                            {{ isActionExpanded('llm-failure-' + block.id + '-' + itemIdx) ? 'expand_less' : 'expand_more' }}
+                          </span>
                        </div>
                        <div class="action-card-body">
                          <div class="action-detail-row">
                            <span class="detail-label">Reason:</span>
                            <span class="detail-value">{{ getLLMErrorText(item.data) }}</span>
                          </div>
+                          <div class="action-detail-row" *ngIf="getLLMFailureWaited(item.data) as waited">
+                            <span class="detail-label">Waited:</span>
+                            <span class="detail-value">{{ waited }}</span>
+                          </div>
+                          <div class="failure-retry-details" *ngIf="isActionExpanded('llm-failure-' + block.id + '-' + itemIdx)">
+                            <div class="failure-retry-title">Retry details</div>
+                            <div class="failure-retry-row" *ngFor="let retry of getLLMFailureRetryEntries(item.data); let retryIndex = index">
+                              <span>Retry {{ retryIndex + 1 }}</span>
+                              <span>
+                                Planned {{ getLLMRetryEntryDelay(retry) || '0s' }}
+                                · Waited {{ getLLMRetryEntryWaited(retry) }}
+                              </span>
+                            </div>
+                          </div>
                        </div>
                      </div>

@@ -1955,6 +1955,55 @@ $uniform-gap: 8px;
            margin-top: 0;
            margin-bottom: 0;

+            .llm-retry-notes {
+              display: flex;
+              flex-direction: column;
+              gap: $uniform-gap;
+              padding: 0;
+
+              .llm-retry-note {
+                min-height: 20px;
+                display: flex;
+                align-items: center;
+                gap: 8px;
+                color: #71717a;
+                font-family: 'Google Sans', 'Google Sans Text', 'Inter', Arial, sans-serif;
+                font-size: 12px;
+                line-height: 1.5;
+                letter-spacing: 0.2px;
+
+                .stream-status-dot {
+                  width: 8px;
+                  height: 8px;
+                  border-radius: 50%;
+                  background: #d4d4d8;
+                  flex: 0 0 auto;
+
+                  &.streaming {
+                    background: #3b82f6;
+                    box-shadow: 0 0 0 0 rgba(59, 130, 246, 0.45);
+                    animation: pulse-stream-dot 1.8s cubic-bezier(0.4, 0, 0.6, 1) infinite;
+                  }
+                }
+
+                .llm-retry-note-text {
+                  color: #71717a;
+                  display: flex;
+                  flex-direction: column;
+                  min-width: 0;
+
+                  .llm-retry-status-line,
+                  .llm-retry-guidance {
+                    display: block;
+                  }
+
+                  .llm-retry-guidance {
+                    color: #a1a1aa;
+                  }
+                }
+              }
+            }
+
            .step-generic-tool {
              border: 1px solid transparent;

@@ -2245,6 +2294,10 @@ $uniform-gap: 8px;
                  background: #f8fafc;
                  border-color: #fca5a5;

+                  &.has-details {
+                    cursor: pointer;
+                  }
+
                  &:hover {
                    border-color: #f87171;
                  }
@@ -2255,19 +2308,6 @@ $uniform-gap: 8px;
                  }
                }

-                &.is-retrying {
-                  background: #fffbeb;
-                  border-color: #fcd34d;
-
-                  &:hover {
-                    border-color: #f59e0b;
-                  }
-
-                  .action-card-body .detail-value {
-                    color: #78350f;
-                  }
-                }
-
               &.task-report-card {
                  background: #f0fdf4;
                  border: 1px solid #bbf7d0;
@@ -2373,9 +2413,6 @@ $uniform-gap: 8px;
                     color: #ef4444;
                   }

-                   &.is-retrying {
-                     color: #d97706;
-                   }
                 }

                 .action-title {
@@ -2442,27 +2479,6 @@ $uniform-gap: 8px;
                     }
                   }

-                   &.is-retrying {
-                     background: #fef3c7;
-                     color: #92400e;
-                     border: 1px solid #fde68a;
-
-                     .status-icon {
-                       color: #d97706;
-
-                       &.is-active {
-                         animation: spin 1.5s linear infinite;
-                       }
-                     }
-                   }
-                 }
-
-                 &.llm-retry-header {
-                   border-bottom-color: #fde68a;
-
-                   .action-title {
-                     color: #78350f;
-                   }
                 }

                 &.llm-failure-header {
@@ -2528,83 +2544,6 @@ $uniform-gap: 8px;
                 flex-direction: column;
                 gap: 6px;

-                 .retry-summary-grid {
-                   display: grid;
-                   grid-template-columns: repeat(3, minmax(0, 1fr));
-                   gap: 6px;
-                 }
-
-                 .retry-summary-item {
-                   min-width: 0;
-                   padding: 7px 8px;
-                   border: 1px solid rgba(245, 158, 11, 0.18);
-                   border-radius: 7px;
-                   background: rgba(255, 255, 255, 0.68);
-                   display: flex;
-                   flex-direction: column;
-                   gap: 2px;
-
-                   .retry-summary-label {
-                     color: #a16207;
-                     font-size: 9.5px;
-                     font-weight: 600;
-                     letter-spacing: 0.02em;
-                     text-transform: uppercase;
-                   }
-
-                   strong {
-                     color: #78350f;
-                     font-size: 11.5px;
-                     font-weight: 700;
-                     overflow: hidden;
-                     text-overflow: ellipsis;
-                     white-space: nowrap;
-                   }
-                 }
-
-                 .retry-attempt-list {
-                   border-top: 1px solid rgba(245, 158, 11, 0.22);
-                   display: flex;
-                   flex-direction: column;
-                   gap: 6px;
-                   margin-top: 2px;
-                   padding-top: 8px;
-                   animation: fadeInDetails 0.2s ease-out;
-
-                   .retry-attempt-item {
-                     padding: 8px 9px;
-                     border: 1px solid rgba(245, 158, 11, 0.2);
-                     border-radius: 7px;
-                     background: rgba(255, 255, 255, 0.76);
-                   }
-
-                   .retry-attempt-heading {
-                     display: flex;
-                     align-items: center;
-                     justify-content: space-between;
-                     gap: 8px;
-                     margin-bottom: 4px;
-                     color: #92400e;
-                     font-size: 10px;
-                     font-weight: 700;
-                   }
-
-                   .retry-wait-pill {
-                     padding: 1px 5px;
-                     border-radius: 999px;
-                     background: #fef3c7;
-                     color: #b45309;
-                   }
-
-                   .retry-attempt-error {
-                     color: #78350f;
-                     font-size: 10.5px;
-                     line-height: 1.45;
-                     overflow-wrap: anywhere;
-                     user-select: text;
-                   }
-                 }
-
                 .action-detail-row {
                   display: flex;
                   align-items: baseline;
@@ -2654,6 +2593,38 @@ $uniform-gap: 8px;
                     }
                   }
                 }
+
+                 .failure-retry-details {
+                   margin-top: 4px;
+                   padding-top: 9px;
+                   border-top: 1px solid #fecaca;
+                   display: flex;
+                   flex-direction: column;
+                   gap: 7px;
+                   animation: fadeInDetails 0.2s ease-out;
+
+                   .failure-retry-title {
+                     color: #991b1b;
+                     font-size: 10px;
+                     font-weight: 700;
+                     text-transform: uppercase;
+                     letter-spacing: 0.03em;
+                   }
+
+                   .failure-retry-row {
+                     display: flex;
+                     justify-content: space-between;
+                     gap: 12px;
+                     color: #7f1d1d;
+                     font-size: 10.5px;
+                     line-height: 1.4;
+
+                     span:last-child {
+                       color: #64748b;
+                       text-align: right;
+                     }
+                   }
+                 }
               }

               .action-card-expanded-details {
@@ -158,6 +158,7 @@ export class AgentStreamComponent implements AfterViewInit {
  // Set to track expanded and collapsed state of action cards
  public expandedActionCards = signal<Set<string>>(new Set<string>());
  public collapsedActionCards = signal<Set<string>>(new Set<string>());
+  public retryClock = signal<number>(Date.now());

  // Performance caches for parameter and event extractions
  private actionParamsCache = new WeakMap<any, ActionParam[]>();
@@ -280,6 +281,9 @@ export class AgentStreamComponent implements AfterViewInit {
  }

  constructor() {
+    const retryClockInterval = setInterval(() => this.retryClock.set(Date.now()), 1000);
+    this.destroyRef.onDestroy(() => clearInterval(retryClockInterval));
+
    // Auto scroll stream box during active text streaming
    effect(() => {
      const logs = this.filteredLogs();
@@ -827,7 +831,12 @@ export class AgentStreamComponent implements AfterViewInit {
    if (!block) return false;
    const hasNative = Boolean(this.getNativeThinking(block));
    const hasRaw = Boolean(this.getRawThinking(block));
-    const hasVisibleTools = Boolean(block.data?.generic_tools) && block.data.generic_tools.some((t: any) => this.shouldShowTool(t, block.data) || (t.type === 'llm_call' && (t.status === 'failed' || t.status === 'retrying')) || this.isReportStatusAction(t));
+    const hasVisibleTools = Boolean(block.data?.generic_tools) && block.data.generic_tools.some((t: any) =>
+      this.shouldShowTool(t, block.data)
+      || this.isDisplayableLLMFailure(t)
+      || this.isLLMRetry(t)
+      || this.isReportStatusAction(t)
+    );
    const hasAndroidActions = Boolean(block.data?.action_taken) && (this.isAndroidAction(block.data.action_taken) || this.isReportStatusAction(block.data.action_taken));
    return hasNative || hasRaw || hasVisibleTools || hasAndroidActions;
  }
@@ -852,6 +861,12 @@ export class AgentStreamComponent implements AfterViewInit {
    return !!tool && this.retryablePauseTrace() === tool;
  }

+  public isDisplayableLLMFailure(tool: any): boolean {
+    return tool?.type === 'llm_call'
+      && tool?.status === 'failed'
+      && (tool?.name === 'llm_pause' || tool?.payload?.pause === true);
+  }
+
  public resumePausedTask(event: Event): void {
    event.stopPropagation();
    this.agentService.resumeTask();
@@ -865,7 +880,10 @@ export class AgentStreamComponent implements AfterViewInit {
  }

  public isLLMRetry(tool: any): boolean {
-    return tool?.type === 'llm_call' && tool?.status === 'retrying';
+    return tool?.type === 'llm_call'
+      && tool?.status === 'retrying'
+      && tool?.payload?.source === 'provider_sdk'
+      && ['google', 'gemini'].includes(String(tool?.payload?.provider || '').toLowerCase());
  }

  public getLLMRetryEntries(tool: any): any[] {
@@ -873,25 +891,17 @@ export class AgentStreamComponent implements AfterViewInit {
    return Array.isArray(retries) && retries.length > 0 ? retries : [tool?.payload || tool];
  }

-  public getLLMRetryCount(tool: any): number {
-    const count = Number(tool?.payload?.retry_count);
-    return Number.isFinite(count) && count > 0 ? count : this.getLLMRetryEntries(tool).length;
+  public getLLMFailureRetryEntries(tool: any): any[] {
+    const retries = tool?.payload?.retries;
+    if (!Array.isArray(retries)) return [];
+    return retries.filter((retry: any) =>
+      retry?.source === 'provider_sdk'
+      && ['google', 'gemini'].includes(String(retry?.provider || '').toLowerCase())
+    );
  }

-  public getLLMRetryTotalDelay(tool: any): string | null {
-    const configuredTotal = Number(tool?.payload?.total_delay);
-    const total = Number.isFinite(configuredTotal)
-      ? configuredTotal
-      : this.getLLMRetryEntries(tool).reduce(
-          (sum: number, retry: any) => sum + (Number(retry?.delay) || 0),
-          0
-        );
-    if (total <= 0) return null;
-    return `${total.toFixed(2).replace(/\.00$/, '')}s`;
-  }
-
-  public getLLMRetryEntryError(entry: any): string {
-    return this.cleanErrorMessage(entry?.error || 'Unknown error');
+  public hasLLMFailureRetryEntries(tool: any): boolean {
+    return this.getLLMFailureRetryEntries(tool).length > 0;
  }

  public getLLMRetryEntryDelay(entry: any): string | null {
@@ -900,14 +910,37 @@ export class AgentStreamComponent implements AfterViewInit {
    return `${delay.toFixed(2).replace(/\.00$/, '')}s`;
  }

-  public getLLMRetryProvider(tool: any): string | null {
-    const providers = Array.isArray(tool?.payload?.providers)
-      ? tool.payload.providers
-      : [tool?.payload?.provider];
-    const labels = providers
-      .filter((provider: any) => typeof provider === 'string' && provider.trim())
-      .map((provider: string) => provider.trim().replace(/^./, (value: string) => value.toUpperCase()));
-    return labels.length > 0 ? Array.from(new Set(labels)).join(', ') : null;
+  public getLLMRetryEntryWaited(entry: any): string {
+    const delay = Number(entry?.delay);
+    if (!Number.isFinite(delay) || delay <= 0) return '0s';
+
+    const rawStartedAt = Number(entry?.scheduled_at ?? entry?.timestamp);
+    if (!Number.isFinite(rawStartedAt) || rawStartedAt <= 0) {
+      return this.formatLLMDuration(delay);
+    }
+    const startedAtMs = rawStartedAt < 1e11 ? rawStartedAt * 1000 : rawStartedAt;
+    const elapsed = Math.min(delay, Math.max(0, (this.retryClock() - startedAtMs) / 1000));
+    return this.formatLLMDuration(elapsed);
+  }
+
+  public isLLMRetryEntryWaiting(entry: any): boolean {
+    const delay = Number(entry?.delay);
+    const rawStartedAt = Number(entry?.scheduled_at ?? entry?.timestamp);
+    if (!Number.isFinite(delay) || delay <= 0 || !Number.isFinite(rawStartedAt)) return false;
+    const startedAtMs = rawStartedAt < 1e11 ? rawStartedAt * 1000 : rawStartedAt;
+    return this.retryClock() < startedAtMs + delay * 1000;
+  }
+
+  public getLLMFailureWaited(tool: any): string | null {
+    const waited = Number(tool?.payload?.waited_seconds);
+    if (!Number.isFinite(waited) || waited < 0) return null;
+    return this.formatLLMDuration(waited);
+  }
+
+  public formatLLMDuration(seconds: number): string {
+    if (seconds < 0.05) return '0s';
+    if (seconds < 10) return `${seconds.toFixed(1).replace(/\.0$/, '')}s`;
+    return `${Math.round(seconds)}s`;
  }

  public cleanErrorMessage(rawError: any): string {
@@ -0,0 +1,110 @@
+import { signal } from '@angular/core';
+import { of } from 'rxjs';
+
+import { AgentService } from './agent.service';
+
+describe('AgentService live LLM retry timeline', () => {
+  function createServiceWithoutPolling(): AgentService {
+    const service = Object.create(AgentService.prototype) as AgentService;
+    service.sessionLogs = signal<any[]>([]);
+    return service;
+  }
+
+  it('normalizes a live retry event into the historical trace contract', () => {
+    const service = createServiceWithoutPolling();
+    const event = {
+      trace_id: 'retry-1',
+      step_id: 'step-1',
+      timestamp: 100,
+      error: '503 high demand',
+      delay: 1.5,
+      provider: 'google',
+      source: 'provider_sdk',
+      recoverable: true,
+      request_id: 'request-1',
+      scheduled_at: 100
+    };
+
+    (service as any).appendLiveLLMRetryTrace(event, 'session-1');
+
+    const [log] = service.sessionLogs();
+    expect(log.type).toBe('trace_recorded');
+    expect(log.data).toEqual(jasmine.objectContaining({
+      trace_id: 'retry-1',
+      session_id: 'session-1',
+      step_id: 'step-1',
+      type: 'llm_call',
+      name: 'llm_retry',
+      status: 'retrying'
+    }));
+    expect(log.data.payload).toEqual(jasmine.objectContaining({
+      error: '503 high demand',
+      delay: 1.5,
+      provider: 'google',
+      source: 'provider_sdk',
+      request_id: 'request-1'
+    }));
+  });
+
+  it('upserts duplicate live retry events instead of adding another row', () => {
+    const service = createServiceWithoutPolling();
+    const event = {
+      trace_id: 'retry-1',
+      timestamp: 100,
+      delay: 1.5,
+      provider: 'google',
+      source: 'provider_sdk',
+      request_id: 'request-1',
+      scheduled_at: 100
+    };
+
+    (service as any).appendLiveLLMRetryTrace(event, 'session-1');
+    (service as any).appendLiveLLMRetryTrace({ ...event, error: 'updated 503' }, 'session-1');
+
+    expect(service.sessionLogs().length).toBe(1);
+    expect(service.sessionLogs()[0].data.payload.error).toBe('updated 503');
+  });
+
+  it('replaces an old history snapshot while preserving live retry events', () => {
+    const service = createServiceWithoutPolling();
+    (service as any).currentSessionId = signal<string | null>('session-1');
+    (service as any).http = {
+      get: () => of([{
+        step_id: 'step-1',
+        timestamp: 100,
+        generic_tools: [{ trace_id: 'persisted-retry', name: 'llm_retry' }]
+      }])
+    };
+    service.sessionLogs.set([
+      { type: 'step_updated', history_snapshot: true, data: { step_id: 'old-step' } },
+      { type: 'trace_recorded', data: { trace_id: 'live-retry', name: 'llm_retry' } }
+    ]);
+
+    (service as any).backfillSessionSteps('session-1');
+
+    const logs = service.sessionLogs();
+    expect(logs.length).toBe(2);
+    expect(logs[0].history_snapshot).toBeTrue();
+    expect(logs[0].data.step_id).toBe('step-1');
+    expect(logs[1].data.trace_id).toBe('live-retry');
+  });
+
+  it('follows a just-started task even when status polling saw it first', () => {
+    const service = createServiceWithoutPolling();
+    (service as any).http = {
+      post: () => of({ tasks: [{ session_id: 'new-session' }] })
+    };
+    service.agentStatus = signal('running');
+    service.runningSessionId = signal<string | null>('new-session');
+    service.userPinnedSessionId = signal<string | null>(null);
+    (service as any).sessions = signal<any[]>([{
+      session_id: 'new-session',
+      status: 'running'
+    }]);
+    const selectSpy = spyOn(service, 'selectSession');
+
+    service.runTask('test goal').subscribe();
+
+    expect(selectSpy).toHaveBeenCalledWith('new-session', false);
+  });
+});
@@ -184,7 +184,12 @@ export class AgentService {
              const isCurrentlyRunning =
                this.agentStatus() === 'running' ||
                this.sessions().some((s) => s.status === 'running');
-              if (!isCurrentlyRunning) {
+              const activeSessionId = this.runningSessionId()
+                || this.sessions().find((s) => s.status === 'running' || s.status === 'paused')?.session_id;
+              // The status poll can observe the new runner before /api/run
+              // returns. In that case it is still the task we just submitted,
+              // not an older task that should remain selected.
+              if (!isCurrentlyRunning || activeSessionId === newSessionId) {
                this.selectSession(newSessionId, false);
              }
            }
@@ -401,45 +406,16 @@ export class AgentService {
    }

    console.log(`Connecting to session: ${sessionId}`);
-    this.http.get<StepItemData[]>(`/api/sessions/${sessionId}/steps`).subscribe({
-      next: (steps) => {
-        if (this.currentSessionId() !== sessionId) return;
-        const historicalLogs: any[] = [];
-        steps.forEach((step) => {
-          const stepTimeIso = new Date((step.timestamp || Date.now() / 1000) * 1000).toISOString();
-          historicalLogs.push({
-            type: 'step_updated',
-            session_id: sessionId,
-            timestamp: stepTimeIso,
-            data: step
-          });
-        });
-        // The history request and EventSource start together. On slower Windows
-        // hosts, live events can arrive first; replacing the signal here would
-        // erase them until a browser refresh reloads the persisted steps.
-        this.sessionLogs.update((liveLogs) => {
-          if (liveLogs.length === 0) return historicalLogs;
-
-          const liveStepIds = new Set(
-            liveLogs
-              .filter((log) => log.type === 'step_recorded' || log.type === 'step_updated')
-              .map((log) => String(log.data?.step_id || log.data?.step_number || ''))
-              .filter(Boolean)
-          );
-          const missingHistorical = historicalLogs.filter((log) => {
-            const stepKey = String(log.data?.step_id || log.data?.step_number || '');
-            return !stepKey || !liveStepIds.has(stepKey);
-          });
-          return [...missingHistorical, ...liveLogs];
-        });
-      },
-      error: (err) => {
-        console.error('Failed to pre-load historical steps:', err);
-      }
-    });
-
    this.eventSource = new EventSource(`/api/stream/${sessionId}`);

+    // The server emits info only after this client has been registered as an
+    // SSE subscriber. Fetching the persisted snapshot after that acknowledgement
+    // closes the snapshot-to-stream gap: earlier events are in history and later
+    // events are already queued for this connection.
+    this.eventSource.addEventListener('info', () => {
+      this.backfillSessionSteps(sessionId);
+    });
+
    // Listen to standard SSE keep-alive
    this.eventSource.addEventListener('keep-alive', () => {
      // No-op keep-alive tick
@@ -524,6 +500,7 @@ export class AgentService {
            const attemptText = attempt && max ? ` (Attempt ${attempt}/${max})` : '';
            const delayText = delay > 0 ? `; retrying in ${delay.toFixed(2).replace(/\.00$/, '')}s` : '';
            this.retryMessage.set(`AI service is temporarily busy${attemptText}${delayText}...`);
+            this.appendLiveLLMRetryTrace(parsedData, sessionId);
            return;
          }

@@ -536,7 +513,8 @@ export class AgentService {
              pauseError,
              sessionId,
              parsedData.timestamp,
-              parsedData.step_id
+              parsedData.step_id,
+              parsedData
            );
            return;
          }
@@ -628,14 +606,28 @@ export class AgentService {
                ? new Date(parsedData.timestamp * 1000).toISOString()
                : new Date().toISOString();

-              return [
-                ...updatedLogs,
-                {
-                  type: eventType,
-                  timestamp: evtTime,
-                  data: parsedData
+              const nextLog = {
+                type: eventType,
+                timestamp: evtTime,
+                data: parsedData
+              };
+
+              // A retry is published both as trace_recorded and as the
+              // purpose-built llm_retrying event. Replace the optimistic live
+              // copy when the persisted trace arrives instead of duplicating it.
+              if (eventType === 'trace_recorded' && parsedData?.trace_id) {
+                const existingTraceIndex = updatedLogs.findIndex((log) =>
+                  log.type === 'trace_recorded'
+                  && log.data?.trace_id === parsedData.trace_id
+                );
+                if (existingTraceIndex > -1) {
+                  const deduplicatedLogs = [...updatedLogs];
+                  deduplicatedLogs[existingTraceIndex] = nextLog;
+                  return deduplicatedLogs;
                }
-              ];
+              }
+
+              return [...updatedLogs, nextLog];
            });
          }

@@ -668,6 +660,93 @@ export class AgentService {
    };
  }

+  /**
+   * Load a fresh persisted snapshot without replacing events already received
+   * from SSE. Snapshot logs are replaceable, while live logs remain append-only;
+   * the stream aggregator merges matching steps and trace IDs within each step.
+   */
+  private backfillSessionSteps(sessionId: string): void {
+    this.http.get<StepItemData[]>(`/api/sessions/${sessionId}/steps`).subscribe({
+      next: (steps) => {
+        if (this.currentSessionId() !== sessionId) return;
+        const historicalLogs = steps.map((step) => ({
+          type: 'step_updated',
+          session_id: sessionId,
+          timestamp: new Date((step.timestamp || Date.now() / 1000) * 1000).toISOString(),
+          data: step,
+          history_snapshot: true
+        }));
+
+        this.sessionLogs.update((logs) => [
+          ...historicalLogs,
+          ...logs.filter((log) => !log.history_snapshot)
+        ]);
+      },
+      error: (err) => {
+        console.error('Failed to backfill session steps:', err);
+      }
+    });
+  }
+
+  /**
+   * Convert the dedicated live retry event into the same trace shape used by
+   * history APIs. This keeps live streaming and post-refresh rendering on one
+   * data contract instead of maintaining a separate retry UI path.
+   */
+  private appendLiveLLMRetryTrace(data: any, sessionId: string): void {
+    const timestampSeconds = typeof data?.timestamp === 'number'
+      ? (data.timestamp > 1e11 ? data.timestamp / 1000 : data.timestamp)
+      : Date.now() / 1000;
+    const traceId = data?.trace_id
+      || `llm-retry-live-${data?.request_id || 'unknown'}-${timestampSeconds}`;
+    const payload = {
+      error: data?.error,
+      delay: data?.delay,
+      attempt: data?.attempt,
+      max_retries: data?.max_retries,
+      provider: data?.provider,
+      source: data?.source,
+      recoverable: data?.recoverable,
+      request_id: data?.request_id,
+      scheduled_at: data?.scheduled_at ?? timestampSeconds
+    };
+    const retryTrace = {
+      trace_id: traceId,
+      session_id: sessionId,
+      step_id: data?.step_id || null,
+      type: 'llm_call',
+      name: 'llm_retry',
+      timestamp: timestampSeconds,
+      status: 'retrying',
+      payload
+    };
+
+    this.sessionLogs.update((logs) => {
+      const existingIndex = logs.findIndex((log) =>
+        log.type === 'trace_recorded'
+        && (
+          log.data?.trace_id === traceId
+          || (
+            data?.request_id
+            && log.data?.payload?.request_id === data.request_id
+            && Number(log.data?.payload?.scheduled_at) === Number(payload.scheduled_at)
+          )
+        )
+      );
+      const retryLog = {
+        type: 'trace_recorded',
+        session_id: sessionId,
+        timestamp: new Date(timestampSeconds * 1000).toISOString(),
+        data: retryTrace
+      };
+      if (existingIndex < 0) return [...logs, retryLog];
+
+      const updatedLogs = [...logs];
+      updatedLogs[existingIndex] = retryLog;
+      return updatedLogs;
+    });
+  }
+
  /**
   * Preserve pause failures in the normal failed-LLM card stream. New runners
   * emit a persisted failed trace first; this fallback also supports older
@@ -677,7 +756,8 @@ export class AgentService {
    error: unknown,
    sessionId: string,
    timestamp?: number | string,
-    stepId?: string | null
+    stepId?: string | null,
+    details?: any
  ): void {
    const errorText = typeof error === 'string' ? error : JSON.stringify(error);
    const pauseKey = `${sessionId}:${errorText}`;
@@ -717,7 +797,14 @@ export class AgentService {
            name: 'llm_pause',
            status: 'failed',
            timestamp: timestampMs / 1000,
-            payload: { error: errorText, pause: true }
+            payload: {
+              error: errorText,
+              pause: true,
+              request_id: details?.request_id,
+              provider: details?.provider,
+              waited_seconds: details?.waited_seconds,
+              retries: Array.isArray(details?.retries) ? details.retries : []
+            }
          }
        }
      ];
@@ -9,7 +9,7 @@ describe('getUniqueGenericTools retry aggregation', () => {
        name: 'llm_retry',
        status: 'retrying',
        timestamp: 100,
-        payload: { error: '503 first', delay: 1.18, provider: 'google' }
+        payload: { error: '503 first', delay: 1.18, provider: 'google', source: 'provider_sdk', request_id: 'request-1' }
      },
      {
        trace_id: 'retry-2',
@@ -17,7 +17,7 @@ describe('getUniqueGenericTools retry aggregation', () => {
        name: 'llm_retry',
        status: 'retrying',
        timestamp: 101,
-        payload: { error: '503 second', delay: 2.96, provider: 'google' }
+        payload: { error: '503 second', delay: 2.96, provider: 'google', source: 'provider_sdk', request_id: 'request-1' }
      }
    ];

@@ -27,31 +27,51 @@ describe('getUniqueGenericTools retry aggregation', () => {
    expect(result[0].name).toBe('llm_retry_group');
    expect(result[0].payload.retry_count).toBe(2);
    expect(result[0].payload.total_delay).toBeCloseTo(4.14);
-    expect(result[0].payload.providers).toEqual(['google']);
    expect(result[0].payload.retries.map((retry: any) => retry.error)).toEqual([
      '503 first',
      '503 second'
    ]);
  });

-  it('keeps a terminal LLM failure separate from its preceding retries', () => {
+  it('folds SDK retry rows into their matching terminal failure', () => {
    const result = getUniqueGenericTools([
      {
        trace_id: 'retry-1',
        type: 'llm_call',
        name: 'llm_retry',
        status: 'retrying',
-        payload: { error: '503', delay: 1 }
+        payload: { error: '503', delay: 1, provider: 'google', source: 'provider_sdk', request_id: 'request-1' }
      },
      {
        trace_id: 'failure-1',
        type: 'llm_call',
        name: 'llm_pause',
        status: 'failed',
-        payload: { error: '503 exhausted', pause: true }
+        payload: { error: '503 exhausted', pause: true, request_id: 'request-1', retries: [{ delay: 1 }] }
      }
    ]);

-    expect(result.map(tool => tool.name)).toEqual(['llm_retry_group', 'llm_pause']);
+    expect(result.map(tool => tool.name)).toEqual(['llm_pause']);
+  });
+
+  it('does not expose opaque retries from non-Gemini providers', () => {
+    const result = getUniqueGenericTools([
+      {
+        trace_id: 'retry-openai',
+        type: 'llm_call',
+        name: 'llm_retry',
+        status: 'retrying',
+        payload: { error: '429', delay: 2, provider: 'openai', source: 'artemis_wrapper' }
+      },
+      {
+        trace_id: 'failure-openai',
+        type: 'llm_call',
+        name: 'llm_pause',
+        status: 'failed',
+        payload: { error: '429 exhausted', pause: true, waited_seconds: 12 }
+      }
+    ]);
+
+    expect(result.map(tool => tool.name)).toEqual(['llm_pause']);
  });
 });
@@ -185,41 +185,57 @@ export function getToolAgentName(tool: any): string | null {
 export function getUniqueGenericTools(tools: any[] | undefined): any[] {
  if (!tools) return [];

+  // Only Google GenAI SDK retries are observable. Artemis wrapper retries and
+  // opaque provider retries must never be presented as provider internals.
  const retryTools = tools.filter(tool =>
    tool?.type === 'llm_call'
    && tool?.status === 'retrying'
    && (tool?.name === 'llm_retry' || tool?.name === 'llm_retry_group')
+    && tool?.payload?.source === 'provider_sdk'
+    && ['google', 'gemini'].includes(String(tool?.payload?.provider || '').toLowerCase())
  );
-  const firstRetry = retryTools[0];
-  const latestRetry = retryTools[retryTools.length - 1];
-  const retryAggregate = firstRetry ? {
-    ...firstRetry,
-    trace_id: `llm-retry-group-${firstRetry.trace_id || firstRetry.timestamp || 'unknown'}`,
-    name: 'llm_retry_group',
-    payload: {
-      ...(latestRetry?.payload || {}),
-      retry_count: retryTools.length,
-      total_delay: retryTools.reduce(
-        (total, retry) => total + (Number(retry?.payload?.delay) || 0),
-        0
-      ),
-      providers: Array.from(new Set(
-        retryTools
-          .map(retry => retry?.payload?.provider)
-          .filter((provider): provider is string => typeof provider === 'string' && !!provider)
-      )),
-      retries: retryTools.map(retry => ({
-        trace_id: retry.trace_id,
-        timestamp: retry.timestamp,
-        error: retry?.payload?.error || retry.error,
-        delay: Number(retry?.payload?.delay) || 0,
-        provider: retry?.payload?.provider,
-        source: retry?.payload?.source,
-        attempt: retry?.payload?.attempt,
-        max_retries: retry?.payload?.max_retries
-      }))
-    }
-  } : null;
+  const retryKey = (tool: any) => String(
+    tool?.payload?.request_id || `legacy-${tool?.step_id || 'unassigned'}`
+  );
+  const retryGroups = new Map<string, any[]>();
+  for (const retry of retryTools) {
+    const key = retryKey(retry);
+    retryGroups.set(key, [...(retryGroups.get(key) || []), retry]);
+  }
+  const terminalRequestIds = new Set(
+    tools
+      .filter(tool => tool?.type === 'llm_call' && tool?.status === 'failed')
+      .map(tool => tool?.payload?.request_id)
+      .filter((requestId): requestId is string => typeof requestId === 'string' && !!requestId)
+  );
+
+  const buildRetryAggregate = (group: any[]) => {
+    const firstRetry = group[0];
+    const latestRetry = group[group.length - 1];
+    return {
+      ...firstRetry,
+      trace_id: `llm-retry-group-${firstRetry?.payload?.request_id || firstRetry?.trace_id || firstRetry?.timestamp || 'unknown'}`,
+      name: 'llm_retry_group',
+      payload: {
+        ...(latestRetry?.payload || {}),
+        retry_count: group.length,
+        total_delay: group.reduce(
+          (total, retry) => total + (Number(retry?.payload?.delay) || 0),
+          0
+        ),
+        retries: group.map(retry => ({
+          trace_id: retry.trace_id,
+          timestamp: retry.timestamp,
+          error: retry?.payload?.error || retry.error,
+          delay: Number(retry?.payload?.delay) || 0,
+          provider: retry?.payload?.provider,
+          source: retry?.payload?.source,
+          request_id: retry?.payload?.request_id,
+          scheduled_at: retry?.payload?.scheduled_at
+        }))
+      }
+    };
+  };

  const toolMap = new Map<string, any>();
  for (const t of tools) {
@@ -232,22 +248,33 @@ export function getUniqueGenericTools(tools: any[] | undefined): any[] {

  const result: any[] = [];
  const seenTraces = new Set<string>();
-  let retryAggregateAdded = false;
+  const addedRetryGroups = new Set<string>();

  for (const tool of tools) {
    if (!tool || !tool.name) continue;
    if (tool.type === 'agent') continue;

    if (tool.type === 'llm_call' && tool.status === 'retrying') {
-      if (!retryAggregateAdded && retryAggregate) {
-        result.push(retryAggregate);
-        retryAggregateAdded = true;
+      const groupKey = retryKey(tool);
+      const group = retryGroups.get(groupKey);
+      const requestId = tool?.payload?.request_id;
+      if (
+        group
+        && !addedRetryGroups.has(groupKey)
+        && !(requestId && terminalRequestIds.has(requestId))
+      ) {
+        result.push(buildRetryAggregate(group));
+        addedRetryGroups.add(groupKey);
      }
      continue;
    }

    if (tool.type === 'llm_call' && tool.status === 'failed') {
-      result.push(tool);
+      // Callback-level attempt errors are implementation details. The wrapper
+      // emits one terminal llm_pause trace after retries are exhausted.
+      if (tool.name === 'llm_pause' || tool?.payload?.pause === true) {
+        result.push(tool);
+      }
      continue;
    }