chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

runtime: harden LLM retry handling and operator workflows

Browse Source
This commit is contained in:
somew1nd committed 2026-08-21 00:34:44 -07:00
1 parent 0f5fb1d32c
commit 71e5bba551
18 files changed
+937 -348
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html
+42 -39
View File
@@ -528,50 +528,36 @@
<!-- Generic Tools & LLM Calls (when item is a tool) -->
<ng-container *ngIf="item.type === 'tool'">
<!-- Recoverable provider/LLM retries remain visible after the task continues. -->
<div class="step-android-action-card tool-action-card is-retrying"
[class.is-expanded]="isActionExpanded('retry-' + block.id + '-' + itemIdx)"
*ngIf="isLLMRetry(item.data)">
<div class="action-card-header llm-retry-header"
(click)="toggleActionExpanded('retry-' + block.id + '-' + itemIdx, $event)">
<span class="material-symbols-outlined action-icon is-retrying">cloud_sync</span>
<span class="action-title">AI service retry</span>
<span class="action-status-badge is-retrying">
{{ getLLMRetryCount(item.data) }}× retried
</span>
<span class="action-expand-icon material-symbols-outlined">
{{ isActionExpanded('retry-' + block.id + '-' + itemIdx) ? 'expand_less' : 'expand_more' }}
</span>
</div>
<div class="action-card-body">
<div class="retry-summary-grid">
<div class="retry-summary-item">
<span class="retry-summary-label">Attempts</span>
<strong>{{ getLLMRetryCount(item.data) }}</strong>
</div>
<div class="retry-summary-item" *ngIf="getLLMRetryTotalDelay(item.data) as totalDelay">
<span class="retry-summary-label">Extra wait</span>
<strong>{{ totalDelay }}</strong>
</div>
<div class="retry-summary-item" *ngIf="getLLMRetryProvider(item.data) as retryProvider">
<span class="retry-summary-label">Provider</span>
<strong>{{ retryProvider }}</strong>
</div>
</div>
<div class="retry-attempt-list" *ngIf="isActionExpanded('retry-' + block.id + '-' + itemIdx)">
<div class="retry-attempt-item" *ngFor="let retry of getLLMRetryEntries(item.data); let retryIndex = index">
<div class="retry-attempt-heading">
<span>Retry {{ retryIndex + 1 }}</span>
<span class="retry-wait-pill" *ngIf="getLLMRetryEntryDelay(retry) as retryDelay">+{{ retryDelay }}</span>
</div>
<div class="retry-attempt-error">{{ getLLMRetryEntryError(retry) }}</div>
</div>
<!-- Only SDK-observed Gemini retries are shown as quiet stream notes. -->
<div class="llm-retry-notes" *ngIf="isLLMRetry(item.data)">
<div class="llm-retry-note" *ngFor="let retry of getLLMRetryEntries(item.data); let retryIndex = index">
<span class="stream-status-dot"
[class.streaming]="retryIndex === getLLMRetryEntries(item.data).length - 1 && isLLMRetryEntryWaiting(retry)"></span>
<div class="llm-retry-note-text">
<ng-container *ngIf="isLLMRetryEntryWaiting(retry)">
<span class="llm-retry-status-line">
LLM temporarily unavailable
<ng-container *ngIf="getLLMRetryEntryDelay(retry) as retryDelay">
· Retrying in {{ retryDelay }}…
</ng-container>
</span>
<span class="llm-retry-guidance">
Frequent retries may indicate a low API rate limit. Consider increasing the limit or reducing request concurrency.
</span>
</ng-container>
<span class="llm-retry-status-line" *ngIf="!isLLMRetryEntryWaiting(retry)">
Retrying LLM request · Waited {{ getLLMRetryEntryWaited(retry) }}
</span>
</div>
</div>
</div>
<!-- Failed LLM Calls (Permanently maintain Action Card format in red) -->
<div class="step-android-action-card tool-action-card is-failed" *ngIf="item.data.type === 'llm_call' && item.data.status === 'failed'">
<div class="step-android-action-card tool-action-card is-failed"
[class.has-details]="hasLLMFailureRetryEntries(item.data)"
[class.is-expanded]="isActionExpanded('llm-failure-' + block.id + '-' + itemIdx)"
(click)="hasLLMFailureRetryEntries(item.data) && toggleActionExpanded('llm-failure-' + block.id + '-' + itemIdx, $event)"
*ngIf="isDisplayableLLMFailure(item.data)">
<div class="action-card-header llm-failure-header">
<span class="material-symbols-outlined action-icon is-failed">error</span>
<span class="action-title">LLM Request Failed</span>
@@ -596,12 +582,29 @@
<span class="material-symbols-outlined">play_arrow</span>
Resume
</button>
<span class="action-expand-icon material-symbols-outlined" *ngIf="hasLLMFailureRetryEntries(item.data)">
{{ isActionExpanded('llm-failure-' + block.id + '-' + itemIdx) ? 'expand_less' : 'expand_more' }}
</span>
</div>
<div class="action-card-body">
<div class="action-detail-row">
<span class="detail-label">Reason:</span>
<span class="detail-value">{{ getLLMErrorText(item.data) }}</span>
</div>
<div class="action-detail-row" *ngIf="getLLMFailureWaited(item.data) as waited">
<span class="detail-label">Waited:</span>
<span class="detail-value">{{ waited }}</span>
</div>
<div class="failure-retry-details" *ngIf="isActionExpanded('llm-failure-' + block.id + '-' + itemIdx)">
<div class="failure-retry-title">Retry details</div>
<div class="failure-retry-row" *ngFor="let retry of getLLMFailureRetryEntries(item.data); let retryIndex = index">
<span>Retry {{ retryIndex + 1 }}</span>
<span>
Planned {{ getLLMRetryEntryDelay(retry) || '0s' }}
· Waited {{ getLLMRetryEntryWaited(retry) }}
</span>
</div>
</div>
</div>
</div>
apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss
+85 -114
View File
@@ -1955,6 +1955,55 @@ $uniform-gap: 8px;
margin-top: 0;
margin-bottom: 0;
.llm-retry-notes {
display: flex;
flex-direction: column;
gap: $uniform-gap;
padding: 0;
.llm-retry-note {
min-height: 20px;
display: flex;
align-items: center;
gap: 8px;
color: #71717a;
font-family: 'Google Sans', 'Google Sans Text', 'Inter', Arial, sans-serif;
font-size: 12px;
line-height: 1.5;
letter-spacing: 0.2px;
.stream-status-dot {
width: 8px;
height: 8px;
border-radius: 50%;
background: #d4d4d8;
flex: 0 0 auto;
&.streaming {
background: #3b82f6;
box-shadow: 0 0 0 0 rgba(59, 130, 246, 0.45);
animation: pulse-stream-dot 1.8s cubic-bezier(0.4, 0, 0.6, 1) infinite;
}
}
.llm-retry-note-text {
color: #71717a;
display: flex;
flex-direction: column;
min-width: 0;
.llm-retry-status-line,
.llm-retry-guidance {
display: block;
}
.llm-retry-guidance {
color: #a1a1aa;
}
}
}
}
.step-generic-tool {
border: 1px solid transparent;
@@ -2245,6 +2294,10 @@ $uniform-gap: 8px;
background: #f8fafc;
border-color: #fca5a5;
&.has-details {
cursor: pointer;
}
&:hover {
border-color: #f87171;
}
@@ -2255,19 +2308,6 @@ $uniform-gap: 8px;
}
}
&.is-retrying {
background: #fffbeb;
border-color: #fcd34d;
&:hover {
border-color: #f59e0b;
}
.action-card-body .detail-value {
color: #78350f;
}
}
&.task-report-card {
background: #f0fdf4;
border: 1px solid #bbf7d0;
@@ -2373,9 +2413,6 @@ $uniform-gap: 8px;
color: #ef4444;
}
&.is-retrying {
color: #d97706;
}
}
.action-title {
@@ -2442,27 +2479,6 @@ $uniform-gap: 8px;
}
}
&.is-retrying {
background: #fef3c7;
color: #92400e;
border: 1px solid #fde68a;
.status-icon {
color: #d97706;
&.is-active {
animation: spin 1.5s linear infinite;
}
}
}
}
&.llm-retry-header {
border-bottom-color: #fde68a;
.action-title {
color: #78350f;
}
}
&.llm-failure-header {
@@ -2528,83 +2544,6 @@ $uniform-gap: 8px;
flex-direction: column;
gap: 6px;
.retry-summary-grid {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 6px;
}
.retry-summary-item {
min-width: 0;
padding: 7px 8px;
border: 1px solid rgba(245, 158, 11, 0.18);
border-radius: 7px;
background: rgba(255, 255, 255, 0.68);
display: flex;
flex-direction: column;
gap: 2px;
.retry-summary-label {
color: #a16207;
font-size: 9.5px;
font-weight: 600;
letter-spacing: 0.02em;
text-transform: uppercase;
}
strong {
color: #78350f;
font-size: 11.5px;
font-weight: 700;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
}
.retry-attempt-list {
border-top: 1px solid rgba(245, 158, 11, 0.22);
display: flex;
flex-direction: column;
gap: 6px;
margin-top: 2px;
padding-top: 8px;
animation: fadeInDetails 0.2s ease-out;
.retry-attempt-item {
padding: 8px 9px;
border: 1px solid rgba(245, 158, 11, 0.2);
border-radius: 7px;
background: rgba(255, 255, 255, 0.76);
}
.retry-attempt-heading {
display: flex;
align-items: center;
justify-content: space-between;
gap: 8px;
margin-bottom: 4px;
color: #92400e;
font-size: 10px;
font-weight: 700;
}
.retry-wait-pill {
padding: 1px 5px;
border-radius: 999px;
background: #fef3c7;
color: #b45309;
}
.retry-attempt-error {
color: #78350f;
font-size: 10.5px;
line-height: 1.45;
overflow-wrap: anywhere;
user-select: text;
}
}
.action-detail-row {
display: flex;
align-items: baseline;
@@ -2654,6 +2593,38 @@ $uniform-gap: 8px;
}
}
}
.failure-retry-details {
margin-top: 4px;
padding-top: 9px;
border-top: 1px solid #fecaca;
display: flex;
flex-direction: column;
gap: 7px;
animation: fadeInDetails 0.2s ease-out;
.failure-retry-title {
color: #991b1b;
font-size: 10px;
font-weight: 700;
text-transform: uppercase;
letter-spacing: 0.03em;
}
.failure-retry-row {
display: flex;
justify-content: space-between;
gap: 12px;
color: #7f1d1d;
font-size: 10.5px;
line-height: 1.4;
span:last-child {
color: #64748b;
text-align: right;
}
}
}
}
.action-card-expanded-details {
apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts
+60 -27
View File
@@ -158,6 +158,7 @@ export class AgentStreamComponent implements AfterViewInit {
// Set to track expanded and collapsed state of action cards
public expandedActionCards = signal<Set<string>>(new Set<string>());
public collapsedActionCards = signal<Set<string>>(new Set<string>());
public retryClock = signal<number>(Date.now());
// Performance caches for parameter and event extractions
private actionParamsCache = new WeakMap<any, ActionParam[]>();
@@ -280,6 +281,9 @@ export class AgentStreamComponent implements AfterViewInit {
}
constructor() {
const retryClockInterval = setInterval(() => this.retryClock.set(Date.now()), 1000);
this.destroyRef.onDestroy(() => clearInterval(retryClockInterval));
// Auto scroll stream box during active text streaming
effect(() => {
const logs = this.filteredLogs();
@@ -827,7 +831,12 @@ export class AgentStreamComponent implements AfterViewInit {
if (!block) return false;
const hasNative = Boolean(this.getNativeThinking(block));
const hasRaw = Boolean(this.getRawThinking(block));
const hasVisibleTools = Boolean(block.data?.generic_tools) && block.data.generic_tools.some((t: any) => this.shouldShowTool(t, block.data) || (t.type === 'llm_call' && (t.status === 'failed' || t.status === 'retrying')) || this.isReportStatusAction(t));
const hasVisibleTools = Boolean(block.data?.generic_tools) && block.data.generic_tools.some((t: any) =>
this.shouldShowTool(t, block.data)
|| this.isDisplayableLLMFailure(t)
|| this.isLLMRetry(t)
|| this.isReportStatusAction(t)
);
const hasAndroidActions = Boolean(block.data?.action_taken) && (this.isAndroidAction(block.data.action_taken) || this.isReportStatusAction(block.data.action_taken));
return hasNative || hasRaw || hasVisibleTools || hasAndroidActions;
}
@@ -852,6 +861,12 @@ export class AgentStreamComponent implements AfterViewInit {
return !!tool && this.retryablePauseTrace() === tool;
}
public isDisplayableLLMFailure(tool: any): boolean {
return tool?.type === 'llm_call'
&& tool?.status === 'failed'
&& (tool?.name === 'llm_pause' || tool?.payload?.pause === true);
}
public resumePausedTask(event: Event): void {
event.stopPropagation();
this.agentService.resumeTask();
@@ -865,7 +880,10 @@ export class AgentStreamComponent implements AfterViewInit {
}
public isLLMRetry(tool: any): boolean {
return tool?.type === 'llm_call' && tool?.status === 'retrying';
return tool?.type === 'llm_call'
&& tool?.status === 'retrying'
&& tool?.payload?.source === 'provider_sdk'
&& ['google', 'gemini'].includes(String(tool?.payload?.provider || '').toLowerCase());
}
public getLLMRetryEntries(tool: any): any[] {
@@ -873,25 +891,17 @@ export class AgentStreamComponent implements AfterViewInit {
return Array.isArray(retries) && retries.length > 0 ? retries : [tool?.payload || tool];
}
public getLLMRetryCount(tool: any): number {
const count = Number(tool?.payload?.retry_count);
return Number.isFinite(count) && count > 0 ? count : this.getLLMRetryEntries(tool).length;
public getLLMFailureRetryEntries(tool: any): any[] {
const retries = tool?.payload?.retries;
if (!Array.isArray(retries)) return [];
return retries.filter((retry: any) =>
retry?.source === 'provider_sdk'
&& ['google', 'gemini'].includes(String(retry?.provider || '').toLowerCase())
);
}
public getLLMRetryTotalDelay(tool: any): string | null {
const configuredTotal = Number(tool?.payload?.total_delay);
const total = Number.isFinite(configuredTotal)
? configuredTotal
: this.getLLMRetryEntries(tool).reduce(
(sum: number, retry: any) => sum + (Number(retry?.delay) || 0),
0
);
if (total <= 0) return null;
return `${total.toFixed(2).replace(/\.00$/, '')}s`;
}
public getLLMRetryEntryError(entry: any): string {
return this.cleanErrorMessage(entry?.error || 'Unknown error');
public hasLLMFailureRetryEntries(tool: any): boolean {
return this.getLLMFailureRetryEntries(tool).length > 0;
}
public getLLMRetryEntryDelay(entry: any): string | null {
@@ -900,14 +910,37 @@ export class AgentStreamComponent implements AfterViewInit {
return `${delay.toFixed(2).replace(/\.00$/, '')}s`;
}
public getLLMRetryProvider(tool: any): string | null {
const providers = Array.isArray(tool?.payload?.providers)
? tool.payload.providers
: [tool?.payload?.provider];
const labels = providers
.filter((provider: any) => typeof provider === 'string' && provider.trim())
.map((provider: string) => provider.trim().replace(/^./, (value: string) => value.toUpperCase()));
return labels.length > 0 ? Array.from(new Set(labels)).join(', ') : null;
public getLLMRetryEntryWaited(entry: any): string {
const delay = Number(entry?.delay);
if (!Number.isFinite(delay) || delay <= 0) return '0s';
const rawStartedAt = Number(entry?.scheduled_at ?? entry?.timestamp);
if (!Number.isFinite(rawStartedAt) || rawStartedAt <= 0) {
return this.formatLLMDuration(delay);
}
const startedAtMs = rawStartedAt < 1e11 ? rawStartedAt * 1000 : rawStartedAt;
const elapsed = Math.min(delay, Math.max(0, (this.retryClock() - startedAtMs) / 1000));
return this.formatLLMDuration(elapsed);
}
public isLLMRetryEntryWaiting(entry: any): boolean {
const delay = Number(entry?.delay);
const rawStartedAt = Number(entry?.scheduled_at ?? entry?.timestamp);
if (!Number.isFinite(delay) || delay <= 0 || !Number.isFinite(rawStartedAt)) return false;
const startedAtMs = rawStartedAt < 1e11 ? rawStartedAt * 1000 : rawStartedAt;
return this.retryClock() < startedAtMs + delay * 1000;
}
public getLLMFailureWaited(tool: any): string | null {
const waited = Number(tool?.payload?.waited_seconds);
if (!Number.isFinite(waited) || waited < 0) return null;
return this.formatLLMDuration(waited);
}
public formatLLMDuration(seconds: number): string {
if (seconds < 0.05) return '0s';
if (seconds < 10) return `${seconds.toFixed(1).replace(/\.0$/, '')}s`;
return `${Math.round(seconds)}s`;
}
public cleanErrorMessage(rawError: any): string {
apps/showcase_ui/src/app/services/agent.service.spec.ts
+110
View File
@@ -0,0 +1,110 @@
import { signal } from '@angular/core';
import { of } from 'rxjs';
import { AgentService } from './agent.service';
describe('AgentService live LLM retry timeline', () => {
function createServiceWithoutPolling(): AgentService {
const service = Object.create(AgentService.prototype) as AgentService;
service.sessionLogs = signal<any[]>([]);
return service;
}
it('normalizes a live retry event into the historical trace contract', () => {
const service = createServiceWithoutPolling();
const event = {
trace_id: 'retry-1',
step_id: 'step-1',
timestamp: 100,
error: '503 high demand',
delay: 1.5,
provider: 'google',
source: 'provider_sdk',
recoverable: true,
request_id: 'request-1',
scheduled_at: 100
};
(service as any).appendLiveLLMRetryTrace(event, 'session-1');
const [log] = service.sessionLogs();
expect(log.type).toBe('trace_recorded');
expect(log.data).toEqual(jasmine.objectContaining({
trace_id: 'retry-1',
session_id: 'session-1',
step_id: 'step-1',
type: 'llm_call',
name: 'llm_retry',
status: 'retrying'
}));
expect(log.data.payload).toEqual(jasmine.objectContaining({
error: '503 high demand',
delay: 1.5,
provider: 'google',
source: 'provider_sdk',
request_id: 'request-1'
}));
});
it('upserts duplicate live retry events instead of adding another row', () => {
const service = createServiceWithoutPolling();
const event = {
trace_id: 'retry-1',
timestamp: 100,
delay: 1.5,
provider: 'google',
source: 'provider_sdk',
request_id: 'request-1',
scheduled_at: 100
};
(service as any).appendLiveLLMRetryTrace(event, 'session-1');
(service as any).appendLiveLLMRetryTrace({ ...event, error: 'updated 503' }, 'session-1');
expect(service.sessionLogs().length).toBe(1);
expect(service.sessionLogs()[0].data.payload.error).toBe('updated 503');
});
it('replaces an old history snapshot while preserving live retry events', () => {
const service = createServiceWithoutPolling();
(service as any).currentSessionId = signal<string | null>('session-1');
(service as any).http = {
get: () => of([{
step_id: 'step-1',
timestamp: 100,
generic_tools: [{ trace_id: 'persisted-retry', name: 'llm_retry' }]
}])
};
service.sessionLogs.set([
{ type: 'step_updated', history_snapshot: true, data: { step_id: 'old-step' } },
{ type: 'trace_recorded', data: { trace_id: 'live-retry', name: 'llm_retry' } }
]);
(service as any).backfillSessionSteps('session-1');
const logs = service.sessionLogs();
expect(logs.length).toBe(2);
expect(logs[0].history_snapshot).toBeTrue();
expect(logs[0].data.step_id).toBe('step-1');
expect(logs[1].data.trace_id).toBe('live-retry');
});
it('follows a just-started task even when status polling saw it first', () => {
const service = createServiceWithoutPolling();
(service as any).http = {
post: () => of({ tasks: [{ session_id: 'new-session' }] })
};
service.agentStatus = signal('running');
service.runningSessionId = signal<string | null>('new-session');
service.userPinnedSessionId = signal<string | null>(null);
(service as any).sessions = signal<any[]>([{
session_id: 'new-session',
status: 'running'
}]);
const selectSpy = spyOn(service, 'selectSession');
service.runTask('test goal').subscribe();
expect(selectSpy).toHaveBeenCalledWith('new-session', false);
});
});
apps/showcase_ui/src/app/services/agent.service.ts
+135 -48
View File
@@ -184,7 +184,12 @@ export class AgentService {
const isCurrentlyRunning =
this.agentStatus() === 'running' ||
this.sessions().some((s) => s.status === 'running');
if (!isCurrentlyRunning) {
const activeSessionId = this.runningSessionId()
|| this.sessions().find((s) => s.status === 'running' || s.status === 'paused')?.session_id;
// The status poll can observe the new runner before /api/run
// returns. In that case it is still the task we just submitted,
// not an older task that should remain selected.
if (!isCurrentlyRunning || activeSessionId === newSessionId) {
this.selectSession(newSessionId, false);
}
}
@@ -401,45 +406,16 @@ export class AgentService {
}
console.log(`Connecting to session: ${sessionId}`);
this.http.get<StepItemData[]>(`/api/sessions/${sessionId}/steps`).subscribe({
next: (steps) => {
if (this.currentSessionId() !== sessionId) return;
const historicalLogs: any[] = [];
steps.forEach((step) => {
const stepTimeIso = new Date((step.timestamp || Date.now() / 1000) * 1000).toISOString();
historicalLogs.push({
type: 'step_updated',
session_id: sessionId,
timestamp: stepTimeIso,
data: step
});
});
// The history request and EventSource start together. On slower Windows
// hosts, live events can arrive first; replacing the signal here would
// erase them until a browser refresh reloads the persisted steps.
this.sessionLogs.update((liveLogs) => {
if (liveLogs.length === 0) return historicalLogs;
const liveStepIds = new Set(
liveLogs
.filter((log) => log.type === 'step_recorded' || log.type === 'step_updated')
.map((log) => String(log.data?.step_id || log.data?.step_number || ''))
.filter(Boolean)
);
const missingHistorical = historicalLogs.filter((log) => {
const stepKey = String(log.data?.step_id || log.data?.step_number || '');
return !stepKey || !liveStepIds.has(stepKey);
});
return [...missingHistorical, ...liveLogs];
});
},
error: (err) => {
console.error('Failed to pre-load historical steps:', err);
}
});
this.eventSource = new EventSource(`/api/stream/${sessionId}`);
// The server emits info only after this client has been registered as an
// SSE subscriber. Fetching the persisted snapshot after that acknowledgement
// closes the snapshot-to-stream gap: earlier events are in history and later
// events are already queued for this connection.
this.eventSource.addEventListener('info', () => {
this.backfillSessionSteps(sessionId);
});
// Listen to standard SSE keep-alive
this.eventSource.addEventListener('keep-alive', () => {
// No-op keep-alive tick
@@ -524,6 +500,7 @@ export class AgentService {
const attemptText = attempt && max ? ` (Attempt ${attempt}/${max})` : '';
const delayText = delay > 0 ? `; retrying in ${delay.toFixed(2).replace(/\.00$/, '')}s` : '';
this.retryMessage.set(`AI service is temporarily busy${attemptText}${delayText}...`);
this.appendLiveLLMRetryTrace(parsedData, sessionId);
return;
}
@@ -536,7 +513,8 @@ export class AgentService {
pauseError,
sessionId,
parsedData.timestamp,
parsedData.step_id
parsedData.step_id,
parsedData
);
return;
}
@@ -628,14 +606,28 @@ export class AgentService {
? new Date(parsedData.timestamp * 1000).toISOString()
: new Date().toISOString();
return [
...updatedLogs,
{
type: eventType,
timestamp: evtTime,
data: parsedData
const nextLog = {
type: eventType,
timestamp: evtTime,
data: parsedData
};
// A retry is published both as trace_recorded and as the
// purpose-built llm_retrying event. Replace the optimistic live
// copy when the persisted trace arrives instead of duplicating it.
if (eventType === 'trace_recorded' && parsedData?.trace_id) {
const existingTraceIndex = updatedLogs.findIndex((log) =>
log.type === 'trace_recorded'
&& log.data?.trace_id === parsedData.trace_id
);
if (existingTraceIndex > -1) {
const deduplicatedLogs = [...updatedLogs];
deduplicatedLogs[existingTraceIndex] = nextLog;
return deduplicatedLogs;
}
];
}
return [...updatedLogs, nextLog];
});
}
@@ -668,6 +660,93 @@ export class AgentService {
};
}
/**
* Load a fresh persisted snapshot without replacing events already received
* from SSE. Snapshot logs are replaceable, while live logs remain append-only;
* the stream aggregator merges matching steps and trace IDs within each step.
*/
private backfillSessionSteps(sessionId: string): void {
this.http.get<StepItemData[]>(`/api/sessions/${sessionId}/steps`).subscribe({
next: (steps) => {
if (this.currentSessionId() !== sessionId) return;
const historicalLogs = steps.map((step) => ({
type: 'step_updated',
session_id: sessionId,
timestamp: new Date((step.timestamp || Date.now() / 1000) * 1000).toISOString(),
data: step,
history_snapshot: true
}));
this.sessionLogs.update((logs) => [
...historicalLogs,
...logs.filter((log) => !log.history_snapshot)
]);
},
error: (err) => {
console.error('Failed to backfill session steps:', err);
}
});
}
/**
* Convert the dedicated live retry event into the same trace shape used by
* history APIs. This keeps live streaming and post-refresh rendering on one
* data contract instead of maintaining a separate retry UI path.
*/
private appendLiveLLMRetryTrace(data: any, sessionId: string): void {
const timestampSeconds = typeof data?.timestamp === 'number'
? (data.timestamp > 1e11 ? data.timestamp / 1000 : data.timestamp)
: Date.now() / 1000;
const traceId = data?.trace_id
|| `llm-retry-live-${data?.request_id || 'unknown'}-${timestampSeconds}`;
const payload = {
error: data?.error,
delay: data?.delay,
attempt: data?.attempt,
max_retries: data?.max_retries,
provider: data?.provider,
source: data?.source,
recoverable: data?.recoverable,
request_id: data?.request_id,
scheduled_at: data?.scheduled_at ?? timestampSeconds
};
const retryTrace = {
trace_id: traceId,
session_id: sessionId,
step_id: data?.step_id || null,
type: 'llm_call',
name: 'llm_retry',
timestamp: timestampSeconds,
status: 'retrying',
payload
};
this.sessionLogs.update((logs) => {
const existingIndex = logs.findIndex((log) =>
log.type === 'trace_recorded'
&& (
log.data?.trace_id === traceId
|| (
data?.request_id
&& log.data?.payload?.request_id === data.request_id
&& Number(log.data?.payload?.scheduled_at) === Number(payload.scheduled_at)
)
)
);
const retryLog = {
type: 'trace_recorded',
session_id: sessionId,
timestamp: new Date(timestampSeconds * 1000).toISOString(),
data: retryTrace
};
if (existingIndex < 0) return [...logs, retryLog];
const updatedLogs = [...logs];
updatedLogs[existingIndex] = retryLog;
return updatedLogs;
});
}
/**
* Preserve pause failures in the normal failed-LLM card stream. New runners
* emit a persisted failed trace first; this fallback also supports older
@@ -677,7 +756,8 @@ export class AgentService {
error: unknown,
sessionId: string,
timestamp?: number | string,
stepId?: string | null
stepId?: string | null,
details?: any
): void {
const errorText = typeof error === 'string' ? error : JSON.stringify(error);
const pauseKey = `${sessionId}:${errorText}`;
@@ -717,7 +797,14 @@ export class AgentService {
name: 'llm_pause',
status: 'failed',
timestamp: timestampMs / 1000,
payload: { error: errorText, pause: true }
payload: {
error: errorText,
pause: true,
request_id: details?.request_id,
provider: details?.provider,
waited_seconds: details?.waited_seconds,
retries: Array.isArray(details?.retries) ? details.retries : []
}
}
}
];
apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts
+27 -7
View File
@@ -9,7 +9,7 @@ describe('getUniqueGenericTools retry aggregation', () => {
name: 'llm_retry',
status: 'retrying',
timestamp: 100,
payload: { error: '503 first', delay: 1.18, provider: 'google' }
payload: { error: '503 first', delay: 1.18, provider: 'google', source: 'provider_sdk', request_id: 'request-1' }
},
{
trace_id: 'retry-2',
@@ -17,7 +17,7 @@ describe('getUniqueGenericTools retry aggregation', () => {
name: 'llm_retry',
status: 'retrying',
timestamp: 101,
payload: { error: '503 second', delay: 2.96, provider: 'google' }
payload: { error: '503 second', delay: 2.96, provider: 'google', source: 'provider_sdk', request_id: 'request-1' }
}
];
@@ -27,31 +27,51 @@ describe('getUniqueGenericTools retry aggregation', () => {
expect(result[0].name).toBe('llm_retry_group');
expect(result[0].payload.retry_count).toBe(2);
expect(result[0].payload.total_delay).toBeCloseTo(4.14);
expect(result[0].payload.providers).toEqual(['google']);
expect(result[0].payload.retries.map((retry: any) => retry.error)).toEqual([
'503 first',
'503 second'
]);
});
it('keeps a terminal LLM failure separate from its preceding retries', () => {
it('folds SDK retry rows into their matching terminal failure', () => {
const result = getUniqueGenericTools([
{
trace_id: 'retry-1',
type: 'llm_call',
name: 'llm_retry',
status: 'retrying',
payload: { error: '503', delay: 1 }
payload: { error: '503', delay: 1, provider: 'google', source: 'provider_sdk', request_id: 'request-1' }
},
{
trace_id: 'failure-1',
type: 'llm_call',
name: 'llm_pause',
status: 'failed',
payload: { error: '503 exhausted', pause: true }
payload: { error: '503 exhausted', pause: true, request_id: 'request-1', retries: [{ delay: 1 }] }
}
]);
expect(result.map(tool => tool.name)).toEqual(['llm_retry_group', 'llm_pause']);
expect(result.map(tool => tool.name)).toEqual(['llm_pause']);
});
it('does not expose opaque retries from non-Gemini providers', () => {
const result = getUniqueGenericTools([
{
trace_id: 'retry-openai',
type: 'llm_call',
name: 'llm_retry',
status: 'retrying',
payload: { error: '429', delay: 2, provider: 'openai', source: 'artemis_wrapper' }
},
{
trace_id: 'failure-openai',
type: 'llm_call',
name: 'llm_pause',
status: 'failed',
payload: { error: '429 exhausted', pause: true, waited_seconds: 12 }
}
]);
expect(result.map(tool => tool.name)).toEqual(['llm_pause']);
});
});
apps/showcase_ui/src/app/utils/tool-formatter.util.ts
+62 -35
View File
@@ -185,41 +185,57 @@ export function getToolAgentName(tool: any): string | null {
export function getUniqueGenericTools(tools: any[] | undefined): any[] {
if (!tools) return [];
// Only Google GenAI SDK retries are observable. Artemis wrapper retries and
// opaque provider retries must never be presented as provider internals.
const retryTools = tools.filter(tool =>
tool?.type === 'llm_call'
&& tool?.status === 'retrying'
&& (tool?.name === 'llm_retry' || tool?.name === 'llm_retry_group')
&& tool?.payload?.source === 'provider_sdk'
&& ['google', 'gemini'].includes(String(tool?.payload?.provider || '').toLowerCase())
);
const firstRetry = retryTools[0];
const latestRetry = retryTools[retryTools.length - 1];
const retryAggregate = firstRetry ? {
...firstRetry,
trace_id: `llm-retry-group-${firstRetry.trace_id || firstRetry.timestamp || 'unknown'}`,
name: 'llm_retry_group',
payload: {
...(latestRetry?.payload || {}),
retry_count: retryTools.length,
total_delay: retryTools.reduce(
(total, retry) => total + (Number(retry?.payload?.delay) || 0),
0
),
providers: Array.from(new Set(
retryTools
.map(retry => retry?.payload?.provider)
.filter((provider): provider is string => typeof provider === 'string' && !!provider)
)),
retries: retryTools.map(retry => ({
trace_id: retry.trace_id,
timestamp: retry.timestamp,
error: retry?.payload?.error || retry.error,
delay: Number(retry?.payload?.delay) || 0,
provider: retry?.payload?.provider,
source: retry?.payload?.source,
attempt: retry?.payload?.attempt,
max_retries: retry?.payload?.max_retries
}))
}
} : null;
const retryKey = (tool: any) => String(
tool?.payload?.request_id || `legacy-${tool?.step_id || 'unassigned'}`
);
const retryGroups = new Map<string, any[]>();
for (const retry of retryTools) {
const key = retryKey(retry);
retryGroups.set(key, [...(retryGroups.get(key) || []), retry]);
}
const terminalRequestIds = new Set(
tools
.filter(tool => tool?.type === 'llm_call' && tool?.status === 'failed')
.map(tool => tool?.payload?.request_id)
.filter((requestId): requestId is string => typeof requestId === 'string' && !!requestId)
);
const buildRetryAggregate = (group: any[]) => {
const firstRetry = group[0];
const latestRetry = group[group.length - 1];
return {
...firstRetry,
trace_id: `llm-retry-group-${firstRetry?.payload?.request_id || firstRetry?.trace_id || firstRetry?.timestamp || 'unknown'}`,
name: 'llm_retry_group',
payload: {
...(latestRetry?.payload || {}),
retry_count: group.length,
total_delay: group.reduce(
(total, retry) => total + (Number(retry?.payload?.delay) || 0),
0
),
retries: group.map(retry => ({
trace_id: retry.trace_id,
timestamp: retry.timestamp,
error: retry?.payload?.error || retry.error,
delay: Number(retry?.payload?.delay) || 0,
provider: retry?.payload?.provider,
source: retry?.payload?.source,
request_id: retry?.payload?.request_id,
scheduled_at: retry?.payload?.scheduled_at
}))
}
};
};
const toolMap = new Map<string, any>();
for (const t of tools) {
@@ -232,22 +248,33 @@ export function getUniqueGenericTools(tools: any[] | undefined): any[] {
const result: any[] = [];
const seenTraces = new Set<string>();
let retryAggregateAdded = false;
const addedRetryGroups = new Set<string>();
for (const tool of tools) {
if (!tool || !tool.name) continue;
if (tool.type === 'agent') continue;
if (tool.type === 'llm_call' && tool.status === 'retrying') {
if (!retryAggregateAdded && retryAggregate) {
result.push(retryAggregate);
retryAggregateAdded = true;
const groupKey = retryKey(tool);
const group = retryGroups.get(groupKey);
const requestId = tool?.payload?.request_id;
if (
group
&& !addedRetryGroups.has(groupKey)
&& !(requestId && terminalRequestIds.has(requestId))
) {
result.push(buildRetryAggregate(group));
addedRetryGroups.add(groupKey);
}
continue;
}
if (tool.type === 'llm_call' && tool.status === 'failed') {
result.push(tool);
// Callback-level attempt errors are implementation details. The wrapper
// emits one terminal llm_pause trace after retries are exhausted.
if (tool.name === 'llm_pause' || tool?.payload?.pause === true) {
result.push(tool);
}
continue;
}