Improve task resilience and LLM retry visibility
Persist and publish recoverable provider retries so temporary 503 and throttling events remain visible in live and historical task streams. Aggregate repeated retries into an expandable action card, preserve terminal pause recovery, harden Windows worker isolation and output forwarding, and retain reliable Android hierarchy fallbacks with regression coverage.
This commit is contained in:
somew1nd committed
1 parent
c1a4b49a21
commit
0f5fb1d32c
17 files changed
+946
-97
No files matched your search
@@ -285,18 +285,33 @@
|
||||
|
||||
<!-- Event Logs Section -->
|
||||
<main class="stream-logs-content" (scroll)="onStreamScroll($event)">
|
||||
<!-- Keep recovery available even when a persisted failure trace is absent. -->
|
||||
<div class="paused-task-banner" *ngIf="isViewingPausedTask()" role="status">
|
||||
<div class="paused-task-message">
|
||||
<span class="material-symbols-outlined paused-task-icon">pause_circle</span>
|
||||
<div>
|
||||
<strong>Task paused</strong>
|
||||
<span>The AI request failed. Resume the task to try again.</span>
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
class="btn-resume-task"
|
||||
(click)="resumePausedTask($event)"
|
||||
title="Resume the paused task"
|
||||
aria-label="Resume the paused task"
|
||||
>
|
||||
<span class="material-symbols-outlined">play_arrow</span>
|
||||
Resume
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div *ngIf="consolidatedBlocks().length === 0 && !showPlanningLoader()" class="empty-state">
|
||||
<div class="empty-icon"><span class="material-symbols-outlined">sensors</span></div>
|
||||
<h3>No Stream Activity Selected</h3>
|
||||
<p>Please select a session from the Task List on the right to view its real-time log payloads.</p>
|
||||
</div>
|
||||
|
||||
<!-- Retrying Banner -->
|
||||
<div class="retrying-banner" *ngIf="agentService.isRetrying()">
|
||||
<span class="material-symbols-outlined spinning-icon">sync</span>
|
||||
<span>{{ agentService.retryMessage() || 'Reconnecting to AI model...' }}</span>
|
||||
</div>
|
||||
|
||||
<!-- The phase-based session logs list -->
|
||||
<div class="logs-list-container" *ngIf="phases().length > 0 || showPlanningLoader()">
|
||||
<div class="phase-container" *ngFor="let phase of phases(); let first = first; trackBy: trackPhase">
|
||||
@@ -513,6 +528,48 @@
|
||||
|
||||
<!-- Generic Tools & LLM Calls (when item is a tool) -->
|
||||
<ng-container *ngIf="item.type === 'tool'">
|
||||
<!-- Recoverable provider/LLM retries remain visible after the task continues. -->
|
||||
<div class="step-android-action-card tool-action-card is-retrying"
|
||||
[class.is-expanded]="isActionExpanded('retry-' + block.id + '-' + itemIdx)"
|
||||
*ngIf="isLLMRetry(item.data)">
|
||||
<div class="action-card-header llm-retry-header"
|
||||
(click)="toggleActionExpanded('retry-' + block.id + '-' + itemIdx, $event)">
|
||||
<span class="material-symbols-outlined action-icon is-retrying">cloud_sync</span>
|
||||
<span class="action-title">AI service retry</span>
|
||||
<span class="action-status-badge is-retrying">
|
||||
{{ getLLMRetryCount(item.data) }}× retried
|
||||
</span>
|
||||
<span class="action-expand-icon material-symbols-outlined">
|
||||
{{ isActionExpanded('retry-' + block.id + '-' + itemIdx) ? 'expand_less' : 'expand_more' }}
|
||||
</span>
|
||||
</div>
|
||||
<div class="action-card-body">
|
||||
<div class="retry-summary-grid">
|
||||
<div class="retry-summary-item">
|
||||
<span class="retry-summary-label">Attempts</span>
|
||||
<strong>{{ getLLMRetryCount(item.data) }}</strong>
|
||||
</div>
|
||||
<div class="retry-summary-item" *ngIf="getLLMRetryTotalDelay(item.data) as totalDelay">
|
||||
<span class="retry-summary-label">Extra wait</span>
|
||||
<strong>{{ totalDelay }}</strong>
|
||||
</div>
|
||||
<div class="retry-summary-item" *ngIf="getLLMRetryProvider(item.data) as retryProvider">
|
||||
<span class="retry-summary-label">Provider</span>
|
||||
<strong>{{ retryProvider }}</strong>
|
||||
</div>
|
||||
</div>
|
||||
<div class="retry-attempt-list" *ngIf="isActionExpanded('retry-' + block.id + '-' + itemIdx)">
|
||||
<div class="retry-attempt-item" *ngFor="let retry of getLLMRetryEntries(item.data); let retryIndex = index">
|
||||
<div class="retry-attempt-heading">
|
||||
<span>Retry {{ retryIndex + 1 }}</span>
|
||||
<span class="retry-wait-pill" *ngIf="getLLMRetryEntryDelay(retry) as retryDelay">+{{ retryDelay }}</span>
|
||||
</div>
|
||||
<div class="retry-attempt-error">{{ getLLMRetryEntryError(retry) }}</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Failed LLM Calls (Permanently maintain Action Card format in red) -->
|
||||
<div class="step-android-action-card tool-action-card is-failed" *ngIf="item.data.type === 'llm_call' && item.data.status === 'failed'">
|
||||
<div class="action-card-header llm-failure-header">
|
||||
@@ -530,14 +587,14 @@
|
||||
</ng-template>
|
||||
<button
|
||||
type="button"
|
||||
class="btn-retry-llm"
|
||||
class="btn-resume-llm"
|
||||
*ngIf="isTerminalLLMFailure(item.data)"
|
||||
(click)="retryPausedTask($event)"
|
||||
title="Retry the paused AI request"
|
||||
aria-label="Retry the paused AI request"
|
||||
(click)="resumePausedTask($event)"
|
||||
title="Resume the paused task"
|
||||
aria-label="Resume the paused task"
|
||||
>
|
||||
<span class="material-symbols-outlined">refresh</span>
|
||||
Retry
|
||||
<span class="material-symbols-outlined">play_arrow</span>
|
||||
Resume
|
||||
</button>
|
||||
</div>
|
||||
<div class="action-card-body">
|
||||
|
||||
@@ -1053,24 +1053,96 @@ $uniform-gap: 8px;
|
||||
}
|
||||
}
|
||||
|
||||
.retrying-banner {
|
||||
.paused-task-banner {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 10px 16px;
|
||||
background-color: #fef9c3;
|
||||
border: 1px solid #fde047;
|
||||
border-radius: 8px;
|
||||
color: #854d0e;
|
||||
font-size: 13px;
|
||||
font-weight: 500;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
padding: 12px 14px;
|
||||
margin-bottom: 12px;
|
||||
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.05);
|
||||
background: #fff7ed;
|
||||
border: 1px solid #fdba74;
|
||||
border-radius: 10px;
|
||||
color: #9a3412;
|
||||
box-shadow: 0 1px 3px rgba(154, 52, 18, 0.08);
|
||||
|
||||
.spinning-icon {
|
||||
font-size: 18px;
|
||||
color: #d97706;
|
||||
animation: spin 1.5s linear infinite;
|
||||
.paused-task-message {
|
||||
min-width: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
|
||||
.paused-task-icon {
|
||||
flex: 0 0 auto;
|
||||
font-size: 21px;
|
||||
color: #ea580c;
|
||||
}
|
||||
|
||||
div {
|
||||
min-width: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
}
|
||||
|
||||
strong {
|
||||
font-size: 12px;
|
||||
line-height: 1.2;
|
||||
}
|
||||
|
||||
span:not(.paused-task-icon) {
|
||||
font-size: 11px;
|
||||
line-height: 1.35;
|
||||
color: #c2410c;
|
||||
}
|
||||
}
|
||||
|
||||
.btn-resume-task {
|
||||
flex: 0 0 auto;
|
||||
min-height: 32px;
|
||||
padding: 6px 12px;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 5px;
|
||||
border: 1px solid #2563eb;
|
||||
border-radius: 8px;
|
||||
background: #2563eb;
|
||||
color: #ffffff;
|
||||
font-family: inherit;
|
||||
font-size: 12px;
|
||||
font-weight: 700;
|
||||
line-height: 1;
|
||||
cursor: pointer;
|
||||
box-shadow: 0 2px 5px rgba(37, 99, 235, 0.2);
|
||||
transition: background 0.18s ease, box-shadow 0.18s ease, transform 0.18s ease;
|
||||
|
||||
.material-symbols-outlined {
|
||||
font-size: 17px;
|
||||
}
|
||||
|
||||
&:hover {
|
||||
background: #1d4ed8;
|
||||
box-shadow: 0 3px 8px rgba(37, 99, 235, 0.28);
|
||||
}
|
||||
|
||||
&:active {
|
||||
transform: scale(0.97);
|
||||
}
|
||||
|
||||
&:focus-visible {
|
||||
outline: 2px solid rgba(37, 99, 235, 0.35);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 560px) {
|
||||
align-items: stretch;
|
||||
flex-direction: column;
|
||||
|
||||
.btn-resume-task {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2183,6 +2255,19 @@ $uniform-gap: 8px;
|
||||
}
|
||||
}
|
||||
|
||||
&.is-retrying {
|
||||
background: #fffbeb;
|
||||
border-color: #fcd34d;
|
||||
|
||||
&:hover {
|
||||
border-color: #f59e0b;
|
||||
}
|
||||
|
||||
.action-card-body .detail-value {
|
||||
color: #78350f;
|
||||
}
|
||||
}
|
||||
|
||||
&.task-report-card {
|
||||
background: #f0fdf4;
|
||||
border: 1px solid #bbf7d0;
|
||||
@@ -2287,6 +2372,10 @@ $uniform-gap: 8px;
|
||||
&.is-failed {
|
||||
color: #ef4444;
|
||||
}
|
||||
|
||||
&.is-retrying {
|
||||
color: #d97706;
|
||||
}
|
||||
}
|
||||
|
||||
.action-title {
|
||||
@@ -2352,12 +2441,34 @@ $uniform-gap: 8px;
|
||||
color: #f97316;
|
||||
}
|
||||
}
|
||||
|
||||
&.is-retrying {
|
||||
background: #fef3c7;
|
||||
color: #92400e;
|
||||
border: 1px solid #fde68a;
|
||||
|
||||
.status-icon {
|
||||
color: #d97706;
|
||||
|
||||
&.is-active {
|
||||
animation: spin 1.5s linear infinite;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
&.llm-retry-header {
|
||||
border-bottom-color: #fde68a;
|
||||
|
||||
.action-title {
|
||||
color: #78350f;
|
||||
}
|
||||
}
|
||||
|
||||
&.llm-failure-header {
|
||||
flex-wrap: wrap;
|
||||
|
||||
.btn-retry-llm {
|
||||
.btn-resume-llm {
|
||||
flex: 0 0 auto;
|
||||
min-height: 26px;
|
||||
padding: 3px 9px;
|
||||
@@ -2417,6 +2528,83 @@ $uniform-gap: 8px;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
|
||||
.retry-summary-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.retry-summary-item {
|
||||
min-width: 0;
|
||||
padding: 7px 8px;
|
||||
border: 1px solid rgba(245, 158, 11, 0.18);
|
||||
border-radius: 7px;
|
||||
background: rgba(255, 255, 255, 0.68);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
|
||||
.retry-summary-label {
|
||||
color: #a16207;
|
||||
font-size: 9.5px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.02em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
strong {
|
||||
color: #78350f;
|
||||
font-size: 11.5px;
|
||||
font-weight: 700;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
.retry-attempt-list {
|
||||
border-top: 1px solid rgba(245, 158, 11, 0.22);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
margin-top: 2px;
|
||||
padding-top: 8px;
|
||||
animation: fadeInDetails 0.2s ease-out;
|
||||
|
||||
.retry-attempt-item {
|
||||
padding: 8px 9px;
|
||||
border: 1px solid rgba(245, 158, 11, 0.2);
|
||||
border-radius: 7px;
|
||||
background: rgba(255, 255, 255, 0.76);
|
||||
}
|
||||
|
||||
.retry-attempt-heading {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
margin-bottom: 4px;
|
||||
color: #92400e;
|
||||
font-size: 10px;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.retry-wait-pill {
|
||||
padding: 1px 5px;
|
||||
border-radius: 999px;
|
||||
background: #fef3c7;
|
||||
color: #b45309;
|
||||
}
|
||||
|
||||
.retry-attempt-error {
|
||||
color: #78350f;
|
||||
font-size: 10.5px;
|
||||
line-height: 1.45;
|
||||
overflow-wrap: anywhere;
|
||||
user-select: text;
|
||||
}
|
||||
}
|
||||
|
||||
.action-detail-row {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
|
||||
@@ -230,6 +230,14 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
return checkPlanningLoader(this.filteredLogs(), isRunning, isCurrentRunningSession);
|
||||
});
|
||||
|
||||
public isViewingPausedTask = computed(() => {
|
||||
if (!this.agentService.isPaused()) return false;
|
||||
|
||||
const currentSessionId = this.agentService.currentSessionId();
|
||||
const pausedSessionId = this.agentService.runningSessionId();
|
||||
return !currentSessionId || currentSessionId === pausedSessionId;
|
||||
});
|
||||
|
||||
public sessionDuration = computed(() => {
|
||||
const currentSession = this.agentService.sessions().find(s => s.session_id === this.agentService.currentSessionId());
|
||||
return computeSessionDuration(currentSession?.start_time || 0, this.filteredLogs());
|
||||
@@ -819,7 +827,7 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
if (!block) return false;
|
||||
const hasNative = Boolean(this.getNativeThinking(block));
|
||||
const hasRaw = Boolean(this.getRawThinking(block));
|
||||
const hasVisibleTools = Boolean(block.data?.generic_tools) && block.data.generic_tools.some((t: any) => this.shouldShowTool(t, block.data) || (t.type === 'llm_call' && t.status === 'failed') || this.isReportStatusAction(t));
|
||||
const hasVisibleTools = Boolean(block.data?.generic_tools) && block.data.generic_tools.some((t: any) => this.shouldShowTool(t, block.data) || (t.type === 'llm_call' && (t.status === 'failed' || t.status === 'retrying')) || this.isReportStatusAction(t));
|
||||
const hasAndroidActions = Boolean(block.data?.action_taken) && (this.isAndroidAction(block.data.action_taken) || this.isReportStatusAction(block.data.action_taken));
|
||||
return hasNative || hasRaw || hasVisibleTools || hasAndroidActions;
|
||||
}
|
||||
@@ -844,7 +852,7 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
return !!tool && this.retryablePauseTrace() === tool;
|
||||
}
|
||||
|
||||
public retryPausedTask(event: Event): void {
|
||||
public resumePausedTask(event: Event): void {
|
||||
event.stopPropagation();
|
||||
this.agentService.resumeTask();
|
||||
}
|
||||
@@ -856,6 +864,52 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
return this.cleanErrorMessage(rawError);
|
||||
}
|
||||
|
||||
public isLLMRetry(tool: any): boolean {
|
||||
return tool?.type === 'llm_call' && tool?.status === 'retrying';
|
||||
}
|
||||
|
||||
public getLLMRetryEntries(tool: any): any[] {
|
||||
const retries = tool?.payload?.retries;
|
||||
return Array.isArray(retries) && retries.length > 0 ? retries : [tool?.payload || tool];
|
||||
}
|
||||
|
||||
public getLLMRetryCount(tool: any): number {
|
||||
const count = Number(tool?.payload?.retry_count);
|
||||
return Number.isFinite(count) && count > 0 ? count : this.getLLMRetryEntries(tool).length;
|
||||
}
|
||||
|
||||
public getLLMRetryTotalDelay(tool: any): string | null {
|
||||
const configuredTotal = Number(tool?.payload?.total_delay);
|
||||
const total = Number.isFinite(configuredTotal)
|
||||
? configuredTotal
|
||||
: this.getLLMRetryEntries(tool).reduce(
|
||||
(sum: number, retry: any) => sum + (Number(retry?.delay) || 0),
|
||||
0
|
||||
);
|
||||
if (total <= 0) return null;
|
||||
return `${total.toFixed(2).replace(/\.00$/, '')}s`;
|
||||
}
|
||||
|
||||
public getLLMRetryEntryError(entry: any): string {
|
||||
return this.cleanErrorMessage(entry?.error || 'Unknown error');
|
||||
}
|
||||
|
||||
public getLLMRetryEntryDelay(entry: any): string | null {
|
||||
const delay = Number(entry?.delay);
|
||||
if (!Number.isFinite(delay) || delay <= 0) return null;
|
||||
return `${delay.toFixed(2).replace(/\.00$/, '')}s`;
|
||||
}
|
||||
|
||||
public getLLMRetryProvider(tool: any): string | null {
|
||||
const providers = Array.isArray(tool?.payload?.providers)
|
||||
? tool.payload.providers
|
||||
: [tool?.payload?.provider];
|
||||
const labels = providers
|
||||
.filter((provider: any) => typeof provider === 'string' && provider.trim())
|
||||
.map((provider: string) => provider.trim().replace(/^./, (value: string) => value.toUpperCase()));
|
||||
return labels.length > 0 ? Array.from(new Set(labels)).join(', ') : null;
|
||||
}
|
||||
|
||||
public cleanErrorMessage(rawError: any): string {
|
||||
return cleanErrorMessage(rawError);
|
||||
}
|
||||
|
||||
@@ -518,9 +518,12 @@ export class AgentService {
|
||||
|
||||
if (eventType === 'llm_retrying') {
|
||||
this.isRetrying.set(true);
|
||||
const attempt = parsedData.attempt || 1;
|
||||
const max = parsedData.max_retries || 3;
|
||||
this.retryMessage.set(`Reconnecting to AI model (Attempt ${attempt}/${max})...`);
|
||||
const attempt = Number(parsedData.attempt || 0);
|
||||
const max = Number(parsedData.max_retries || 0);
|
||||
const delay = Number(parsedData.delay || 0);
|
||||
const attemptText = attempt && max ? ` (Attempt ${attempt}/${max})` : '';
|
||||
const delayText = delay > 0 ? `; retrying in ${delay.toFixed(2).replace(/\.00$/, '')}s` : '';
|
||||
this.retryMessage.set(`AI service is temporarily busy${attemptText}${delayText}...`);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -162,8 +162,9 @@ export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
} else if (log.type === 'trace_recorded') {
|
||||
const isAction = log.data.type === 'action';
|
||||
const isTool = log.data.type === 'tool' || isAction;
|
||||
const isFailedLLM = log.data.type === 'llm_call' && log.data.status === 'failed';
|
||||
if (!isTool && !isFailedLLM) return;
|
||||
const isVisibleLLMEvent = log.data.type === 'llm_call'
|
||||
&& (log.data.status === 'failed' || log.data.status === 'retrying');
|
||||
if (!isTool && !isVisibleLLMEvent) return;
|
||||
|
||||
let stepId = log.data.step_id;
|
||||
let existingIndex = -1;
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import { getUniqueGenericTools } from './tool-formatter.util';
|
||||
|
||||
describe('getUniqueGenericTools retry aggregation', () => {
|
||||
it('groups recoverable LLM retries while preserving every attempt', () => {
|
||||
const tools = [
|
||||
{
|
||||
trace_id: 'retry-1',
|
||||
type: 'llm_call',
|
||||
name: 'llm_retry',
|
||||
status: 'retrying',
|
||||
timestamp: 100,
|
||||
payload: { error: '503 first', delay: 1.18, provider: 'google' }
|
||||
},
|
||||
{
|
||||
trace_id: 'retry-2',
|
||||
type: 'llm_call',
|
||||
name: 'llm_retry',
|
||||
status: 'retrying',
|
||||
timestamp: 101,
|
||||
payload: { error: '503 second', delay: 2.96, provider: 'google' }
|
||||
}
|
||||
];
|
||||
|
||||
const result = getUniqueGenericTools(tools);
|
||||
|
||||
expect(result.length).toBe(1);
|
||||
expect(result[0].name).toBe('llm_retry_group');
|
||||
expect(result[0].payload.retry_count).toBe(2);
|
||||
expect(result[0].payload.total_delay).toBeCloseTo(4.14);
|
||||
expect(result[0].payload.providers).toEqual(['google']);
|
||||
expect(result[0].payload.retries.map((retry: any) => retry.error)).toEqual([
|
||||
'503 first',
|
||||
'503 second'
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps a terminal LLM failure separate from its preceding retries', () => {
|
||||
const result = getUniqueGenericTools([
|
||||
{
|
||||
trace_id: 'retry-1',
|
||||
type: 'llm_call',
|
||||
name: 'llm_retry',
|
||||
status: 'retrying',
|
||||
payload: { error: '503', delay: 1 }
|
||||
},
|
||||
{
|
||||
trace_id: 'failure-1',
|
||||
type: 'llm_call',
|
||||
name: 'llm_pause',
|
||||
status: 'failed',
|
||||
payload: { error: '503 exhausted', pause: true }
|
||||
}
|
||||
]);
|
||||
|
||||
expect(result.map(tool => tool.name)).toEqual(['llm_retry_group', 'llm_pause']);
|
||||
});
|
||||
});
|
||||
@@ -185,6 +185,42 @@ export function getToolAgentName(tool: any): string | null {
|
||||
export function getUniqueGenericTools(tools: any[] | undefined): any[] {
|
||||
if (!tools) return [];
|
||||
|
||||
const retryTools = tools.filter(tool =>
|
||||
tool?.type === 'llm_call'
|
||||
&& tool?.status === 'retrying'
|
||||
&& (tool?.name === 'llm_retry' || tool?.name === 'llm_retry_group')
|
||||
);
|
||||
const firstRetry = retryTools[0];
|
||||
const latestRetry = retryTools[retryTools.length - 1];
|
||||
const retryAggregate = firstRetry ? {
|
||||
...firstRetry,
|
||||
trace_id: `llm-retry-group-${firstRetry.trace_id || firstRetry.timestamp || 'unknown'}`,
|
||||
name: 'llm_retry_group',
|
||||
payload: {
|
||||
...(latestRetry?.payload || {}),
|
||||
retry_count: retryTools.length,
|
||||
total_delay: retryTools.reduce(
|
||||
(total, retry) => total + (Number(retry?.payload?.delay) || 0),
|
||||
0
|
||||
),
|
||||
providers: Array.from(new Set(
|
||||
retryTools
|
||||
.map(retry => retry?.payload?.provider)
|
||||
.filter((provider): provider is string => typeof provider === 'string' && !!provider)
|
||||
)),
|
||||
retries: retryTools.map(retry => ({
|
||||
trace_id: retry.trace_id,
|
||||
timestamp: retry.timestamp,
|
||||
error: retry?.payload?.error || retry.error,
|
||||
delay: Number(retry?.payload?.delay) || 0,
|
||||
provider: retry?.payload?.provider,
|
||||
source: retry?.payload?.source,
|
||||
attempt: retry?.payload?.attempt,
|
||||
max_retries: retry?.payload?.max_retries
|
||||
}))
|
||||
}
|
||||
} : null;
|
||||
|
||||
const toolMap = new Map<string, any>();
|
||||
for (const t of tools) {
|
||||
if (t && t.trace_id) {
|
||||
@@ -196,11 +232,20 @@ export function getUniqueGenericTools(tools: any[] | undefined): any[] {
|
||||
|
||||
const result: any[] = [];
|
||||
const seenTraces = new Set<string>();
|
||||
let retryAggregateAdded = false;
|
||||
|
||||
for (const tool of tools) {
|
||||
if (!tool || !tool.name) continue;
|
||||
if (tool.type === 'agent') continue;
|
||||
|
||||
if (tool.type === 'llm_call' && tool.status === 'retrying') {
|
||||
if (!retryAggregateAdded && retryAggregate) {
|
||||
result.push(retryAggregate);
|
||||
retryAggregateAdded = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (tool.type === 'llm_call' && tool.status === 'failed') {
|
||||
result.push(tool);
|
||||
continue;
|
||||
|
||||
Reference in new issue
Block a user