runtime: harden LLM retry handling and operator workflows
This commit is contained in:
somew1nd committed
1 parent
0f5fb1d32c
commit
71e5bba551
18 files changed
+937
-348
No files matched your search
@@ -528,50 +528,36 @@
|
||||
|
||||
<!-- Generic Tools & LLM Calls (when item is a tool) -->
|
||||
<ng-container *ngIf="item.type === 'tool'">
|
||||
<!-- Recoverable provider/LLM retries remain visible after the task continues. -->
|
||||
<div class="step-android-action-card tool-action-card is-retrying"
|
||||
[class.is-expanded]="isActionExpanded('retry-' + block.id + '-' + itemIdx)"
|
||||
*ngIf="isLLMRetry(item.data)">
|
||||
<div class="action-card-header llm-retry-header"
|
||||
(click)="toggleActionExpanded('retry-' + block.id + '-' + itemIdx, $event)">
|
||||
<span class="material-symbols-outlined action-icon is-retrying">cloud_sync</span>
|
||||
<span class="action-title">AI service retry</span>
|
||||
<span class="action-status-badge is-retrying">
|
||||
{{ getLLMRetryCount(item.data) }}× retried
|
||||
</span>
|
||||
<span class="action-expand-icon material-symbols-outlined">
|
||||
{{ isActionExpanded('retry-' + block.id + '-' + itemIdx) ? 'expand_less' : 'expand_more' }}
|
||||
</span>
|
||||
</div>
|
||||
<div class="action-card-body">
|
||||
<div class="retry-summary-grid">
|
||||
<div class="retry-summary-item">
|
||||
<span class="retry-summary-label">Attempts</span>
|
||||
<strong>{{ getLLMRetryCount(item.data) }}</strong>
|
||||
</div>
|
||||
<div class="retry-summary-item" *ngIf="getLLMRetryTotalDelay(item.data) as totalDelay">
|
||||
<span class="retry-summary-label">Extra wait</span>
|
||||
<strong>{{ totalDelay }}</strong>
|
||||
</div>
|
||||
<div class="retry-summary-item" *ngIf="getLLMRetryProvider(item.data) as retryProvider">
|
||||
<span class="retry-summary-label">Provider</span>
|
||||
<strong>{{ retryProvider }}</strong>
|
||||
</div>
|
||||
</div>
|
||||
<div class="retry-attempt-list" *ngIf="isActionExpanded('retry-' + block.id + '-' + itemIdx)">
|
||||
<div class="retry-attempt-item" *ngFor="let retry of getLLMRetryEntries(item.data); let retryIndex = index">
|
||||
<div class="retry-attempt-heading">
|
||||
<span>Retry {{ retryIndex + 1 }}</span>
|
||||
<span class="retry-wait-pill" *ngIf="getLLMRetryEntryDelay(retry) as retryDelay">+{{ retryDelay }}</span>
|
||||
</div>
|
||||
<div class="retry-attempt-error">{{ getLLMRetryEntryError(retry) }}</div>
|
||||
</div>
|
||||
<!-- Only SDK-observed Gemini retries are shown as quiet stream notes. -->
|
||||
<div class="llm-retry-notes" *ngIf="isLLMRetry(item.data)">
|
||||
<div class="llm-retry-note" *ngFor="let retry of getLLMRetryEntries(item.data); let retryIndex = index">
|
||||
<span class="stream-status-dot"
|
||||
[class.streaming]="retryIndex === getLLMRetryEntries(item.data).length - 1 && isLLMRetryEntryWaiting(retry)"></span>
|
||||
<div class="llm-retry-note-text">
|
||||
<ng-container *ngIf="isLLMRetryEntryWaiting(retry)">
|
||||
<span class="llm-retry-status-line">
|
||||
LLM temporarily unavailable
|
||||
<ng-container *ngIf="getLLMRetryEntryDelay(retry) as retryDelay">
|
||||
· Retrying in {{ retryDelay }}…
|
||||
</ng-container>
|
||||
</span>
|
||||
<span class="llm-retry-guidance">
|
||||
Frequent retries may indicate a low API rate limit. Consider increasing the limit or reducing request concurrency.
|
||||
</span>
|
||||
</ng-container>
|
||||
<span class="llm-retry-status-line" *ngIf="!isLLMRetryEntryWaiting(retry)">
|
||||
Retrying LLM request · Waited {{ getLLMRetryEntryWaited(retry) }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Failed LLM Calls (Permanently maintain Action Card format in red) -->
|
||||
<div class="step-android-action-card tool-action-card is-failed" *ngIf="item.data.type === 'llm_call' && item.data.status === 'failed'">
|
||||
<div class="step-android-action-card tool-action-card is-failed"
|
||||
[class.has-details]="hasLLMFailureRetryEntries(item.data)"
|
||||
[class.is-expanded]="isActionExpanded('llm-failure-' + block.id + '-' + itemIdx)"
|
||||
(click)="hasLLMFailureRetryEntries(item.data) && toggleActionExpanded('llm-failure-' + block.id + '-' + itemIdx, $event)"
|
||||
*ngIf="isDisplayableLLMFailure(item.data)">
|
||||
<div class="action-card-header llm-failure-header">
|
||||
<span class="material-symbols-outlined action-icon is-failed">error</span>
|
||||
<span class="action-title">LLM Request Failed</span>
|
||||
@@ -596,12 +582,29 @@
|
||||
<span class="material-symbols-outlined">play_arrow</span>
|
||||
Resume
|
||||
</button>
|
||||
<span class="action-expand-icon material-symbols-outlined" *ngIf="hasLLMFailureRetryEntries(item.data)">
|
||||
{{ isActionExpanded('llm-failure-' + block.id + '-' + itemIdx) ? 'expand_less' : 'expand_more' }}
|
||||
</span>
|
||||
</div>
|
||||
<div class="action-card-body">
|
||||
<div class="action-detail-row">
|
||||
<span class="detail-label">Reason:</span>
|
||||
<span class="detail-value">{{ getLLMErrorText(item.data) }}</span>
|
||||
</div>
|
||||
<div class="action-detail-row" *ngIf="getLLMFailureWaited(item.data) as waited">
|
||||
<span class="detail-label">Waited:</span>
|
||||
<span class="detail-value">{{ waited }}</span>
|
||||
</div>
|
||||
<div class="failure-retry-details" *ngIf="isActionExpanded('llm-failure-' + block.id + '-' + itemIdx)">
|
||||
<div class="failure-retry-title">Retry details</div>
|
||||
<div class="failure-retry-row" *ngFor="let retry of getLLMFailureRetryEntries(item.data); let retryIndex = index">
|
||||
<span>Retry {{ retryIndex + 1 }}</span>
|
||||
<span>
|
||||
Planned {{ getLLMRetryEntryDelay(retry) || '0s' }}
|
||||
· Waited {{ getLLMRetryEntryWaited(retry) }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -1955,6 +1955,55 @@ $uniform-gap: 8px;
|
||||
margin-top: 0;
|
||||
margin-bottom: 0;
|
||||
|
||||
.llm-retry-notes {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: $uniform-gap;
|
||||
padding: 0;
|
||||
|
||||
.llm-retry-note {
|
||||
min-height: 20px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
color: #71717a;
|
||||
font-family: 'Google Sans', 'Google Sans Text', 'Inter', Arial, sans-serif;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
letter-spacing: 0.2px;
|
||||
|
||||
.stream-status-dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
background: #d4d4d8;
|
||||
flex: 0 0 auto;
|
||||
|
||||
&.streaming {
|
||||
background: #3b82f6;
|
||||
box-shadow: 0 0 0 0 rgba(59, 130, 246, 0.45);
|
||||
animation: pulse-stream-dot 1.8s cubic-bezier(0.4, 0, 0.6, 1) infinite;
|
||||
}
|
||||
}
|
||||
|
||||
.llm-retry-note-text {
|
||||
color: #71717a;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-width: 0;
|
||||
|
||||
.llm-retry-status-line,
|
||||
.llm-retry-guidance {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.llm-retry-guidance {
|
||||
color: #a1a1aa;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.step-generic-tool {
|
||||
border: 1px solid transparent;
|
||||
|
||||
@@ -2245,6 +2294,10 @@ $uniform-gap: 8px;
|
||||
background: #f8fafc;
|
||||
border-color: #fca5a5;
|
||||
|
||||
&.has-details {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
&:hover {
|
||||
border-color: #f87171;
|
||||
}
|
||||
@@ -2255,19 +2308,6 @@ $uniform-gap: 8px;
|
||||
}
|
||||
}
|
||||
|
||||
&.is-retrying {
|
||||
background: #fffbeb;
|
||||
border-color: #fcd34d;
|
||||
|
||||
&:hover {
|
||||
border-color: #f59e0b;
|
||||
}
|
||||
|
||||
.action-card-body .detail-value {
|
||||
color: #78350f;
|
||||
}
|
||||
}
|
||||
|
||||
&.task-report-card {
|
||||
background: #f0fdf4;
|
||||
border: 1px solid #bbf7d0;
|
||||
@@ -2373,9 +2413,6 @@ $uniform-gap: 8px;
|
||||
color: #ef4444;
|
||||
}
|
||||
|
||||
&.is-retrying {
|
||||
color: #d97706;
|
||||
}
|
||||
}
|
||||
|
||||
.action-title {
|
||||
@@ -2442,27 +2479,6 @@ $uniform-gap: 8px;
|
||||
}
|
||||
}
|
||||
|
||||
&.is-retrying {
|
||||
background: #fef3c7;
|
||||
color: #92400e;
|
||||
border: 1px solid #fde68a;
|
||||
|
||||
.status-icon {
|
||||
color: #d97706;
|
||||
|
||||
&.is-active {
|
||||
animation: spin 1.5s linear infinite;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
&.llm-retry-header {
|
||||
border-bottom-color: #fde68a;
|
||||
|
||||
.action-title {
|
||||
color: #78350f;
|
||||
}
|
||||
}
|
||||
|
||||
&.llm-failure-header {
|
||||
@@ -2528,83 +2544,6 @@ $uniform-gap: 8px;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
|
||||
.retry-summary-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.retry-summary-item {
|
||||
min-width: 0;
|
||||
padding: 7px 8px;
|
||||
border: 1px solid rgba(245, 158, 11, 0.18);
|
||||
border-radius: 7px;
|
||||
background: rgba(255, 255, 255, 0.68);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
|
||||
.retry-summary-label {
|
||||
color: #a16207;
|
||||
font-size: 9.5px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.02em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
strong {
|
||||
color: #78350f;
|
||||
font-size: 11.5px;
|
||||
font-weight: 700;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
}
|
||||
|
||||
.retry-attempt-list {
|
||||
border-top: 1px solid rgba(245, 158, 11, 0.22);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
margin-top: 2px;
|
||||
padding-top: 8px;
|
||||
animation: fadeInDetails 0.2s ease-out;
|
||||
|
||||
.retry-attempt-item {
|
||||
padding: 8px 9px;
|
||||
border: 1px solid rgba(245, 158, 11, 0.2);
|
||||
border-radius: 7px;
|
||||
background: rgba(255, 255, 255, 0.76);
|
||||
}
|
||||
|
||||
.retry-attempt-heading {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
margin-bottom: 4px;
|
||||
color: #92400e;
|
||||
font-size: 10px;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.retry-wait-pill {
|
||||
padding: 1px 5px;
|
||||
border-radius: 999px;
|
||||
background: #fef3c7;
|
||||
color: #b45309;
|
||||
}
|
||||
|
||||
.retry-attempt-error {
|
||||
color: #78350f;
|
||||
font-size: 10.5px;
|
||||
line-height: 1.45;
|
||||
overflow-wrap: anywhere;
|
||||
user-select: text;
|
||||
}
|
||||
}
|
||||
|
||||
.action-detail-row {
|
||||
display: flex;
|
||||
align-items: baseline;
|
||||
@@ -2654,6 +2593,38 @@ $uniform-gap: 8px;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.failure-retry-details {
|
||||
margin-top: 4px;
|
||||
padding-top: 9px;
|
||||
border-top: 1px solid #fecaca;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 7px;
|
||||
animation: fadeInDetails 0.2s ease-out;
|
||||
|
||||
.failure-retry-title {
|
||||
color: #991b1b;
|
||||
font-size: 10px;
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.03em;
|
||||
}
|
||||
|
||||
.failure-retry-row {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
color: #7f1d1d;
|
||||
font-size: 10.5px;
|
||||
line-height: 1.4;
|
||||
|
||||
span:last-child {
|
||||
color: #64748b;
|
||||
text-align: right;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.action-card-expanded-details {
|
||||
|
||||
@@ -158,6 +158,7 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
// Set to track expanded and collapsed state of action cards
|
||||
public expandedActionCards = signal<Set<string>>(new Set<string>());
|
||||
public collapsedActionCards = signal<Set<string>>(new Set<string>());
|
||||
public retryClock = signal<number>(Date.now());
|
||||
|
||||
// Performance caches for parameter and event extractions
|
||||
private actionParamsCache = new WeakMap<any, ActionParam[]>();
|
||||
@@ -280,6 +281,9 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
}
|
||||
|
||||
constructor() {
|
||||
const retryClockInterval = setInterval(() => this.retryClock.set(Date.now()), 1000);
|
||||
this.destroyRef.onDestroy(() => clearInterval(retryClockInterval));
|
||||
|
||||
// Auto scroll stream box during active text streaming
|
||||
effect(() => {
|
||||
const logs = this.filteredLogs();
|
||||
@@ -827,7 +831,12 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
if (!block) return false;
|
||||
const hasNative = Boolean(this.getNativeThinking(block));
|
||||
const hasRaw = Boolean(this.getRawThinking(block));
|
||||
const hasVisibleTools = Boolean(block.data?.generic_tools) && block.data.generic_tools.some((t: any) => this.shouldShowTool(t, block.data) || (t.type === 'llm_call' && (t.status === 'failed' || t.status === 'retrying')) || this.isReportStatusAction(t));
|
||||
const hasVisibleTools = Boolean(block.data?.generic_tools) && block.data.generic_tools.some((t: any) =>
|
||||
this.shouldShowTool(t, block.data)
|
||||
|| this.isDisplayableLLMFailure(t)
|
||||
|| this.isLLMRetry(t)
|
||||
|| this.isReportStatusAction(t)
|
||||
);
|
||||
const hasAndroidActions = Boolean(block.data?.action_taken) && (this.isAndroidAction(block.data.action_taken) || this.isReportStatusAction(block.data.action_taken));
|
||||
return hasNative || hasRaw || hasVisibleTools || hasAndroidActions;
|
||||
}
|
||||
@@ -852,6 +861,12 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
return !!tool && this.retryablePauseTrace() === tool;
|
||||
}
|
||||
|
||||
public isDisplayableLLMFailure(tool: any): boolean {
|
||||
return tool?.type === 'llm_call'
|
||||
&& tool?.status === 'failed'
|
||||
&& (tool?.name === 'llm_pause' || tool?.payload?.pause === true);
|
||||
}
|
||||
|
||||
public resumePausedTask(event: Event): void {
|
||||
event.stopPropagation();
|
||||
this.agentService.resumeTask();
|
||||
@@ -865,7 +880,10 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
}
|
||||
|
||||
public isLLMRetry(tool: any): boolean {
|
||||
return tool?.type === 'llm_call' && tool?.status === 'retrying';
|
||||
return tool?.type === 'llm_call'
|
||||
&& tool?.status === 'retrying'
|
||||
&& tool?.payload?.source === 'provider_sdk'
|
||||
&& ['google', 'gemini'].includes(String(tool?.payload?.provider || '').toLowerCase());
|
||||
}
|
||||
|
||||
public getLLMRetryEntries(tool: any): any[] {
|
||||
@@ -873,25 +891,17 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
return Array.isArray(retries) && retries.length > 0 ? retries : [tool?.payload || tool];
|
||||
}
|
||||
|
||||
public getLLMRetryCount(tool: any): number {
|
||||
const count = Number(tool?.payload?.retry_count);
|
||||
return Number.isFinite(count) && count > 0 ? count : this.getLLMRetryEntries(tool).length;
|
||||
public getLLMFailureRetryEntries(tool: any): any[] {
|
||||
const retries = tool?.payload?.retries;
|
||||
if (!Array.isArray(retries)) return [];
|
||||
return retries.filter((retry: any) =>
|
||||
retry?.source === 'provider_sdk'
|
||||
&& ['google', 'gemini'].includes(String(retry?.provider || '').toLowerCase())
|
||||
);
|
||||
}
|
||||
|
||||
public getLLMRetryTotalDelay(tool: any): string | null {
|
||||
const configuredTotal = Number(tool?.payload?.total_delay);
|
||||
const total = Number.isFinite(configuredTotal)
|
||||
? configuredTotal
|
||||
: this.getLLMRetryEntries(tool).reduce(
|
||||
(sum: number, retry: any) => sum + (Number(retry?.delay) || 0),
|
||||
0
|
||||
);
|
||||
if (total <= 0) return null;
|
||||
return `${total.toFixed(2).replace(/\.00$/, '')}s`;
|
||||
}
|
||||
|
||||
public getLLMRetryEntryError(entry: any): string {
|
||||
return this.cleanErrorMessage(entry?.error || 'Unknown error');
|
||||
public hasLLMFailureRetryEntries(tool: any): boolean {
|
||||
return this.getLLMFailureRetryEntries(tool).length > 0;
|
||||
}
|
||||
|
||||
public getLLMRetryEntryDelay(entry: any): string | null {
|
||||
@@ -900,14 +910,37 @@ export class AgentStreamComponent implements AfterViewInit {
|
||||
return `${delay.toFixed(2).replace(/\.00$/, '')}s`;
|
||||
}
|
||||
|
||||
public getLLMRetryProvider(tool: any): string | null {
|
||||
const providers = Array.isArray(tool?.payload?.providers)
|
||||
? tool.payload.providers
|
||||
: [tool?.payload?.provider];
|
||||
const labels = providers
|
||||
.filter((provider: any) => typeof provider === 'string' && provider.trim())
|
||||
.map((provider: string) => provider.trim().replace(/^./, (value: string) => value.toUpperCase()));
|
||||
return labels.length > 0 ? Array.from(new Set(labels)).join(', ') : null;
|
||||
public getLLMRetryEntryWaited(entry: any): string {
|
||||
const delay = Number(entry?.delay);
|
||||
if (!Number.isFinite(delay) || delay <= 0) return '0s';
|
||||
|
||||
const rawStartedAt = Number(entry?.scheduled_at ?? entry?.timestamp);
|
||||
if (!Number.isFinite(rawStartedAt) || rawStartedAt <= 0) {
|
||||
return this.formatLLMDuration(delay);
|
||||
}
|
||||
const startedAtMs = rawStartedAt < 1e11 ? rawStartedAt * 1000 : rawStartedAt;
|
||||
const elapsed = Math.min(delay, Math.max(0, (this.retryClock() - startedAtMs) / 1000));
|
||||
return this.formatLLMDuration(elapsed);
|
||||
}
|
||||
|
||||
public isLLMRetryEntryWaiting(entry: any): boolean {
|
||||
const delay = Number(entry?.delay);
|
||||
const rawStartedAt = Number(entry?.scheduled_at ?? entry?.timestamp);
|
||||
if (!Number.isFinite(delay) || delay <= 0 || !Number.isFinite(rawStartedAt)) return false;
|
||||
const startedAtMs = rawStartedAt < 1e11 ? rawStartedAt * 1000 : rawStartedAt;
|
||||
return this.retryClock() < startedAtMs + delay * 1000;
|
||||
}
|
||||
|
||||
public getLLMFailureWaited(tool: any): string | null {
|
||||
const waited = Number(tool?.payload?.waited_seconds);
|
||||
if (!Number.isFinite(waited) || waited < 0) return null;
|
||||
return this.formatLLMDuration(waited);
|
||||
}
|
||||
|
||||
public formatLLMDuration(seconds: number): string {
|
||||
if (seconds < 0.05) return '0s';
|
||||
if (seconds < 10) return `${seconds.toFixed(1).replace(/\.0$/, '')}s`;
|
||||
return `${Math.round(seconds)}s`;
|
||||
}
|
||||
|
||||
public cleanErrorMessage(rawError: any): string {
|
||||
|
||||
Reference in new issue
Block a user