runtime: harden LLM retry handling and operator workflows

This commit is contained in:
somew1nd committed 2026-08-21 00:34:44 -07:00
1 parent 0f5fb1d32c
commit 71e5bba551
18 files changed
+937 -348

No files matched your search

@@ -9,7 +9,7 @@ describe('getUniqueGenericTools retry aggregation', () => {
name: 'llm_retry',
status: 'retrying',
timestamp: 100,
payload: { error: '503 first', delay: 1.18, provider: 'google' }
payload: { error: '503 first', delay: 1.18, provider: 'google', source: 'provider_sdk', request_id: 'request-1' }
},
{
trace_id: 'retry-2',
@@ -17,7 +17,7 @@ describe('getUniqueGenericTools retry aggregation', () => {
name: 'llm_retry',
status: 'retrying',
timestamp: 101,
payload: { error: '503 second', delay: 2.96, provider: 'google' }
payload: { error: '503 second', delay: 2.96, provider: 'google', source: 'provider_sdk', request_id: 'request-1' }
}
];
@@ -27,31 +27,51 @@ describe('getUniqueGenericTools retry aggregation', () => {
expect(result[0].name).toBe('llm_retry_group');
expect(result[0].payload.retry_count).toBe(2);
expect(result[0].payload.total_delay).toBeCloseTo(4.14);
expect(result[0].payload.providers).toEqual(['google']);
expect(result[0].payload.retries.map((retry: any) => retry.error)).toEqual([
'503 first',
'503 second'
]);
});
it('keeps a terminal LLM failure separate from its preceding retries', () => {
it('folds SDK retry rows into their matching terminal failure', () => {
const result = getUniqueGenericTools([
{
trace_id: 'retry-1',
type: 'llm_call',
name: 'llm_retry',
status: 'retrying',
payload: { error: '503', delay: 1 }
payload: { error: '503', delay: 1, provider: 'google', source: 'provider_sdk', request_id: 'request-1' }
},
{
trace_id: 'failure-1',
type: 'llm_call',
name: 'llm_pause',
status: 'failed',
payload: { error: '503 exhausted', pause: true }
payload: { error: '503 exhausted', pause: true, request_id: 'request-1', retries: [{ delay: 1 }] }
}
]);
expect(result.map(tool => tool.name)).toEqual(['llm_retry_group', 'llm_pause']);
expect(result.map(tool => tool.name)).toEqual(['llm_pause']);
});
it('does not expose opaque retries from non-Gemini providers', () => {
const result = getUniqueGenericTools([
{
trace_id: 'retry-openai',
type: 'llm_call',
name: 'llm_retry',
status: 'retrying',
payload: { error: '429', delay: 2, provider: 'openai', source: 'artemis_wrapper' }
},
{
trace_id: 'failure-openai',
type: 'llm_call',
name: 'llm_pause',
status: 'failed',
payload: { error: '429 exhausted', pause: true, waited_seconds: 12 }
}
]);
expect(result.map(tool => tool.name)).toEqual(['llm_pause']);
});
});