/** * Copyright 2026 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import { ActionParam } from '../core/models/stream.model'; import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util'; import { formatNativeActionParam, getNativeActionArgs, getNativeActionTitle } from './native-action-labels.util'; // Tool objects are replaced (not mutated) when a trace is updated, so a // WeakMap keyed on the tool is a safe memo for the parsed-args lookup that // templates would otherwise re-run (including JSON.parse) on every render. const toolArgsCache = new WeakMap(); /** * Extract arguments from tool payload or direct args field */ export function getToolArgs(tool: any): any { if (!tool) return {}; if (typeof tool === 'object') { const cached = toolArgsCache.get(tool); if (cached !== undefined) return cached; const args = computeToolArgs(tool); toolArgsCache.set(tool, args); return args; } return computeToolArgs(tool); } function computeToolArgs(tool: any): any { if (tool.payload) { let payloadObj = tool.payload; if (typeof payloadObj === 'string') { try { payloadObj = JSON.parse(payloadObj); } catch {} } if (payloadObj && typeof payloadObj === 'object') { if (payloadObj.args && typeof payloadObj.args === 'object') { return payloadObj.args; } return payloadObj; } } if (tool.args) { let argsObj = tool.args; if (typeof argsObj === 'string') { try { argsObj = JSON.parse(argsObj); } catch {} } if (argsObj && typeof argsObj === 'object') { return argsObj; } } return {}; } /** * Check if a tool is one of the five note-related tools */ export function isNoteTool(tool: any): boolean { if (!tool || !tool.name) return false; const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, ''); const nameLower = cleanName.toLowerCase(); return ['save_note', 'read_note', 'list_notes', 'update_note', 'append_note'].includes(nameLower); } /** * Check if a tool is video analysis related */ export function isVideoTool(tool: any): boolean { if (!tool || !tool.name) return false; const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, ''); const nameLower = cleanName.toLowerCase(); return [ 'video_analysis', 'video_analyzer', 'video_analyzer_pure', 'spawn_sub_agent', 'analyze_audio_only' ].includes(nameLower); } export type VideoAnalysisOutcome = 'running' | 'recovering' | 'waiting' | 'complete' | 'partial' | 'failed'; export interface VideoAnalysisRange { start: number; end: number; category?: string; retryable?: boolean; } export interface VideoAnalysisView { outcome: VideoAnalysisOutcome; title: string; query: string; summary: string; reuse: 'none' | 'partial' | 'full'; requestedRange: VideoAnalysisRange | null; completedRanges: VideoAnalysisRange[]; failedRanges: VideoAnalysisRange[]; evidenceCount: number; completedCount: number; totalCount: number; fallbackUsed: boolean; } function numericRange(value: any): VideoAnalysisRange | null { if (!value || typeof value !== 'object') return null; const start = Number(value.start); const end = Number(value.end); if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start) return null; return { start, end, category: value.category ? String(value.category) : undefined, retryable: value.retryable === true }; } function parseVideoResultText(text: string): Partial { const value = text.trim(); if (!value) return {}; if (value.startsWith('CACHED VIDEO ANALYSIS:')) { return { outcome: 'complete', reuse: 'full', summary: value.replace(/^CACHED VIDEO ANALYSIS:\s*/, '') }; } if (value.startsWith('PARTIAL VIDEO ANALYSIS')) { return { outcome: 'partial', reuse: 'partial', summary: value }; } if (value.startsWith('All sub-agent chunks failed') || value.startsWith('Error:')) { return { outcome: 'failed', summary: value }; } if (value.includes('Analysis is already in progress in another video agent')) { return { outcome: 'waiting', summary: "另一个视频智能体正在分析此段内容。" }; } return { outcome: 'complete', summary: value }; } /** Build the user-facing, backward-compatible video analysis state. */ export function getVideoAnalysisView(tool: any): VideoAnalysisView | null { if (!isVideoTool(tool)) return null; const args = getToolArgs(tool); const payload = tool?.payload && typeof tool.payload === 'object' ? tool.payload : {}; const rawResult = payload.result ?? tool.result ?? null; const structured = rawResult && typeof rawResult === 'object' ? rawResult : {}; const parsed = typeof rawResult === 'string' ? parseVideoResultText(rawResult) : {}; let outcome = String(structured.outcome || payload.outcome || parsed.outcome || '').toLowerCase() as VideoAnalysisOutcome; if (!['running', 'recovering', 'waiting', 'complete', 'partial', 'failed'].includes(outcome)) { outcome = tool?.status === 'failed' || tool?.status === 'error' ? 'failed' : tool?.status === 'running' ? 'running' : 'complete'; } const recovering = structured.recovering === true || structured.fallback_used === true; if (outcome === 'running' && recovering) outcome = 'recovering'; const start = Number(args.start_time ?? structured.requested_range?.start); const end = Number(args.end_time ?? structured.requested_range?.end); const requestedRange = Number.isFinite(start) && Number.isFinite(end) && end > start ? { start, end } : numericRange(structured.requested_range); const completedRanges = Array.isArray(structured.completed_ranges) ? structured.completed_ranges.map(numericRange).filter((range: VideoAnalysisRange | null): range is VideoAnalysisRange => Boolean(range)) : []; const failedRanges = Array.isArray(structured.failed_ranges) ? structured.failed_ranges.map(numericRange).filter((range: VideoAnalysisRange | null): range is VideoAnalysisRange => Boolean(range)) : []; const completedCount = Number(structured.completed_count ?? completedRanges.length ?? 0); const totalCount = Number(structured.total_count ?? (completedRanges.length + failedRanges.length)); const titleByOutcome: Record = { running: "正在分析屏幕录像", recovering: "正在分析尚未结束的录像片段", waiting: "正在等待已有的视频分析", complete: structured.reuse === 'full' || parsed.reuse === 'full' ? "已复用视频分析" : "已分析屏幕录像", partial: "视频分析部分完成", failed: "视频分析未返回结果" }; return { outcome, title: titleByOutcome[outcome], query: String(args.specific_query || args.query || args.prompt || structured.query || ''), summary: String(structured.summary || parsed.summary || ''), reuse: (structured.reuse || parsed.reuse || 'none') as 'none' | 'partial' | 'full', requestedRange, completedRanges, failedRanges, evidenceCount: Number(structured.evidence_count || 0), completedCount: Number.isFinite(completedCount) ? completedCount : 0, totalCount: Number.isFinite(totalCount) ? totalCount : 0, fallbackUsed: structured.fallback_used === true }; } export function formatVideoTime(seconds: number): string { const safe = Math.max(0, Number(seconds) || 0); const minutes = Math.floor(safe / 60); const secs = Math.floor(safe % 60); return `${String(minutes).padStart(2, '0')}:${String(secs).padStart(2, '0')}`; } /** * Extract target video filename or description for video analysis tools */ export function getVideoToolTarget(tool: any): string | null { if (!tool) return null; const args = getToolArgs(tool); if (args.video_path || args.video_file || args.file_path || args.file) { const p = String(args.video_path || args.video_file || args.file_path || args.file); return p.split('/').pop() || p; } if (args.time_description) { return String(args.time_description); } if (args.start_time !== undefined || args.end_time !== undefined) { const start = args.start_time ?? 0; const end = args.end_time ? `${args.end_time}s` : 'end'; return `${start}s - ${end}`; } if (args.purpose) { return String(args.purpose); } return 'screen_record.mp4'; } /** * Canonical list of mobile device automation and ADB command tool names. */ export const DEVICE_ACTION_TOOL_NAMES = new Set([ 'click', 'click_sequence', 'long_press', 'long_press_on', 'input', 'input_text', 'swipe', 'scroll', 'press_key', 'press_home', 'press_back', 'launch_app', 'manage_app', 'open_app', 'focus_and_input_text', 'focus_and_clear_text', 'tap', 'wait_for_delay', 'wait', 'run_adb_command', 'run_short_adb_command' ]); /** * Check if a tool represents a real mobile device action or ADB command. */ export function isDeviceActionTool(tool: any): boolean { if (!tool || !tool.name) return false; if (tool.type === 'llm_call') return false; const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '').toLowerCase(); return DEVICE_ACTION_TOOL_NAMES.has(cleanName); } /** * Backward-compatible alias for isDeviceActionTool. */ export function isFailureAnalyzerActionTool(tool: any): boolean { return isDeviceActionTool(tool); } /** * Framework-internal infrastructure tools that are not intended for end-user timeline display. */ export function isInternalPlumbingTool(tool: any): boolean { if (!tool || !tool.name) return true; const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '').toLowerCase(); if (cleanName.includes('safety_net')) return true; if (cleanName === 'report_task_status' || cleanName === 'report_status' || cleanName === 'submit_task_status') { return true; } return false; } /** * Determines whether a tool invocation should be displayed in the timeline. */ export function shouldShowTool(tool: any, _stepData?: any): boolean { if (!tool || !tool.name) return false; if (tool.type === 'llm_call') return false; if (tool.type === 'agent') return false; if (tool.type === 'log') return false; // system notes render as quiet rows, not tool cards if (isInternalPlumbingTool(tool)) return false; return true; } /** * Extracts human-readable initiating agent name / role for a tool. */ export function getToolAgentName(tool: any): string | null { if (!tool) return null; const name = (tool.agent_name || tool.agent || '').toLowerCase(); if (name.includes('failure') || name.includes('analyzer')) { return null; // Omit self-healing label per user instruction } if (name.includes('outputter')) { return 'Outputter'; } if (name.includes('validator')) { return 'Validator'; } if (name.includes('diagnos')) { return 'Diagnoser'; } if (name.includes('explorer')) { return 'Explorer'; } return null; } /** * Filter out nested child wrapper duplicates while preserving independent executions at the same level */ export function getUniqueGenericTools(tools: any[] | undefined): any[] { if (!tools) return []; // Only Google GenAI SDK retries are observable. Artemis wrapper retries and // opaque provider retries must never be presented as provider internals. const retryTools = tools.filter(tool => tool?.type === 'llm_call' && tool?.status === 'retrying' && (tool?.name === 'llm_retry' || tool?.name === 'llm_retry_group') && tool?.payload?.source === 'provider_sdk' && ['google', 'gemini'].includes(String(tool?.payload?.provider || '').toLowerCase()) ); const retryKey = (tool: any) => String( tool?.payload?.request_id || `legacy-${tool?.step_id || 'unassigned'}` ); const retryGroups = new Map(); for (const retry of retryTools) { const key = retryKey(retry); retryGroups.set(key, [...(retryGroups.get(key) || []), retry]); } const terminalRequestIds = new Set( tools .filter(tool => tool?.type === 'llm_call' && tool?.status === 'failed') .map(tool => tool?.payload?.request_id) .filter((requestId): requestId is string => typeof requestId === 'string' && !!requestId) ); const buildRetryAggregate = (group: any[]) => { const firstRetry = group[0]; const latestRetry = group[group.length - 1]; return { ...firstRetry, trace_id: `llm-retry-group-${firstRetry?.payload?.request_id || firstRetry?.trace_id || firstRetry?.timestamp || 'unknown'}`, name: 'llm_retry_group', payload: { ...(latestRetry?.payload || {}), retry_count: group.length, total_delay: group.reduce( (total, retry) => total + (Number(retry?.payload?.delay) || 0), 0 ), retries: group.map(retry => ({ trace_id: retry.trace_id, timestamp: retry.timestamp, error: retry?.payload?.error || retry.error, delay: Number(retry?.payload?.delay) || 0, provider: retry?.payload?.provider, source: retry?.payload?.source, request_id: retry?.payload?.request_id, scheduled_at: retry?.payload?.scheduled_at })) } }; }; const toolMap = new Map(); for (const t of tools) { if (t && t.trace_id) { toolMap.set(t.trace_id, t); } } const cleanName = (nameStr: string) => (nameStr || '').replace(/^(_)?(self\.)?exec_/, '').toLowerCase().trim(); const result: any[] = []; const seenTraces = new Set(); const addedRetryGroups = new Set(); for (const tool of tools) { if (!tool || !tool.name) continue; if (tool.type === 'agent') continue; if (tool.type === 'llm_call' && tool.status === 'retrying') { const groupKey = retryKey(tool); const group = retryGroups.get(groupKey); const requestId = tool?.payload?.request_id; if ( group && !addedRetryGroups.has(groupKey) && !(requestId && terminalRequestIds.has(requestId)) ) { result.push(buildRetryAggregate(group)); addedRetryGroups.add(groupKey); } continue; } if (tool.type === 'llm_call' && tool.status === 'failed') { // Callback-level attempt errors are implementation details. The wrapper // emits one terminal llm_pause trace after retries are exhausted. if (tool.name === 'llm_pause' || tool?.payload?.pause === true) { result.push(tool); } continue; } const currClean = cleanName(tool.name); // Check if this tool is a nested child execution of a parent tool with the exact same tool name let isNestedChildDuplicate = false; if (tool.parent_trace_id) { const parentTool = toolMap.get(tool.parent_trace_id); if (parentTool && cleanName(parentTool.name) === currClean) { isNestedChildDuplicate = true; } } if (!isNestedChildDuplicate) { if (tool.trace_id) { if (!seenTraces.has(tool.trace_id)) { seenTraces.add(tool.trace_id); // Always use the latest state from toolMap for this trace_id result.push(toolMap.get(tool.trace_id) || tool); } } else { result.push(tool); } } } return collapseVideoAnalysisTools(result); } /** * Present one note-style timeline row for one video-analyzer execution. Child * chunk, audio, and wrapper traces stay available in the raw trace but do not * look like repeated user-visible analyses. */ function collapseVideoAnalysisTools(tools: any[]): any[] { const groups = new Map(); const passthrough: Array<{ index: number; tool: any }> = []; tools.forEach((tool, index) => { if (!isVideoTool(tool)) { passthrough.push({ index, tool }); return; } const cleanName = String(tool.name || '').replace(/^(_)?(self\.)?exec_/, '').toLowerCase(); const groupId = cleanName === 'video_analyzer' || cleanName === 'video_analyzer_pure' ? String(tool.trace_id || tool.parent_trace_id || `video-${index}`) : String(tool.parent_trace_id || tool.trace_id || `video-${index}`); const existing = groups.get(groupId); if (existing) { existing.tools.push(tool); } else { groups.set(groupId, { firstIndex: index, tools: [tool] }); } }); const collapsed = [...passthrough]; for (const [groupId, group] of groups) { const views = group.tools .map(getVideoAnalysisView) .filter((view): view is VideoAnalysisView => Boolean(view)); if (!views.length) continue; const hasActive = views.some(view => view.outcome === 'running' || view.outcome === 'recovering'); const completed = views.filter(view => view.outcome === 'complete'); const hasPartial = views.some(view => view.outcome === 'partial'); const failed = views.filter(view => view.outcome === 'failed'); const waiting = views.filter(view => view.outcome === 'waiting'); let outcome: VideoAnalysisOutcome; if (hasActive) { outcome = views.some(view => view.outcome === 'recovering') ? 'recovering' : 'running'; } else if (hasPartial || (completed.length > 0 && failed.length > 0)) { outcome = 'partial'; } else if (completed.length > 0) { outcome = 'complete'; } else if (waiting.length > 0) { outcome = 'waiting'; } else { outcome = 'failed'; } const ranges = views .map(view => view.requestedRange) .filter((range): range is VideoAnalysisRange => Boolean(range)); const requestedRange = ranges.length ? { start: Math.min(...ranges.map(range => range.start)), end: Math.max(...ranges.map(range => range.end)) } : null; const reuse = completed.length > 0 && completed.every(view => view.reuse === 'full') ? 'full' : views.some(view => view.reuse !== 'none') ? 'partial' : 'none'; const base = group.tools[0]; const structuredResult = { outcome, reuse, requested_range: requestedRange, completed_count: completed.length, total_count: views.length, evidence_count: views.reduce((sum, view) => sum + view.evidenceCount, 0), query: views.find(view => view.query)?.query || '', recovering: outcome === 'recovering', fallback_used: views.some(view => view.fallbackUsed) }; collapsed.push({ index: group.firstIndex, tool: { ...base, trace_id: `video-analysis-${groupId}`, name: 'video_analysis', status: outcome === 'running' || outcome === 'recovering' ? 'running' : 'success', payload: { ...(base.payload || {}), args: { ...(base.payload?.args || base.args || {}), start_time: requestedRange?.start, end_time: requestedRange?.end }, result: structuredResult } } }); } collapsed.sort((a, b) => a.index - b.index); return collapsed.map(item => item.tool); } /** * Get note key for the tool if it exists */ export function getToolKey(tool: any): string | null { if (!tool || !tool.name) return null; const args = tool.payload?.args || tool.args; const key = args?.key; if (!key) return null; return key.toLowerCase().endsWith('.md') ? key : `${key}.md`; } /** * Get display label for tools (e.g. for pills or headers) */ export function getToolDisplayLabel(tool: any, isFirstSaveNote: boolean = false): string { if (!tool || !tool.name) return ''; const cleanName = tool.name.replace(/^(_)?exec_/, ''); const nameLower = cleanName.toLowerCase(); const args = getToolArgs(tool); const nativeTitle = getNativeActionTitle(tool.name, args); if (nativeTitle) return nativeTitle; switch (nameLower) { case 'manage_app': case 'launch_app': { const rawApp = args.app_name || args.package_name || args.app || ''; const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : "应用"; const rawAction = args.action ? String(args.action).toLowerCase() : ''; const verb = rawAction === 'launch' ? "启动中" : (rawAction === 'stop' || rawAction === 'close' ? "停止中" : "管理中"); return `${verb} "${app}"`; } case 'wait_for_delay': case 'wait_delay': case 'wait': { const delay = args.delay_seconds || args.seconds || args.delay || args.duration; return delay ? `等待 ${delay} second${Number(delay) > 1 ? 's' : ''}...` : "正在等待…"; } case 'wait_for_text': { const text = args.text || args.target_text || ''; return text ? `等待文本 "${text}" 出现在屏幕上` : "正在等待屏幕出现文本"; } case 'input_text': case 'input': { const text = args.text || args.input_text || ''; return text ? `输入文本 "${text}" 至输入框` : "正在向输入框输入文本"; } case 'focus_and_input_text': { const text = args.text || args.input_text || ''; return text ? `聚焦输入框并输入 "${text}"` : "正在聚焦输入框并输入文本"; } case 'focus_and_clear_text': return "正在聚焦输入框并清空文本"; case 'click': case 'tap': { const target = args.target_text || args.text || args.query || ''; return target ? `点击 "${target}"` : "正在点击屏幕元素"; } case 'click_sequence': return "正在执行连续点击"; case 'long_press': { const target = args.target_text || args.text || ''; return target ? `长按 "${target}"` : "正在长按屏幕元素"; } case 'swipe': { const dir = args.action || args.direction || ''; return dir ? `滑动 ${String(dir).toUpperCase()} 在屏幕上` : "正在滑动屏幕"; } case 'press_key': { const key = args.key || args.keycode || ''; return key ? `按键 ${String(key).toUpperCase()}` : "正在按下实体按键"; } case 'save_note': return isFirstSaveNote ? "正在创建笔记" : "正在保存笔记"; case 'read_note': return "正在读取笔记"; case 'list_notes': return "正在浏览已保存笔记"; case 'update_note': return "正在更新笔记"; case 'append_note': return "正在更新笔记"; case 'object_detection': { const q = Array.isArray(args.queries) ? args.queries.join(', ') : (args.queries || ''); return q ? `Locating 在屏幕上: "${q}"` : "正在定位屏幕元素"; } case 'ask_explorer': { const query = args.query || args.prompt || ''; return query ? `Searching 在屏幕上: "${query}"` : "正在搜索屏幕"; } case 'report_failure_analysis': { const reason = args.reason || args.analysis || ''; return reason ? `排查问题:${reason}` : "正在排查执行问题"; } case 'run_adb_command': case 'run_short_adb_command': { const cmd = args.command || args.cmd || ''; return cmd ? `运行命令:${cmd}` : "正在运行系统命令"; } case 'search_logs': case 'read_logs': { const q = args.query || args.filter || ''; return q ? `搜索日志:"${q}"` : "正在分析系统日志"; } case 'log_analyzer': case 'output_analyzer': return "正在分析日志"; case 'diagnoser': case 'diagnose': return "正在诊断问题"; case 'video_analyzer': case 'video_analyzer_pure': return "正在分析屏幕录像"; case 'extract_segment_metadata': { const start = args.start_time !== undefined ? `${args.start_time}s` : ''; const end = args.end_time !== undefined ? `${args.end_time}s` : ''; const range = (start && end) ? ` (${start} - ${end})` : (start ? ` (from ${start})` : ''); return `截取录屏片段${range}`; } case 'spawn_sub_agent': { const q = args.specific_query || args.query || args.prompt || ''; return q ? `分析录屏:"${q}"` : "正在调用子智能体分析录像"; } case 'analyze_audio_only': { const q = args.specific_query || args.query || ''; return q ? `Analyzing audio track: "${q}"` : "正在分析录像音轨"; } case 'search_history': { const q = args.query || ''; const range = Array.isArray(args.step_range) && args.step_range.length ? ` in steps ${args.step_range[0]}–${args.step_range[args.step_range.length - 1]}` : ''; return q ? `Searching execution history for "${q}"${range}` : `Searching execution history${range}`; } case 'replay_steps': { const n = args.start_step; const end = args.end_step; if (n !== undefined && n !== '' && end !== undefined && end !== null && end !== '' && String(end) !== String(n)) { return `Reviewing steps ${n}–${end}`; } return n !== undefined && n !== '' ? `Reviewing step ${n}` : "正在查看步骤详情"; } case 'get_step_screenshot': { const n = args.step_number; const variant = String(args.which || '').toLowerCase(); if (variant === 'overlay') { return n !== undefined && n !== '' ? `Looking at where step ${n}'s action landed` : "正在查看操作位置"; } const which = variant === 'post' ? 'after' : 'before'; return n !== undefined && n !== '' ? `Looking at the screen ${which} step ${n}` : "正在查看步骤截图"; } case 'probe_device': { const kind = args.kind ? String(args.kind).replace(/_/g, ' ') : ''; return kind ? `Reading ${kind} from the device` : "正在读取设备状态"; } case 'outputter': case 'output_synthesis': return "正在生成输出报告"; case 'web_search': { const q = args.query || ''; return q ? `Searching web for "${q}"` : "正在搜索网页"; } case 'read_url': return "正在读取网页"; case 'compress_history': return getCompressionLabel(tool); default: return `Executing ${cleanName.split('_').map((w: string) => w.charAt(0).toUpperCase() + w.slice(1)).join(' ')}`; } } function formatTokenFigure(value: any): string { const n = Number(value); if (!Number.isFinite(n) || n < 0) return ''; if (n < 1000) return `${Math.round(n)}`; return `${(n / 1000).toFixed(1).replace(/\.0$/, '')}k`; } /** Format compression progress and token usage for the timeline. */ export function getCompressionLabel(tool: any): string { const args = getToolArgs(tool) || {}; const start = Number(args.start_step); const end = Number(args.end_step); const hasRange = Number.isFinite(start) && Number.isFinite(end) && start > 0 && end > 0; const range = hasRange ? (start === end ? `step ${start}` : `steps ${start}–${end}`) : "之前的步骤"; const rangeCapitalized = range.charAt(0).toUpperCase() + range.slice(1); const status = String(tool?.status || '').toLowerCase(); const phase = getCompressionPhase(tool); if (status === 'failed' || phase === 'failed') { return `Couldn't condense ${range} yet; keeping the full record and retrying later`; } if (status !== 'success') { const note = String(args.note || '').toLowerCase(); if (note === 'retrying') return `Retrying the memory summary for ${range}…`; if (note === 'held' || phase === 'ready') { const swapAt = Number(args.swap_at_tokens); const heldContext = Number(args.context_tokens); const heldParts = [`Short memory for ${range} is ready; keeping the full record until working memory fills up`]; if (heldContext > 0 && swapAt > 0) { heldParts.push(`≈ ${formatTokenFigure(heldContext)} of ${formatTokenFigure(swapAt)} tokens`); } return heldParts.join(' · '); } return `Condensing ${range} into a short memory to free up room…`; } const parts: string[] = []; const source = Number(args.source_tokens); const summary = Number(args.summary_tokens); if (args.forced) { parts.push(`${rangeCapitalized} condensed into a recap to free up memory`); } else { parts.push(`${rangeCapitalized} condensed into a short memory`); if (source > 0 && summary > 0) { const factor = source / summary; const factorText = factor >= 2 ? ` (${Math.round(factor)}× smaller)` : ''; parts.push(`${formatTokenFigure(source)} → ${formatTokenFigure(summary)} tokens${factorText}`); } } const context = Number(args.context_tokens); const budget = Number(args.context_budget); if (context > 0) { parts.push(budget > 0 ? `working memory ≈ ${formatTokenFigure(context)} of ${formatTokenFigure(budget)} tokens` : `working memory ≈ ${formatTokenFigure(context)} tokens`); } return parts.join(' · '); } /** * Plain-language phases of one compress_history line. Mirrors * COMPRESSION_PHASES in artemis/memory/chunking.py: the backend writes * `args.phase`; the trace `status` (running/success/failed) is left alone. */ export type CompressionPhase = 'summarizing' | 'ready' | 'applied' | 'failed'; const COMPRESSION_PHASES: readonly CompressionPhase[] = ['summarizing', 'ready', 'applied', 'failed']; export function isCompressionTool(tool: any): boolean { const name = String(tool?.name || '').toLowerCase().replace(/^(_)?exec_/, ''); return name === 'compress_history'; } /** * Phase of a compress_history line. Prefers the backend's `args.phase`; * older traces without it fall back to status + note (`held` ⇒ ready). */ export function getCompressionPhase(tool: any): CompressionPhase { const args = getToolArgs(tool) || {}; const declared = String(args.phase || '').toLowerCase() as CompressionPhase; if (COMPRESSION_PHASES.includes(declared)) return declared; const status = String(tool?.status || '').toLowerCase(); if (status === 'failed') return 'failed'; if (status === 'success') return 'applied'; if (String(args.note || '').toLowerCase() === 'held') return 'ready'; return 'summarizing'; } /** One short, non-technical label per compression phase. */ export function getCompressionPhaseLabel(tool: any): string { switch (getCompressionPhase(tool)) { case 'ready': return "摘要已就绪,上下文接近容量上限时替换原文"; case 'applied': return "已使用摘要替换此段记录"; case 'failed': return "摘要失败,保留完整记录"; default: return "正在总结此段记录"; } } /** * A ready-but-held summary is waiting, not working: the card must not keep * its running pulse even though the trace status is still `running`. */ export function isCompressionWaiting(tool: any): boolean { return isCompressionTool(tool) && getCompressionPhase(tool) === 'ready'; } /** * Get Material Symbol icon for tools */ export function getToolIcon(tool: any): string { if (!tool || !tool.name) return 'settings'; const name = tool.name.toLowerCase().replace(/^(_)?exec_/, ''); switch (name) { case 'click': case 'tap': return 'ads_click'; case 'click_sequence': case 'long_press': return 'touch_app'; case 'input_text': case 'input': return 'keyboard'; case 'swipe': return 'swipe'; case 'press_key': return 'keyboard_tab'; case 'manage_app': return 'open_in_new'; case 'wait_for_delay': return 'timer'; case 'wait_for_text': return 'hourglass_empty'; case 'object_detection': return 'search'; case 'ask_explorer': return 'search'; case 'report_failure_analysis': return 'assessment'; case 'run_adb_command': case 'run_short_adb_command': return 'terminal'; case 'web_search': return 'travel_explore'; case 'read_url': return 'language'; case 'search_logs': case 'read_logs': case 'log_analyzer': case 'output_analyzer': return 'receipt_long'; case 'diagnoser': case 'diagnose': return 'medical_services'; case 'video_analyzer': case 'video_analyzer_pure': return 'video_camera_back'; case 'extract_segment_metadata': return 'crop'; case 'spawn_sub_agent': return 'smart_toy'; case 'analyze_audio_only': return 'graphic_eq'; case 'search_history': return 'find_in_page'; case 'replay_steps': return 'manage_search'; case 'get_step_screenshot': return 'image_search'; case 'probe_device': return 'sensors'; case 'outputter': case 'output_synthesis': return 'assignment_turned_in'; case 'compress_history': return 'compress'; default: return 'build'; } } /** * Get formatted title for a tool call card */ export function getToolTitle(tool: any): string { if (!tool || !tool.name) return "工具调用"; const nativeTitle = getNativeActionTitle(tool.name, getToolArgs(tool)); if (nativeTitle) return nativeTitle; const cleanName = tool.name.replace(/^(_)?exec_/, ''); const name = cleanName.toLowerCase(); switch (name) { case 'click': case 'tap': return "点击元素"; case 'click_sequence': return "执行连续点击"; case 'long_press': return "长按元素"; case 'input_text': case 'input': return "输入文本"; case 'swipe': case 'scroll': { const args = getToolArgs(tool); const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : ''); if (dir && isPureDirectionString(dir)) return `滑动屏幕(${String(dir).toUpperCase()})`; return "滑动屏幕"; } case 'drag': case 'drag_and_drop': return "拖动屏幕"; case 'press_key': return "按下实体按键"; case 'manage_app': case 'launch_app': { const args = getToolArgs(tool); const rawAction = args.action ? String(args.action).toLowerCase() : ''; if (rawAction === 'launch') return "启动应用"; if (rawAction === 'stop' || rawAction === 'close') return "停止应用"; return "管理应用"; } case 'wait_for_delay': case 'wait_delay': return "等待"; case 'wait_for_text': return "等待文本"; case 'object_detection': return "定位元素"; case 'ask_explorer': return "搜索屏幕"; case 'report_failure_analysis': return "排查问题"; case 'run_adb_command': case 'run_short_adb_command': return "运行系统命令"; case 'web_search': return "网页搜索"; case 'read_url': return "读取网页"; case 'search_logs': case 'read_logs': return "搜索日志"; case 'log_analyzer': case 'output_analyzer': return "分析日志"; case 'diagnoser': case 'diagnose': return "诊断问题"; case 'video_analyzer': case 'video_analyzer_pure': return "分析屏幕录像"; case 'extract_segment_metadata': return "裁剪屏幕录像"; case 'spawn_sub_agent': return "委派视频分析"; case 'analyze_audio_only': return "分析音轨"; default: return cleanName.replace(/_/g, ' ').replace(/\b\w/g, (c: string) => c.toUpperCase()); } } /** * Join per-point self-described targets (click_sequence `target_descriptions`) * into one label, chained with ' → ' like the coordinate rendering. * Returns '' when the value is missing or holds no usable text. */ export function joinTargetDescriptions(descriptions: any): string { if (!Array.isArray(descriptions)) return ''; const parts = descriptions .map(d => (d === null || d === undefined) ? '' : String(d).trim()) .filter(d => d.length > 0); return parts.join(' → '); } /** * Get target text description for tools */ export function getToolTargetText(tool: any): string { if (!tool || !tool.name) return ''; const args = getToolArgs(tool); const name = tool.name.toLowerCase().replace(/^(_)?exec_/, ''); if (name === 'manage_app' || name === 'launch_app') { const rawApp = args.app_name || args.package_name || args.app || ''; const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : ''; const rawAct = args.action ? String(args.action).toLowerCase() : ''; const act = rawAct ? (rawAct.charAt(0).toUpperCase() + rawAct.slice(1)) : ''; if (act && app) { return `${act} ${app}`; } return app || act || ''; } if (name === 'swipe' && typeof args.action === 'string' && isPureDirectionString(args.action)) { return args.action; } if (name === 'wait_for_text') { return args.text || ''; } if (name === 'object_detection') { if (Array.isArray(args.queries)) return args.queries.join(', '); return args.queries || ''; } if (name === 'ask_explorer' || name === 'web_search') { return args.query || args.prompt || ''; } if (name === 'read_url') { return args.url || ''; } if (name === 'search_logs' || name === 'read_logs') { return args.query || args.filter || ''; } if (name === 'report_failure_analysis') { return args.status || args.reason || args.analysis || ''; } if (name === 'click_sequence' || name === 'tap_sequence') { // One self-described target per point, chained like getToolCoords does. const joined = joinTargetDescriptions(args.target_descriptions); if (joined) return joined; } if (args.target && typeof args.target === 'string') { return args.target; } if (args.target && typeof args.target === 'number') { return `Element #${args.target}`; } if (args.index !== undefined) { return `Element #${args.index}`; } return args.target_text || args.target_description || args.target_class || args.element || args.element_text || (name !== 'input_text' ? args.text : '') || ''; } /** * Get input label for tools */ export function getToolInputLabel(tool: any): string { if (!tool || !tool.name) return "输入"; const name = tool.name.toLowerCase().replace(/^(_)?exec_/, ''); if (name === 'wait_for_delay' || name === 'wait_delay' || name === 'delay' || name === 'wait') { return "耗时"; } if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') { const args = getToolArgs(tool); const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : ''); if (dir && isPureDirectionString(dir)) { return "方向"; } return "输入"; } if (name === 'press_key' || name === 'press_home' || name === 'press_back') { return "按键"; } if (name === 'input_text' || name === 'input') { return "输入文本"; } return "输入"; } /** * Get input value for tools */ export function getToolInputText(tool: any): string { if (!tool || !tool.name) return ''; const args = getToolArgs(tool); const name = tool.name.toLowerCase().replace(/^(_)?exec_/, ''); if (name === 'press_key') { return args.key || args.keycode || ''; } if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') { const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : ''); if (dir && isPureDirectionString(dir)) { return String(dir).toUpperCase(); } return ''; } if (name === 'wait_for_delay' || name === 'wait_delay') { return args.time_in_ms ? `${args.time_in_ms}ms` : (args.delay_ms ? `${args.delay_ms}ms` : (args.time ? `${args.time}` : '')); } if (name === 'long_press' && args.duration) { return `Duration: ${args.duration}ms`; } if (name === 'input_text' || name === 'input') { return args.text || args.input_text || ''; } return args.input_text || ''; } /** * Get coordinates string representation for tools */ export function getToolCoords(tool: any): string { if (!tool || !tool.name) return ''; const args = getToolArgs(tool); const cleanName = tool.name.replace(/^(_)?exec_/, ''); const name = cleanName.toLowerCase(); // Check sequence first if action is sequence or sequence arg is present const isSequenceAction = name === 'click_sequence' || name === 'tap_sequence' || Boolean(args.sequence || args.normalized_sequence); const rawSeq = args.normalized_sequence || args.sequence || args.targets || (Array.isArray(args.coordinates) && Array.isArray(args.coordinates[0]) ? args.coordinates : null); const seqPoints = parseSequenceCoordinates(rawSeq); if (seqPoints && seqPoints.length > 0 && (isSequenceAction || seqPoints.length > 1)) { return seqPoints.map(pt => `[${pt[0]}, ${pt[1]}]`).join(' → '); } // Check normalized start and end first const normStart = extractNumbersFromCoordinateValue(args.normalized_start_coordinates || args.start_coordinates || args.start); const normEnd = extractNumbersFromCoordinateValue(args.normalized_end_coordinates || args.end_coordinates || args.end); if (normStart && normEnd && normStart.length === 2 && normEnd.length === 2) { return `[${normStart[0]}, ${normStart[1]}] → [${normEnd[0]}, ${normEnd[1]}]`; } const coords = extractNumbersFromCoordinateValue(args.normalized_coordinates) || extractNumbersFromCoordinateValue(args.coordinates) || extractNumbersFromCoordinateValue(args.target) || extractNumbersFromCoordinateValue(args.action) || extractNumbersFromCoordinateValue(args.gesture) || extractNumbersFromCoordinateValue(args.point); if (coords && Array.isArray(coords)) { if (coords.length === 4) { return `[${coords[0]}, ${coords[1]}] → [${coords[2]}, ${coords[3]}]`; } if (coords.length === 2) { return `[${coords[0]}, ${coords[1]}]`; } return coords.join(', '); } if (args.x !== undefined && args.y !== undefined) { return `[${args.x}, ${args.y}]`; } return ''; } /** * Get analysis / reasoning text for tools */ export function getToolAnalysisText(tool: any): string { if (!tool || !tool.name) return ''; const args = getToolArgs(tool); return args.analysis || args.reasoning || args.summary || ''; } /** * Check if tool is ADB command */ export function isAdbCommandTool(tool: any): boolean { if (!tool || !tool.name) return false; const name = tool.name.toLowerCase().replace(/^(_)?exec_/, ''); return name === 'run_adb_command' || name === 'run_short_adb_command'; } /** * Get ADB command line string */ export function getAdbCommandLine(tool: any): string { const args = getToolArgs(tool); return args.CommandLine || ''; } /** * Get ADB working directory */ export function getAdbCwd(tool: any): string { const args = getToolArgs(tool); return args.Cwd || ''; } /** * Get ADB requested terminal id */ export function getAdbTerminalId(tool: any): string { const args = getToolArgs(tool); return args.RequestedTerminalID || ''; } /** * Get fallback generic details for unspecified tools */ export function getToolGenericDetails(tool: any): string { if (!tool) return ''; if (getNativeActionTitle(tool.name || '', getToolArgs(tool))) { return extractToolExtraParams(tool).map(param => `${param.key}:${param.value}`).join(','); } if (isAdbCommandTool(tool)) { return ''; } if (getToolTargetText(tool) || getToolInputText(tool) || getToolCoords(tool) || getToolAnalysisText(tool)) { return ''; } const args = getToolArgs(tool); if (!args || typeof args !== 'object') return ''; const ignoredKeys = new Set(['state', 'controller', 'ctx', 'session_id', 'step_id', 'trace_id', 'parent_trace_id', 'times', 'delay_ms']); const entries = Object.entries(args).filter(([k, v]) => !ignoredKeys.has(k) && v !== undefined && v !== null && v !== ''); if (entries.length === 0) return ''; return entries.map(([k, v]) => `${k}: ${typeof v === 'object' ? JSON.stringify(v) : v}`).join(', '); } /** * Check if tool execution failed */ export function isToolFailed(tool: any): boolean { if (!tool) return false; if (tool.status === 'failed' || tool.status === 'error') return true; const args = getToolArgs(tool); if (args.status === 'failed' || args.status === 'error' || args.status === 'cannot_fix') return true; return false; } /** * Get error message for a failed tool */ export function getToolErrorMessage(tool: any): string { if (!tool) return "工具失败"; const args = getToolArgs(tool); if (args.status === 'cannot_fix') return 'Status: cannot_fix'; if (args.error || args.message || args.failure_reason) { return args.error || args.message || args.failure_reason; } if (tool.error || tool.message) return tool.error || tool.message; return "操作失败"; } /** * Extract extra parameters for generic tools */ export function extractToolExtraParams(toolData: any, cache?: WeakMap): ActionParam[] { if (!toolData) return []; const nativeArgs = getNativeActionArgs(getToolArgs(toolData)); const native = getNativeActionTitle(toolData.name || '', nativeArgs) !== null; const payload = native ? nativeArgs : (toolData.payload || toolData.args || toolData); if (!payload || typeof payload !== 'object') return []; if (cache && cache.has(payload)) { return cache.get(payload)!; } const standardKeys = new Set([ 'action', 'name', 'type', 'target_text', 'target_description', 'target_descriptions', 'text', 'input_text', 'target', 'coordinates', 'coords', 'target_bounds', 'bounds', 'target_resource_id', 'resource_id', 'target_class', 'class_name', 'normalized_coordinates', 'pre_image_name', 'post_image_name', 'pre_screenshot', 'post_screenshot', 'before_screenshot', 'after_screenshot', 'status', 'success', 'timestamp', 'created_at', 'start_time', 'execution_id', 'controller', 'agent', 'session_id', 'step_id', 'app_name', 'package_name', 'app', 'key', 'keycode', 'time_in_ms', 'delay_ms', 'delay_seconds', 'duration', 'args', 'kwargs', 'parameters', 'extra_params', 'payload', 'trace_id', 'result', 'error', 'analysis', 'details', 'command', 'cwd', 'terminal_id' ]); const result: ActionParam[] = []; const rawArgs = toolData.args || toolData.Args || toolData.kwargs || toolData.parameters || toolData.payload; let parsedArgs: any = {}; if (rawArgs) { if (typeof rawArgs === 'object') { parsedArgs = rawArgs; } else if (typeof rawArgs === 'string') { try { parsedArgs = JSON.parse(rawArgs); } catch { if (!rawArgs.includes('<') && !rawArgs.includes('object at')) { parsedArgs = { details: rawArgs }; } } } } const mergedObj = { ...toolData, ...(typeof payload === 'object' ? payload : {}), ...parsedArgs }; if (native) Object.assign(mergedObj, nativeArgs); for (const [k, v] of Object.entries(mergedObj)) { const lowerK = k.toLowerCase(); if (standardKeys.has(lowerK) && !(native && lowerK === 'target')) continue; if (v === null || v === undefined || v === '') continue; if (native) { result.push(formatNativeActionParam(k, v)); continue; } let valStr = String(v); if (valStr.includes('object at 0x') || valStr.startsWith(' c.toUpperCase()); if (lowerK === 'time_in_ms' || lowerK === 'delay_ms') prettyKey = "延迟"; result.push({ key: prettyKey, value: valStr }); } if (cache) { cache.set(payload, result); } return result; } /** * Check if text is genuine human thinking rather than raw JSON payload output */ export function isHumanThinking(text: string | null): boolean { if (!text) return false; let trimmed = text.trim(); if (!trimmed) return false; // Strip markdown code blocks if wrapped: ```json ... ``` or ``` ... ``` if (trimmed.startsWith('```')) { const match = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i); if (match) { trimmed = match[1].trim(); } } // Direct object or array start if (trimmed.startsWith('{') || trimmed.startsWith('[')) { return false; } // Try parsing as JSON if it looks like JSON structures try { const parsed = JSON.parse(trimmed); if (typeof parsed === 'object' && parsed !== null) { return false; } } catch { // Not valid standalone JSON, continue checks } // Check for common JSON object detection / grounding / action / tool patterns if (/^\s*\{.*"label"\s*:.*"results"\s*:/s.test(trimmed) || /^\s*\{.*"point"\s*:.*"label"\s*:/s.test(trimmed) || /^\s*\{.*"action"\s*:/s.test(trimmed) || /^\s*\{.*"name"\s*:/s.test(trimmed) || /^\s*\{\s*"[^"]+"\s*:\s*/.test(trimmed) || /^\s*\[\s*\{\s*"[^"]+"\s*:\s*/.test(trimmed)) { return false; } return true; } /** * Clean up raw error messages for display in the UI */ export function cleanErrorMessage(rawError: any): string { if (typeof rawError === 'object' && rawError !== null) { if (rawError.message && typeof rawError.message === 'string') { return cleanErrorMessage(rawError.message); } if (rawError.error?.message && typeof rawError.error?.message === 'string') { return cleanErrorMessage(rawError.error.message); } if (rawError.detail && typeof rawError.detail === 'string') { return cleanErrorMessage(rawError.detail); } try { return JSON.stringify(rawError); } catch { return String(rawError); } } const errorStr = String(rawError).trim(); if (!errorStr) return "未知错误"; // 1. Try regex extraction for "message": "..." const doubleQuoteMsgMatch = errorStr.match(/"message"\s*:\s*"((?:[^"\\]|\\.)*)"/i); if (doubleQuoteMsgMatch && doubleQuoteMsgMatch[1]) { const unescaped = doubleQuoteMsgMatch[1].replace(/\\"/g, '"').replace(/\\n/g, ' ').trim(); if (unescaped) return unescaped; } // 2. Try regex extraction for 'message': '...' const singleQuoteMsgMatch = errorStr.match(/'message'\s*:\s*['"]((?:[^'\\]|\\.)*)['"]/i); if (singleQuoteMsgMatch && singleQuoteMsgMatch[1]) { const unescaped = singleQuoteMsgMatch[1].replace(/\\'/g, "'").replace(/\\n/g, ' ').trim(); if (unescaped) return unescaped; } // 3. Try parsing JSON substrings inside the error string const jsonMatch = errorStr.match(/\{[\s\S]*\}/); if (jsonMatch) { try { const normalizedJsonStr = jsonMatch[0] .replace(/'/g, '"') .replace(/True/g, 'true') .replace(/False/g, 'false') .replace(/None/g, 'null'); const parsed = JSON.parse(normalizedJsonStr); if (parsed?.message) return String(parsed.message); if (parsed?.error?.message) return String(parsed.error.message); } catch { // Ignore } } // 4. Fallback: truncate at trace dump headers or return clean message let fallback = errorStr; if (fallback.includes('=== Source Location Trace')) { fallback = fallback.split('=== Source Location Trace')[0]; } if (fallback.includes('[type.googleapis.com')) { fallback = fallback.split('[type.googleapis.com')[0]; } fallback = fallback .replace(/^Pre-execution validation failed:\s*/i, '') .replace(/^Pixel-level validation failed:\s*/i, '') .replace(/^Execution error:\s*/i, '') .replace(/^ServerError:\s*/i, '') // LLM wrappers may add this prefix more than once. It is context, not the // actual provider reason, and an empty prefix must not be shown as though // it were a useful error message. .replace(/^(?:LLM\s+(?:Request\s+)?Error\s*:\s*)+/i, '') .trim(); return fallback || "未知错误"; }