chenmingxuan
2ddec53954
Localize the web console and persist custom model and capability settings. Use OpenAI Responses exclusively and move runtime limits into environment configuration. Bootstrap uv and the virtual environment on Windows, support optional China mirrors, and package source distributions without environments, build output, or credentials. Update regression tests and repository quality checks for the new behavior.
1420 lines
50 KiB
TypeScript
1420 lines
50 KiB
TypeScript
/**
|
||
* Copyright 2026 Google LLC
|
||
*
|
||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||
* you may not use this file except in compliance with the License.
|
||
* You may obtain a copy of the License at
|
||
*
|
||
* http://www.apache.org/licenses/LICENSE-2.0
|
||
*
|
||
* Unless required by applicable law or agreed to in writing, software
|
||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
* See the License for the specific language governing permissions and
|
||
* limitations under the License.
|
||
*/
|
||
|
||
import { ActionParam } from '../core/models/stream.model';
|
||
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util';
|
||
|
||
// Tool objects are replaced (not mutated) when a trace is updated, so a
|
||
// WeakMap keyed on the tool is a safe memo for the parsed-args lookup that
|
||
// templates would otherwise re-run (including JSON.parse) on every render.
|
||
const toolArgsCache = new WeakMap<object, any>();
|
||
|
||
/**
|
||
* Extract arguments from tool payload or direct args field
|
||
*/
|
||
export function getToolArgs(tool: any): any {
|
||
if (!tool) return {};
|
||
if (typeof tool === 'object') {
|
||
const cached = toolArgsCache.get(tool);
|
||
if (cached !== undefined) return cached;
|
||
const args = computeToolArgs(tool);
|
||
toolArgsCache.set(tool, args);
|
||
return args;
|
||
}
|
||
return computeToolArgs(tool);
|
||
}
|
||
|
||
function computeToolArgs(tool: any): any {
|
||
if (tool.payload) {
|
||
let payloadObj = tool.payload;
|
||
if (typeof payloadObj === 'string') {
|
||
try { payloadObj = JSON.parse(payloadObj); } catch {}
|
||
}
|
||
if (payloadObj && typeof payloadObj === 'object') {
|
||
if (payloadObj.args && typeof payloadObj.args === 'object') {
|
||
return payloadObj.args;
|
||
}
|
||
return payloadObj;
|
||
}
|
||
}
|
||
if (tool.args) {
|
||
let argsObj = tool.args;
|
||
if (typeof argsObj === 'string') {
|
||
try { argsObj = JSON.parse(argsObj); } catch {}
|
||
}
|
||
if (argsObj && typeof argsObj === 'object') {
|
||
return argsObj;
|
||
}
|
||
}
|
||
return {};
|
||
}
|
||
|
||
/**
|
||
* Check if a tool is one of the five note-related tools
|
||
*/
|
||
export function isNoteTool(tool: any): boolean {
|
||
if (!tool || !tool.name) return false;
|
||
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '');
|
||
const nameLower = cleanName.toLowerCase();
|
||
return ['save_note', 'read_note', 'list_notes', 'update_note', 'append_note'].includes(nameLower);
|
||
}
|
||
|
||
/**
|
||
* Check if a tool is video analysis related
|
||
*/
|
||
export function isVideoTool(tool: any): boolean {
|
||
if (!tool || !tool.name) return false;
|
||
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '');
|
||
const nameLower = cleanName.toLowerCase();
|
||
return [
|
||
'video_analysis',
|
||
'video_analyzer',
|
||
'video_analyzer_pure',
|
||
'spawn_sub_agent',
|
||
'analyze_audio_only'
|
||
].includes(nameLower);
|
||
}
|
||
|
||
export type VideoAnalysisOutcome = 'running' | 'recovering' | 'waiting' | 'complete' | 'partial' | 'failed';
|
||
|
||
export interface VideoAnalysisRange {
|
||
start: number;
|
||
end: number;
|
||
category?: string;
|
||
retryable?: boolean;
|
||
}
|
||
|
||
export interface VideoAnalysisView {
|
||
outcome: VideoAnalysisOutcome;
|
||
title: string;
|
||
query: string;
|
||
summary: string;
|
||
reuse: 'none' | 'partial' | 'full';
|
||
requestedRange: VideoAnalysisRange | null;
|
||
completedRanges: VideoAnalysisRange[];
|
||
failedRanges: VideoAnalysisRange[];
|
||
evidenceCount: number;
|
||
completedCount: number;
|
||
totalCount: number;
|
||
fallbackUsed: boolean;
|
||
}
|
||
|
||
function numericRange(value: any): VideoAnalysisRange | null {
|
||
if (!value || typeof value !== 'object') return null;
|
||
const start = Number(value.start);
|
||
const end = Number(value.end);
|
||
if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start) return null;
|
||
return {
|
||
start,
|
||
end,
|
||
category: value.category ? String(value.category) : undefined,
|
||
retryable: value.retryable === true
|
||
};
|
||
}
|
||
|
||
function parseVideoResultText(text: string): Partial<VideoAnalysisView> {
|
||
const value = text.trim();
|
||
if (!value) return {};
|
||
if (value.startsWith('CACHED VIDEO ANALYSIS:')) {
|
||
return { outcome: 'complete', reuse: 'full', summary: value.replace(/^CACHED VIDEO ANALYSIS:\s*/, '') };
|
||
}
|
||
if (value.startsWith('PARTIAL VIDEO ANALYSIS')) {
|
||
return { outcome: 'partial', reuse: 'partial', summary: value };
|
||
}
|
||
if (value.startsWith('All sub-agent chunks failed') || value.startsWith('Error:')) {
|
||
return { outcome: 'failed', summary: value };
|
||
}
|
||
if (value.includes('Analysis is already in progress in another video agent')) {
|
||
return { outcome: 'waiting', summary: "另一个视频智能体正在分析此段内容。" };
|
||
}
|
||
return { outcome: 'complete', summary: value };
|
||
}
|
||
|
||
/** Build the user-facing, backward-compatible video analysis state. */
|
||
export function getVideoAnalysisView(tool: any): VideoAnalysisView | null {
|
||
if (!isVideoTool(tool)) return null;
|
||
const args = getToolArgs(tool);
|
||
const payload = tool?.payload && typeof tool.payload === 'object' ? tool.payload : {};
|
||
const rawResult = payload.result ?? tool.result ?? null;
|
||
const structured = rawResult && typeof rawResult === 'object' ? rawResult : {};
|
||
const parsed = typeof rawResult === 'string' ? parseVideoResultText(rawResult) : {};
|
||
|
||
let outcome = String(structured.outcome || payload.outcome || parsed.outcome || '').toLowerCase() as VideoAnalysisOutcome;
|
||
if (!['running', 'recovering', 'waiting', 'complete', 'partial', 'failed'].includes(outcome)) {
|
||
outcome = tool?.status === 'failed' || tool?.status === 'error'
|
||
? 'failed'
|
||
: tool?.status === 'running' ? 'running' : 'complete';
|
||
}
|
||
const recovering = structured.recovering === true || structured.fallback_used === true;
|
||
if (outcome === 'running' && recovering) outcome = 'recovering';
|
||
|
||
const start = Number(args.start_time ?? structured.requested_range?.start);
|
||
const end = Number(args.end_time ?? structured.requested_range?.end);
|
||
const requestedRange = Number.isFinite(start) && Number.isFinite(end) && end > start
|
||
? { start, end }
|
||
: numericRange(structured.requested_range);
|
||
const completedRanges = Array.isArray(structured.completed_ranges)
|
||
? structured.completed_ranges.map(numericRange).filter((range: VideoAnalysisRange | null): range is VideoAnalysisRange => Boolean(range))
|
||
: [];
|
||
const failedRanges = Array.isArray(structured.failed_ranges)
|
||
? structured.failed_ranges.map(numericRange).filter((range: VideoAnalysisRange | null): range is VideoAnalysisRange => Boolean(range))
|
||
: [];
|
||
const completedCount = Number(structured.completed_count ?? completedRanges.length ?? 0);
|
||
const totalCount = Number(structured.total_count ?? (completedRanges.length + failedRanges.length));
|
||
const titleByOutcome: Record<VideoAnalysisOutcome, string> = {
|
||
running: "正在分析屏幕录像",
|
||
recovering: "正在分析尚未结束的录像片段",
|
||
waiting: "正在等待已有的视频分析",
|
||
complete: structured.reuse === 'full' || parsed.reuse === 'full'
|
||
? "已复用视频分析"
|
||
: "已分析屏幕录像",
|
||
partial: "视频分析部分完成",
|
||
failed: "视频分析未返回结果"
|
||
};
|
||
|
||
return {
|
||
outcome,
|
||
title: titleByOutcome[outcome],
|
||
query: String(args.specific_query || args.query || args.prompt || structured.query || ''),
|
||
summary: String(structured.summary || parsed.summary || ''),
|
||
reuse: (structured.reuse || parsed.reuse || 'none') as 'none' | 'partial' | 'full',
|
||
requestedRange,
|
||
completedRanges,
|
||
failedRanges,
|
||
evidenceCount: Number(structured.evidence_count || 0),
|
||
completedCount: Number.isFinite(completedCount) ? completedCount : 0,
|
||
totalCount: Number.isFinite(totalCount) ? totalCount : 0,
|
||
fallbackUsed: structured.fallback_used === true
|
||
};
|
||
}
|
||
|
||
export function formatVideoTime(seconds: number): string {
|
||
const safe = Math.max(0, Number(seconds) || 0);
|
||
const minutes = Math.floor(safe / 60);
|
||
const secs = Math.floor(safe % 60);
|
||
return `${String(minutes).padStart(2, '0')}:${String(secs).padStart(2, '0')}`;
|
||
}
|
||
|
||
/**
|
||
* Extract target video filename or description for video analysis tools
|
||
*/
|
||
export function getVideoToolTarget(tool: any): string | null {
|
||
if (!tool) return null;
|
||
const args = getToolArgs(tool);
|
||
if (args.video_path || args.video_file || args.file_path || args.file) {
|
||
const p = String(args.video_path || args.video_file || args.file_path || args.file);
|
||
return p.split('/').pop() || p;
|
||
}
|
||
if (args.time_description) {
|
||
return String(args.time_description);
|
||
}
|
||
if (args.start_time !== undefined || args.end_time !== undefined) {
|
||
const start = args.start_time ?? 0;
|
||
const end = args.end_time ? `${args.end_time}s` : 'end';
|
||
return `${start}s - ${end}`;
|
||
}
|
||
if (args.purpose) {
|
||
return String(args.purpose);
|
||
}
|
||
return 'screen_record.mp4';
|
||
}
|
||
|
||
/**
|
||
* Canonical list of mobile device automation and ADB command tool names.
|
||
*/
|
||
export const DEVICE_ACTION_TOOL_NAMES = new Set([
|
||
'click', 'click_sequence', 'long_press', 'long_press_on', 'input', 'input_text',
|
||
'swipe', 'scroll', 'press_key', 'press_home', 'press_back', 'launch_app', 'manage_app', 'open_app',
|
||
'focus_and_input_text', 'focus_and_clear_text', 'tap', 'wait_for_delay', 'wait',
|
||
'run_adb_command', 'run_short_adb_command'
|
||
]);
|
||
|
||
/**
|
||
* Check if a tool represents a real mobile device action or ADB command.
|
||
*/
|
||
export function isDeviceActionTool(tool: any): boolean {
|
||
if (!tool || !tool.name) return false;
|
||
if (tool.type === 'llm_call') return false;
|
||
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '').toLowerCase();
|
||
return DEVICE_ACTION_TOOL_NAMES.has(cleanName);
|
||
}
|
||
|
||
/**
|
||
* Backward-compatible alias for isDeviceActionTool.
|
||
*/
|
||
export function isFailureAnalyzerActionTool(tool: any): boolean {
|
||
return isDeviceActionTool(tool);
|
||
}
|
||
|
||
/**
|
||
* Framework-internal infrastructure tools that are not intended for end-user timeline display.
|
||
*/
|
||
export function isInternalPlumbingTool(tool: any): boolean {
|
||
if (!tool || !tool.name) return true;
|
||
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '').toLowerCase();
|
||
if (cleanName.includes('safety_net')) return true;
|
||
if (cleanName === 'report_task_status' || cleanName === 'report_status' || cleanName === 'submit_task_status') {
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* Determines whether a tool invocation should be displayed in the timeline.
|
||
*/
|
||
export function shouldShowTool(tool: any, _stepData?: any): boolean {
|
||
if (!tool || !tool.name) return false;
|
||
if (tool.type === 'llm_call') return false;
|
||
if (tool.type === 'agent') return false;
|
||
if (tool.type === 'log') return false; // system notes render as quiet rows, not tool cards
|
||
if (isInternalPlumbingTool(tool)) return false;
|
||
|
||
return true;
|
||
}
|
||
|
||
/**
|
||
* Extracts human-readable initiating agent name / role for a tool.
|
||
*/
|
||
export function getToolAgentName(tool: any): string | null {
|
||
if (!tool) return null;
|
||
const name = (tool.agent_name || tool.agent || '').toLowerCase();
|
||
if (name.includes('failure') || name.includes('analyzer')) {
|
||
return null; // Omit self-healing label per user instruction
|
||
}
|
||
if (name.includes('outputter')) {
|
||
return 'Outputter';
|
||
}
|
||
if (name.includes('validator')) {
|
||
return 'Validator';
|
||
}
|
||
if (name.includes('diagnos')) {
|
||
return 'Diagnoser';
|
||
}
|
||
if (name.includes('explorer')) {
|
||
return 'Explorer';
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* Filter out nested child wrapper duplicates while preserving independent executions at the same level
|
||
*/
|
||
export function getUniqueGenericTools(tools: any[] | undefined): any[] {
|
||
if (!tools) return [];
|
||
|
||
// Only Google GenAI SDK retries are observable. Artemis wrapper retries and
|
||
// opaque provider retries must never be presented as provider internals.
|
||
const retryTools = tools.filter(tool =>
|
||
tool?.type === 'llm_call'
|
||
&& tool?.status === 'retrying'
|
||
&& (tool?.name === 'llm_retry' || tool?.name === 'llm_retry_group')
|
||
&& tool?.payload?.source === 'provider_sdk'
|
||
&& ['google', 'gemini'].includes(String(tool?.payload?.provider || '').toLowerCase())
|
||
);
|
||
const retryKey = (tool: any) => String(
|
||
tool?.payload?.request_id || `legacy-${tool?.step_id || 'unassigned'}`
|
||
);
|
||
const retryGroups = new Map<string, any[]>();
|
||
for (const retry of retryTools) {
|
||
const key = retryKey(retry);
|
||
retryGroups.set(key, [...(retryGroups.get(key) || []), retry]);
|
||
}
|
||
const terminalRequestIds = new Set(
|
||
tools
|
||
.filter(tool => tool?.type === 'llm_call' && tool?.status === 'failed')
|
||
.map(tool => tool?.payload?.request_id)
|
||
.filter((requestId): requestId is string => typeof requestId === 'string' && !!requestId)
|
||
);
|
||
|
||
const buildRetryAggregate = (group: any[]) => {
|
||
const firstRetry = group[0];
|
||
const latestRetry = group[group.length - 1];
|
||
return {
|
||
...firstRetry,
|
||
trace_id: `llm-retry-group-${firstRetry?.payload?.request_id || firstRetry?.trace_id || firstRetry?.timestamp || 'unknown'}`,
|
||
name: 'llm_retry_group',
|
||
payload: {
|
||
...(latestRetry?.payload || {}),
|
||
retry_count: group.length,
|
||
total_delay: group.reduce(
|
||
(total, retry) => total + (Number(retry?.payload?.delay) || 0),
|
||
0
|
||
),
|
||
retries: group.map(retry => ({
|
||
trace_id: retry.trace_id,
|
||
timestamp: retry.timestamp,
|
||
error: retry?.payload?.error || retry.error,
|
||
delay: Number(retry?.payload?.delay) || 0,
|
||
provider: retry?.payload?.provider,
|
||
source: retry?.payload?.source,
|
||
request_id: retry?.payload?.request_id,
|
||
scheduled_at: retry?.payload?.scheduled_at
|
||
}))
|
||
}
|
||
};
|
||
};
|
||
|
||
const toolMap = new Map<string, any>();
|
||
for (const t of tools) {
|
||
if (t && t.trace_id) {
|
||
toolMap.set(t.trace_id, t);
|
||
}
|
||
}
|
||
|
||
const cleanName = (nameStr: string) => (nameStr || '').replace(/^(_)?(self\.)?exec_/, '').toLowerCase().trim();
|
||
|
||
const result: any[] = [];
|
||
const seenTraces = new Set<string>();
|
||
const addedRetryGroups = new Set<string>();
|
||
|
||
for (const tool of tools) {
|
||
if (!tool || !tool.name) continue;
|
||
if (tool.type === 'agent') continue;
|
||
|
||
if (tool.type === 'llm_call' && tool.status === 'retrying') {
|
||
const groupKey = retryKey(tool);
|
||
const group = retryGroups.get(groupKey);
|
||
const requestId = tool?.payload?.request_id;
|
||
if (
|
||
group
|
||
&& !addedRetryGroups.has(groupKey)
|
||
&& !(requestId && terminalRequestIds.has(requestId))
|
||
) {
|
||
result.push(buildRetryAggregate(group));
|
||
addedRetryGroups.add(groupKey);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (tool.type === 'llm_call' && tool.status === 'failed') {
|
||
// Callback-level attempt errors are implementation details. The wrapper
|
||
// emits one terminal llm_pause trace after retries are exhausted.
|
||
if (tool.name === 'llm_pause' || tool?.payload?.pause === true) {
|
||
result.push(tool);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
const currClean = cleanName(tool.name);
|
||
|
||
// Check if this tool is a nested child execution of a parent tool with the exact same tool name
|
||
let isNestedChildDuplicate = false;
|
||
if (tool.parent_trace_id) {
|
||
const parentTool = toolMap.get(tool.parent_trace_id);
|
||
if (parentTool && cleanName(parentTool.name) === currClean) {
|
||
isNestedChildDuplicate = true;
|
||
}
|
||
}
|
||
|
||
if (!isNestedChildDuplicate) {
|
||
if (tool.trace_id) {
|
||
if (!seenTraces.has(tool.trace_id)) {
|
||
seenTraces.add(tool.trace_id);
|
||
// Always use the latest state from toolMap for this trace_id
|
||
result.push(toolMap.get(tool.trace_id) || tool);
|
||
}
|
||
} else {
|
||
result.push(tool);
|
||
}
|
||
}
|
||
}
|
||
return collapseVideoAnalysisTools(result);
|
||
}
|
||
|
||
/**
|
||
* Present one note-style timeline row for one video-analyzer execution. Child
|
||
* chunk, audio, and wrapper traces stay available in the raw trace but do not
|
||
* look like repeated user-visible analyses.
|
||
*/
|
||
function collapseVideoAnalysisTools(tools: any[]): any[] {
|
||
const groups = new Map<string, { firstIndex: number; tools: any[] }>();
|
||
const passthrough: Array<{ index: number; tool: any }> = [];
|
||
|
||
tools.forEach((tool, index) => {
|
||
if (!isVideoTool(tool)) {
|
||
passthrough.push({ index, tool });
|
||
return;
|
||
}
|
||
const cleanName = String(tool.name || '').replace(/^(_)?(self\.)?exec_/, '').toLowerCase();
|
||
const groupId = cleanName === 'video_analyzer' || cleanName === 'video_analyzer_pure'
|
||
? String(tool.trace_id || tool.parent_trace_id || `video-${index}`)
|
||
: String(tool.parent_trace_id || tool.trace_id || `video-${index}`);
|
||
const existing = groups.get(groupId);
|
||
if (existing) {
|
||
existing.tools.push(tool);
|
||
} else {
|
||
groups.set(groupId, { firstIndex: index, tools: [tool] });
|
||
}
|
||
});
|
||
|
||
const collapsed = [...passthrough];
|
||
for (const [groupId, group] of groups) {
|
||
const views = group.tools
|
||
.map(getVideoAnalysisView)
|
||
.filter((view): view is VideoAnalysisView => Boolean(view));
|
||
if (!views.length) continue;
|
||
|
||
const hasActive = views.some(view => view.outcome === 'running' || view.outcome === 'recovering');
|
||
const completed = views.filter(view => view.outcome === 'complete');
|
||
const hasPartial = views.some(view => view.outcome === 'partial');
|
||
const failed = views.filter(view => view.outcome === 'failed');
|
||
const waiting = views.filter(view => view.outcome === 'waiting');
|
||
let outcome: VideoAnalysisOutcome;
|
||
if (hasActive) {
|
||
outcome = views.some(view => view.outcome === 'recovering') ? 'recovering' : 'running';
|
||
} else if (hasPartial || (completed.length > 0 && failed.length > 0)) {
|
||
outcome = 'partial';
|
||
} else if (completed.length > 0) {
|
||
outcome = 'complete';
|
||
} else if (waiting.length > 0) {
|
||
outcome = 'waiting';
|
||
} else {
|
||
outcome = 'failed';
|
||
}
|
||
|
||
const ranges = views
|
||
.map(view => view.requestedRange)
|
||
.filter((range): range is VideoAnalysisRange => Boolean(range));
|
||
const requestedRange = ranges.length
|
||
? {
|
||
start: Math.min(...ranges.map(range => range.start)),
|
||
end: Math.max(...ranges.map(range => range.end))
|
||
}
|
||
: null;
|
||
const reuse = completed.length > 0 && completed.every(view => view.reuse === 'full')
|
||
? 'full'
|
||
: views.some(view => view.reuse !== 'none') ? 'partial' : 'none';
|
||
const base = group.tools[0];
|
||
const structuredResult = {
|
||
outcome,
|
||
reuse,
|
||
requested_range: requestedRange,
|
||
completed_count: completed.length,
|
||
total_count: views.length,
|
||
evidence_count: views.reduce((sum, view) => sum + view.evidenceCount, 0),
|
||
query: views.find(view => view.query)?.query || '',
|
||
recovering: outcome === 'recovering',
|
||
fallback_used: views.some(view => view.fallbackUsed)
|
||
};
|
||
|
||
collapsed.push({
|
||
index: group.firstIndex,
|
||
tool: {
|
||
...base,
|
||
trace_id: `video-analysis-${groupId}`,
|
||
name: 'video_analysis',
|
||
status: outcome === 'running' || outcome === 'recovering' ? 'running' : 'success',
|
||
payload: {
|
||
...(base.payload || {}),
|
||
args: {
|
||
...(base.payload?.args || base.args || {}),
|
||
start_time: requestedRange?.start,
|
||
end_time: requestedRange?.end
|
||
},
|
||
result: structuredResult
|
||
}
|
||
}
|
||
});
|
||
}
|
||
|
||
collapsed.sort((a, b) => a.index - b.index);
|
||
return collapsed.map(item => item.tool);
|
||
}
|
||
|
||
/**
|
||
* Get note key for the tool if it exists
|
||
*/
|
||
export function getToolKey(tool: any): string | null {
|
||
if (!tool || !tool.name) return null;
|
||
const args = tool.payload?.args || tool.args;
|
||
const key = args?.key;
|
||
if (!key) return null;
|
||
return key.toLowerCase().endsWith('.md') ? key : `${key}.md`;
|
||
}
|
||
|
||
/**
|
||
* Get display label for tools (e.g. for pills or headers)
|
||
*/
|
||
export function getToolDisplayLabel(tool: any, isFirstSaveNote: boolean = false): string {
|
||
if (!tool || !tool.name) return '';
|
||
const cleanName = tool.name.replace(/^(_)?exec_/, '');
|
||
const nameLower = cleanName.toLowerCase();
|
||
const args = getToolArgs(tool);
|
||
|
||
switch (nameLower) {
|
||
case 'manage_app':
|
||
case 'launch_app': {
|
||
const rawApp = args.app_name || args.package_name || args.app || '';
|
||
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : "应用";
|
||
const rawAction = args.action ? String(args.action).toLowerCase() : '';
|
||
const verb = rawAction === 'launch' ? "启动中" : (rawAction === 'stop' || rawAction === 'close' ? "停止中" : "管理中");
|
||
return `${verb} "${app}"`;
|
||
}
|
||
|
||
case 'wait_for_delay':
|
||
case 'wait_delay':
|
||
case 'wait': {
|
||
const delay = args.delay_seconds || args.seconds || args.delay || args.duration;
|
||
return delay ? `等待 ${delay} second${Number(delay) > 1 ? 's' : ''}...` : "正在等待…";
|
||
}
|
||
case 'wait_for_text': {
|
||
const text = args.text || args.target_text || '';
|
||
return text ? `等待文本 "${text}" 出现在屏幕上` : "正在等待屏幕出现文本";
|
||
}
|
||
|
||
case 'input_text':
|
||
case 'input': {
|
||
const text = args.text || args.input_text || '';
|
||
return text ? `输入文本 "${text}" 至输入框` : "正在向输入框输入文本";
|
||
}
|
||
case 'focus_and_input_text': {
|
||
const text = args.text || args.input_text || '';
|
||
return text ? `聚焦输入框并输入 "${text}"` : "正在聚焦输入框并输入文本";
|
||
}
|
||
case 'focus_and_clear_text':
|
||
return "正在聚焦输入框并清空文本";
|
||
|
||
case 'click':
|
||
case 'tap': {
|
||
const target = args.target_text || args.text || args.query || '';
|
||
return target ? `点击 "${target}"` : "正在点击屏幕元素";
|
||
}
|
||
case 'click_sequence':
|
||
return "正在执行连续点击";
|
||
case 'long_press': {
|
||
const target = args.target_text || args.text || '';
|
||
return target ? `长按 "${target}"` : "正在长按屏幕元素";
|
||
}
|
||
case 'swipe': {
|
||
const dir = args.action || args.direction || '';
|
||
return dir ? `滑动 ${String(dir).toUpperCase()} 在屏幕上` : "正在滑动屏幕";
|
||
}
|
||
case 'press_key': {
|
||
const key = args.key || args.keycode || '';
|
||
return key ? `按键 ${String(key).toUpperCase()}` : "正在按下实体按键";
|
||
}
|
||
|
||
case 'save_note':
|
||
return isFirstSaveNote ? "正在创建笔记" : "正在保存笔记";
|
||
case 'read_note':
|
||
return "正在读取笔记";
|
||
case 'list_notes':
|
||
return "正在浏览已保存笔记";
|
||
case 'update_note':
|
||
return "正在更新笔记";
|
||
case 'append_note':
|
||
return "正在更新笔记";
|
||
|
||
case 'object_detection': {
|
||
const q = Array.isArray(args.queries) ? args.queries.join(', ') : (args.queries || '');
|
||
return q ? `Locating 在屏幕上: "${q}"` : "正在定位屏幕元素";
|
||
}
|
||
case 'ask_explorer': {
|
||
const query = args.query || args.prompt || '';
|
||
return query ? `Searching 在屏幕上: "${query}"` : "正在搜索屏幕";
|
||
}
|
||
case 'report_failure_analysis': {
|
||
const reason = args.reason || args.analysis || '';
|
||
return reason ? `排查问题：${reason}` : "正在排查执行问题";
|
||
}
|
||
case 'run_adb_command':
|
||
case 'run_short_adb_command': {
|
||
const cmd = args.command || args.cmd || '';
|
||
return cmd ? `运行命令：${cmd}` : "正在运行系统命令";
|
||
}
|
||
case 'search_logs':
|
||
case 'read_logs': {
|
||
const q = args.query || args.filter || '';
|
||
return q ? `搜索日志："${q}"` : "正在分析系统日志";
|
||
}
|
||
case 'log_analyzer':
|
||
case 'output_analyzer':
|
||
return "正在分析日志";
|
||
case 'diagnoser':
|
||
case 'diagnose':
|
||
return "正在诊断问题";
|
||
case 'video_analyzer':
|
||
case 'video_analyzer_pure':
|
||
return "正在分析屏幕录像";
|
||
case 'extract_segment_metadata': {
|
||
const start = args.start_time !== undefined ? `${args.start_time}s` : '';
|
||
const end = args.end_time !== undefined ? `${args.end_time}s` : '';
|
||
const range = (start && end) ? ` (${start} - ${end})` : (start ? ` (from ${start})` : '');
|
||
return `截取录屏片段${range}`;
|
||
}
|
||
case 'spawn_sub_agent': {
|
||
const q = args.specific_query || args.query || args.prompt || '';
|
||
return q ? `分析录屏："${q}"` : "正在调用子智能体分析录像";
|
||
}
|
||
case 'analyze_audio_only': {
|
||
const q = args.specific_query || args.query || '';
|
||
return q ? `Analyzing audio track: "${q}"` : "正在分析录像音轨";
|
||
}
|
||
case 'search_history': {
|
||
const q = args.query || '';
|
||
const range = Array.isArray(args.step_range) && args.step_range.length
|
||
? ` in steps ${args.step_range[0]}–${args.step_range[args.step_range.length - 1]}`
|
||
: '';
|
||
return q ? `Searching execution history for "${q}"${range}` : `Searching execution history${range}`;
|
||
}
|
||
case 'replay_steps': {
|
||
const n = args.start_step;
|
||
const end = args.end_step;
|
||
if (n !== undefined && n !== '' && end !== undefined && end !== null && end !== '' && String(end) !== String(n)) {
|
||
return `Reviewing steps ${n}–${end}`;
|
||
}
|
||
return n !== undefined && n !== '' ? `Reviewing step ${n}` : "正在查看步骤详情";
|
||
}
|
||
case 'get_step_screenshot': {
|
||
const n = args.step_number;
|
||
const variant = String(args.which || '').toLowerCase();
|
||
if (variant === 'overlay') {
|
||
return n !== undefined && n !== '' ? `Looking at where step ${n}'s action landed` : "正在查看操作位置";
|
||
}
|
||
const which = variant === 'post' ? 'after' : 'before';
|
||
return n !== undefined && n !== '' ? `Looking at the screen ${which} step ${n}` : "正在查看步骤截图";
|
||
}
|
||
case 'probe_device': {
|
||
const kind = args.kind ? String(args.kind).replace(/_/g, ' ') : '';
|
||
return kind ? `Reading ${kind} from the device` : "正在读取设备状态";
|
||
}
|
||
case 'outputter':
|
||
case 'output_synthesis':
|
||
return "正在生成输出报告";
|
||
case 'web_search': {
|
||
const q = args.query || '';
|
||
return q ? `Searching web for "${q}"` : "正在搜索网页";
|
||
}
|
||
case 'read_url':
|
||
return "正在读取网页";
|
||
case 'compress_history':
|
||
return getCompressionLabel(tool);
|
||
|
||
default:
|
||
return `Executing ${cleanName.split('_').map((w: string) => w.charAt(0).toUpperCase() + w.slice(1)).join(' ')}`;
|
||
}
|
||
}
|
||
|
||
function formatTokenFigure(value: any): string {
|
||
const n = Number(value);
|
||
if (!Number.isFinite(n) || n < 0) return '';
|
||
if (n < 1000) return `${Math.round(n)}`;
|
||
return `${(n / 1000).toFixed(1).replace(/\.0$/, '')}k`;
|
||
}
|
||
|
||
/** Format compression progress and token usage for the timeline. */
|
||
export function getCompressionLabel(tool: any): string {
|
||
const args = getToolArgs(tool) || {};
|
||
const start = Number(args.start_step);
|
||
const end = Number(args.end_step);
|
||
const hasRange = Number.isFinite(start) && Number.isFinite(end) && start > 0 && end > 0;
|
||
const range = hasRange
|
||
? (start === end ? `step ${start}` : `steps ${start}–${end}`)
|
||
: "之前的步骤";
|
||
const rangeCapitalized = range.charAt(0).toUpperCase() + range.slice(1);
|
||
const status = String(tool?.status || '').toLowerCase();
|
||
const phase = getCompressionPhase(tool);
|
||
|
||
if (status === 'failed' || phase === 'failed') {
|
||
return `Couldn't condense ${range} yet; keeping the full record and retrying later`;
|
||
}
|
||
if (status !== 'success') {
|
||
const note = String(args.note || '').toLowerCase();
|
||
if (note === 'retrying') return `Retrying the memory summary for ${range}…`;
|
||
if (note === 'held' || phase === 'ready') {
|
||
const swapAt = Number(args.swap_at_tokens);
|
||
const heldContext = Number(args.context_tokens);
|
||
const heldParts = [`Short memory for ${range} is ready; keeping the full record until working memory fills up`];
|
||
if (heldContext > 0 && swapAt > 0) {
|
||
heldParts.push(`≈ ${formatTokenFigure(heldContext)} of ${formatTokenFigure(swapAt)} tokens`);
|
||
}
|
||
return heldParts.join(' · ');
|
||
}
|
||
return `Condensing ${range} into a short memory to free up room…`;
|
||
}
|
||
|
||
const parts: string[] = [];
|
||
const source = Number(args.source_tokens);
|
||
const summary = Number(args.summary_tokens);
|
||
if (args.forced) {
|
||
parts.push(`${rangeCapitalized} condensed into a recap to free up memory`);
|
||
} else {
|
||
parts.push(`${rangeCapitalized} condensed into a short memory`);
|
||
if (source > 0 && summary > 0) {
|
||
const factor = source / summary;
|
||
const factorText = factor >= 2 ? ` (${Math.round(factor)}× smaller)` : '';
|
||
parts.push(`${formatTokenFigure(source)} → ${formatTokenFigure(summary)} tokens${factorText}`);
|
||
}
|
||
}
|
||
const context = Number(args.context_tokens);
|
||
const budget = Number(args.context_budget);
|
||
if (context > 0) {
|
||
parts.push(budget > 0
|
||
? `working memory ≈ ${formatTokenFigure(context)} of ${formatTokenFigure(budget)} tokens`
|
||
: `working memory ≈ ${formatTokenFigure(context)} tokens`);
|
||
}
|
||
return parts.join(' · ');
|
||
}
|
||
|
||
/**
|
||
* Plain-language phases of one compress_history line. Mirrors
|
||
* COMPRESSION_PHASES in artemis/memory/chunking.py: the backend writes
|
||
* `args.phase`; the trace `status` (running/success/failed) is left alone.
|
||
*/
|
||
export type CompressionPhase = 'summarizing' | 'ready' | 'applied' | 'failed';
|
||
|
||
const COMPRESSION_PHASES: readonly CompressionPhase[] = ['summarizing', 'ready', 'applied', 'failed'];
|
||
|
||
export function isCompressionTool(tool: any): boolean {
|
||
const name = String(tool?.name || '').toLowerCase().replace(/^(_)?exec_/, '');
|
||
return name === 'compress_history';
|
||
}
|
||
|
||
/**
|
||
* Phase of a compress_history line. Prefers the backend's `args.phase`;
|
||
* older traces without it fall back to status + note (`held` ⇒ ready).
|
||
*/
|
||
export function getCompressionPhase(tool: any): CompressionPhase {
|
||
const args = getToolArgs(tool) || {};
|
||
const declared = String(args.phase || '').toLowerCase() as CompressionPhase;
|
||
if (COMPRESSION_PHASES.includes(declared)) return declared;
|
||
const status = String(tool?.status || '').toLowerCase();
|
||
if (status === 'failed') return 'failed';
|
||
if (status === 'success') return 'applied';
|
||
if (String(args.note || '').toLowerCase() === 'held') return 'ready';
|
||
return 'summarizing';
|
||
}
|
||
|
||
/** One short, non-technical label per compression phase. */
|
||
export function getCompressionPhaseLabel(tool: any): string {
|
||
switch (getCompressionPhase(tool)) {
|
||
case 'ready':
|
||
return "摘要已就绪，上下文接近容量上限时替换原文";
|
||
case 'applied':
|
||
return "已使用摘要替换此段记录";
|
||
case 'failed':
|
||
return "摘要失败，保留完整记录";
|
||
default:
|
||
return "正在总结此段记录";
|
||
}
|
||
}
|
||
|
||
/**
|
||
* A ready-but-held summary is waiting, not working: the card must not keep
|
||
* its running pulse even though the trace status is still `running`.
|
||
*/
|
||
export function isCompressionWaiting(tool: any): boolean {
|
||
return isCompressionTool(tool) && getCompressionPhase(tool) === 'ready';
|
||
}
|
||
|
||
/**
|
||
* Get Material Symbol icon for tools
|
||
*/
|
||
export function getToolIcon(tool: any): string {
|
||
if (!tool || !tool.name) return 'settings';
|
||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||
switch (name) {
|
||
case 'click':
|
||
case 'tap':
|
||
return 'ads_click';
|
||
case 'click_sequence':
|
||
case 'long_press':
|
||
return 'touch_app';
|
||
case 'input_text':
|
||
case 'input':
|
||
return 'keyboard';
|
||
case 'swipe':
|
||
return 'swipe';
|
||
case 'press_key':
|
||
return 'keyboard_tab';
|
||
case 'manage_app':
|
||
return 'open_in_new';
|
||
case 'wait_for_delay':
|
||
return 'timer';
|
||
case 'wait_for_text':
|
||
return 'hourglass_empty';
|
||
case 'object_detection':
|
||
return 'search';
|
||
case 'ask_explorer':
|
||
return 'search';
|
||
case 'report_failure_analysis':
|
||
return 'assessment';
|
||
case 'run_adb_command':
|
||
case 'run_short_adb_command':
|
||
return 'terminal';
|
||
case 'web_search':
|
||
return 'travel_explore';
|
||
case 'read_url':
|
||
return 'language';
|
||
case 'search_logs':
|
||
case 'read_logs':
|
||
case 'log_analyzer':
|
||
case 'output_analyzer':
|
||
return 'receipt_long';
|
||
case 'diagnoser':
|
||
case 'diagnose':
|
||
return 'medical_services';
|
||
case 'video_analyzer':
|
||
case 'video_analyzer_pure':
|
||
return 'video_camera_back';
|
||
case 'extract_segment_metadata':
|
||
return 'crop';
|
||
case 'spawn_sub_agent':
|
||
return 'smart_toy';
|
||
case 'analyze_audio_only':
|
||
return 'graphic_eq';
|
||
case 'search_history':
|
||
return 'find_in_page';
|
||
case 'replay_steps':
|
||
return 'manage_search';
|
||
case 'get_step_screenshot':
|
||
return 'image_search';
|
||
case 'probe_device':
|
||
return 'sensors';
|
||
case 'outputter':
|
||
case 'output_synthesis':
|
||
return 'assignment_turned_in';
|
||
case 'compress_history':
|
||
return 'compress';
|
||
default:
|
||
return 'build';
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Get formatted title for a tool call card
|
||
*/
|
||
export function getToolTitle(tool: any): string {
|
||
if (!tool || !tool.name) return "工具调用";
|
||
const cleanName = tool.name.replace(/^(_)?exec_/, '');
|
||
const name = cleanName.toLowerCase();
|
||
switch (name) {
|
||
case 'click':
|
||
case 'tap':
|
||
return "点击元素";
|
||
case 'click_sequence':
|
||
return "执行连续点击";
|
||
case 'long_press':
|
||
return "长按元素";
|
||
case 'input_text':
|
||
case 'input':
|
||
return "输入文本";
|
||
case 'swipe':
|
||
case 'scroll': {
|
||
const args = getToolArgs(tool);
|
||
const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '');
|
||
if (dir && isPureDirectionString(dir)) return `滑动屏幕（${String(dir).toUpperCase()})`;
|
||
return "滑动屏幕";
|
||
}
|
||
case 'drag':
|
||
case 'drag_and_drop':
|
||
return "拖动屏幕";
|
||
case 'press_key':
|
||
return "按下实体按键";
|
||
case 'manage_app':
|
||
case 'launch_app': {
|
||
const args = getToolArgs(tool);
|
||
const rawAction = args.action ? String(args.action).toLowerCase() : '';
|
||
if (rawAction === 'launch') return "启动应用";
|
||
if (rawAction === 'stop' || rawAction === 'close') return "停止应用";
|
||
return "管理应用";
|
||
}
|
||
case 'wait_for_delay':
|
||
case 'wait_delay':
|
||
return "等待";
|
||
case 'wait_for_text':
|
||
return "等待文本";
|
||
case 'object_detection':
|
||
return "定位元素";
|
||
case 'ask_explorer':
|
||
return "搜索屏幕";
|
||
case 'report_failure_analysis':
|
||
return "排查问题";
|
||
case 'run_adb_command':
|
||
case 'run_short_adb_command':
|
||
return "运行系统命令";
|
||
case 'web_search':
|
||
return "网页搜索";
|
||
case 'read_url':
|
||
return "读取网页";
|
||
case 'search_logs':
|
||
case 'read_logs':
|
||
return "搜索日志";
|
||
case 'log_analyzer':
|
||
case 'output_analyzer':
|
||
return "分析日志";
|
||
case 'diagnoser':
|
||
case 'diagnose':
|
||
return "诊断问题";
|
||
case 'video_analyzer':
|
||
case 'video_analyzer_pure':
|
||
return "分析屏幕录像";
|
||
case 'extract_segment_metadata':
|
||
return "裁剪屏幕录像";
|
||
case 'spawn_sub_agent':
|
||
return "委派视频分析";
|
||
case 'analyze_audio_only':
|
||
return "分析音轨";
|
||
default:
|
||
return cleanName.replace(/_/g, ' ').replace(/\b\w/g, (c: string) => c.toUpperCase());
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Join per-point self-described targets (click_sequence `target_descriptions`)
|
||
* into one label, chained with ' → ' like the coordinate rendering.
|
||
* Returns '' when the value is missing or holds no usable text.
|
||
*/
|
||
export function joinTargetDescriptions(descriptions: any): string {
|
||
if (!Array.isArray(descriptions)) return '';
|
||
const parts = descriptions
|
||
.map(d => (d === null || d === undefined) ? '' : String(d).trim())
|
||
.filter(d => d.length > 0);
|
||
return parts.join(' → ');
|
||
}
|
||
|
||
/**
|
||
* Get target text description for tools
|
||
*/
|
||
export function getToolTargetText(tool: any): string {
|
||
if (!tool || !tool.name) return '';
|
||
const args = getToolArgs(tool);
|
||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||
if (name === 'manage_app' || name === 'launch_app') {
|
||
const rawApp = args.app_name || args.package_name || args.app || '';
|
||
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : '';
|
||
const rawAct = args.action ? String(args.action).toLowerCase() : '';
|
||
const act = rawAct ? (rawAct.charAt(0).toUpperCase() + rawAct.slice(1)) : '';
|
||
if (act && app) {
|
||
return `${act} ${app}`;
|
||
}
|
||
return app || act || '';
|
||
}
|
||
if (name === 'swipe' && typeof args.action === 'string' && isPureDirectionString(args.action)) {
|
||
return args.action;
|
||
}
|
||
if (name === 'wait_for_text') {
|
||
return args.text || '';
|
||
}
|
||
if (name === 'object_detection') {
|
||
if (Array.isArray(args.queries)) return args.queries.join(', ');
|
||
return args.queries || '';
|
||
}
|
||
if (name === 'ask_explorer' || name === 'web_search') {
|
||
return args.query || args.prompt || '';
|
||
}
|
||
if (name === 'read_url') {
|
||
return args.url || '';
|
||
}
|
||
if (name === 'search_logs' || name === 'read_logs') {
|
||
return args.query || args.filter || '';
|
||
}
|
||
if (name === 'report_failure_analysis') {
|
||
return args.status || args.reason || args.analysis || '';
|
||
}
|
||
if (name === 'click_sequence' || name === 'tap_sequence') {
|
||
// One self-described target per point, chained like getToolCoords does.
|
||
const joined = joinTargetDescriptions(args.target_descriptions);
|
||
if (joined) return joined;
|
||
}
|
||
if (args.target && typeof args.target === 'string') {
|
||
return args.target;
|
||
}
|
||
if (args.target && typeof args.target === 'number') {
|
||
return `Element #${args.target}`;
|
||
}
|
||
if (args.index !== undefined) {
|
||
return `Element #${args.index}`;
|
||
}
|
||
return args.target_text || args.target_description || args.target_class || args.element || args.element_text || (name !== 'input_text' ? args.text : '') || '';
|
||
}
|
||
|
||
/**
|
||
* Get input label for tools
|
||
*/
|
||
export function getToolInputLabel(tool: any): string {
|
||
if (!tool || !tool.name) return "输入";
|
||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||
if (name === 'wait_for_delay' || name === 'wait_delay' || name === 'delay' || name === 'wait') {
|
||
return "耗时";
|
||
}
|
||
if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') {
|
||
const args = getToolArgs(tool);
|
||
const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '');
|
||
if (dir && isPureDirectionString(dir)) {
|
||
return "方向";
|
||
}
|
||
return "输入";
|
||
}
|
||
if (name === 'press_key' || name === 'press_home' || name === 'press_back') {
|
||
return "按键";
|
||
}
|
||
if (name === 'input_text' || name === 'input') {
|
||
return "输入文本";
|
||
}
|
||
return "输入";
|
||
}
|
||
|
||
/**
|
||
* Get input value for tools
|
||
*/
|
||
export function getToolInputText(tool: any): string {
|
||
if (!tool || !tool.name) return '';
|
||
const args = getToolArgs(tool);
|
||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||
if (name === 'press_key') {
|
||
return args.key || args.keycode || '';
|
||
}
|
||
if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') {
|
||
const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '');
|
||
if (dir && isPureDirectionString(dir)) {
|
||
return String(dir).toUpperCase();
|
||
}
|
||
return '';
|
||
}
|
||
if (name === 'wait_for_delay' || name === 'wait_delay') {
|
||
return args.time_in_ms ? `${args.time_in_ms}ms` : (args.delay_ms ? `${args.delay_ms}ms` : (args.time ? `${args.time}` : ''));
|
||
}
|
||
if (name === 'long_press' && args.duration) {
|
||
return `Duration: ${args.duration}ms`;
|
||
}
|
||
if (name === 'input_text' || name === 'input') {
|
||
return args.text || args.input_text || '';
|
||
}
|
||
return args.input_text || '';
|
||
}
|
||
|
||
/**
|
||
* Get coordinates string representation for tools
|
||
*/
|
||
export function getToolCoords(tool: any): string {
|
||
if (!tool || !tool.name) return '';
|
||
const args = getToolArgs(tool);
|
||
const cleanName = tool.name.replace(/^(_)?exec_/, '');
|
||
const name = cleanName.toLowerCase();
|
||
|
||
// Check sequence first if action is sequence or sequence arg is present
|
||
const isSequenceAction = name === 'click_sequence' || name === 'tap_sequence' || Boolean(args.sequence || args.normalized_sequence);
|
||
const rawSeq = args.normalized_sequence || args.sequence || args.targets || (Array.isArray(args.coordinates) && Array.isArray(args.coordinates[0]) ? args.coordinates : null);
|
||
const seqPoints = parseSequenceCoordinates(rawSeq);
|
||
if (seqPoints && seqPoints.length > 0 && (isSequenceAction || seqPoints.length > 1)) {
|
||
return seqPoints.map(pt => `[${pt[0]}, ${pt[1]}]`).join(' → ');
|
||
}
|
||
|
||
// Check normalized start and end first
|
||
const normStart = extractNumbersFromCoordinateValue(args.normalized_start_coordinates || args.start_coordinates || args.start);
|
||
const normEnd = extractNumbersFromCoordinateValue(args.normalized_end_coordinates || args.end_coordinates || args.end);
|
||
if (normStart && normEnd && normStart.length === 2 && normEnd.length === 2) {
|
||
return `[${normStart[0]}, ${normStart[1]}] → [${normEnd[0]}, ${normEnd[1]}]`;
|
||
}
|
||
|
||
const coords = extractNumbersFromCoordinateValue(args.normalized_coordinates) ||
|
||
extractNumbersFromCoordinateValue(args.coordinates) ||
|
||
extractNumbersFromCoordinateValue(args.target) ||
|
||
extractNumbersFromCoordinateValue(args.action) ||
|
||
extractNumbersFromCoordinateValue(args.gesture) ||
|
||
extractNumbersFromCoordinateValue(args.point);
|
||
|
||
if (coords && Array.isArray(coords)) {
|
||
if (coords.length === 4) {
|
||
return `[${coords[0]}, ${coords[1]}] → [${coords[2]}, ${coords[3]}]`;
|
||
}
|
||
if (coords.length === 2) {
|
||
return `[${coords[0]}, ${coords[1]}]`;
|
||
}
|
||
return coords.join(', ');
|
||
}
|
||
|
||
if (args.x !== undefined && args.y !== undefined) {
|
||
return `[${args.x}, ${args.y}]`;
|
||
}
|
||
return '';
|
||
}
|
||
|
||
/**
|
||
* Get analysis / reasoning text for tools
|
||
*/
|
||
export function getToolAnalysisText(tool: any): string {
|
||
if (!tool || !tool.name) return '';
|
||
const args = getToolArgs(tool);
|
||
return args.analysis || args.reasoning || args.summary || '';
|
||
}
|
||
|
||
/**
|
||
* Check if tool is ADB command
|
||
*/
|
||
export function isAdbCommandTool(tool: any): boolean {
|
||
if (!tool || !tool.name) return false;
|
||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||
return name === 'run_adb_command' || name === 'run_short_adb_command';
|
||
}
|
||
|
||
/**
|
||
* Get ADB command line string
|
||
*/
|
||
export function getAdbCommandLine(tool: any): string {
|
||
const args = getToolArgs(tool);
|
||
return args.CommandLine || '';
|
||
}
|
||
|
||
/**
|
||
* Get ADB working directory
|
||
*/
|
||
export function getAdbCwd(tool: any): string {
|
||
const args = getToolArgs(tool);
|
||
return args.Cwd || '';
|
||
}
|
||
|
||
/**
|
||
* Get ADB requested terminal id
|
||
*/
|
||
export function getAdbTerminalId(tool: any): string {
|
||
const args = getToolArgs(tool);
|
||
return args.RequestedTerminalID || '';
|
||
}
|
||
|
||
/**
|
||
* Get fallback generic details for unspecified tools
|
||
*/
|
||
export function getToolGenericDetails(tool: any): string {
|
||
if (!tool) return '';
|
||
if (isAdbCommandTool(tool)) {
|
||
return '';
|
||
}
|
||
if (getToolTargetText(tool) || getToolInputText(tool) || getToolCoords(tool) || getToolAnalysisText(tool)) {
|
||
return '';
|
||
}
|
||
const args = getToolArgs(tool);
|
||
if (!args || typeof args !== 'object') return '';
|
||
const ignoredKeys = new Set(['state', 'controller', 'ctx', 'session_id', 'step_id', 'trace_id', 'parent_trace_id', 'times', 'delay_ms']);
|
||
const entries = Object.entries(args).filter(([k, v]) => !ignoredKeys.has(k) && v !== undefined && v !== null && v !== '');
|
||
if (entries.length === 0) return '';
|
||
return entries.map(([k, v]) => `${k}: ${typeof v === 'object' ? JSON.stringify(v) : v}`).join(', ');
|
||
}
|
||
|
||
/**
|
||
* Check if tool execution failed
|
||
*/
|
||
export function isToolFailed(tool: any): boolean {
|
||
if (!tool) return false;
|
||
if (tool.status === 'failed' || tool.status === 'error') return true;
|
||
const args = getToolArgs(tool);
|
||
if (args.status === 'failed' || args.status === 'error' || args.status === 'cannot_fix') return true;
|
||
return false;
|
||
}
|
||
|
||
/**
|
||
* Get error message for a failed tool
|
||
*/
|
||
export function getToolErrorMessage(tool: any): string {
|
||
if (!tool) return "工具失败";
|
||
const args = getToolArgs(tool);
|
||
if (args.status === 'cannot_fix') return 'Status: cannot_fix';
|
||
if (args.error || args.message || args.failure_reason) {
|
||
return args.error || args.message || args.failure_reason;
|
||
}
|
||
if (tool.error || tool.message) return tool.error || tool.message;
|
||
return "操作失败";
|
||
}
|
||
|
||
/**
|
||
* Extract extra parameters for generic tools
|
||
*/
|
||
export function extractToolExtraParams(toolData: any, cache?: WeakMap<any, ActionParam[]>): ActionParam[] {
|
||
if (!toolData) return [];
|
||
const payload = toolData.payload || toolData.args || toolData;
|
||
if (!payload || typeof payload !== 'object') return [];
|
||
if (cache && cache.has(payload)) {
|
||
return cache.get(payload)!;
|
||
}
|
||
|
||
const standardKeys = new Set([
|
||
'action', 'name', 'type', 'target_text', 'target_description', 'target_descriptions',
|
||
'text', 'input_text', 'target',
|
||
'coordinates', 'coords', 'target_bounds', 'bounds', 'target_resource_id',
|
||
'resource_id', 'target_class', 'class_name', 'normalized_coordinates',
|
||
'pre_image_name', 'post_image_name', 'pre_screenshot', 'post_screenshot',
|
||
'before_screenshot', 'after_screenshot', 'status', 'success', 'timestamp',
|
||
'created_at', 'start_time', 'execution_id', 'controller', 'agent', 'session_id', 'step_id',
|
||
'app_name', 'package_name', 'app', 'key', 'keycode', 'time_in_ms', 'delay_ms', 'delay_seconds', 'duration',
|
||
'args', 'kwargs', 'parameters', 'extra_params', 'payload', 'trace_id', 'result', 'error', 'analysis', 'details', 'command', 'cwd', 'terminal_id'
|
||
]);
|
||
|
||
const result: ActionParam[] = [];
|
||
|
||
const rawArgs = toolData.args || toolData.Args || toolData.kwargs || toolData.parameters || toolData.payload;
|
||
let parsedArgs: any = {};
|
||
if (rawArgs) {
|
||
if (typeof rawArgs === 'object') {
|
||
parsedArgs = rawArgs;
|
||
} else if (typeof rawArgs === 'string') {
|
||
try {
|
||
parsedArgs = JSON.parse(rawArgs);
|
||
} catch {
|
||
if (!rawArgs.includes('<') && !rawArgs.includes('object at')) {
|
||
parsedArgs = { details: rawArgs };
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
const mergedObj = { ...toolData, ...(typeof payload === 'object' ? payload : {}), ...parsedArgs };
|
||
|
||
for (const [k, v] of Object.entries(mergedObj)) {
|
||
const lowerK = k.toLowerCase();
|
||
if (standardKeys.has(lowerK)) continue;
|
||
if (v === null || v === undefined || v === '') continue;
|
||
|
||
let valStr = String(v);
|
||
if (valStr.includes('object at 0x') || valStr.startsWith('<artemis.') || valStr.includes('<controller') || valStr.includes('<android_world')) continue;
|
||
|
||
if (typeof v === 'object') {
|
||
try { valStr = JSON.stringify(v); } catch { valStr = String(v); }
|
||
}
|
||
|
||
let prettyKey = k.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
|
||
if (lowerK === 'time_in_ms' || lowerK === 'delay_ms') prettyKey = "延迟";
|
||
|
||
result.push({ key: prettyKey, value: valStr });
|
||
}
|
||
|
||
if (cache) {
|
||
cache.set(payload, result);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
/**
|
||
* Check if text is genuine human thinking rather than raw JSON payload output
|
||
*/
|
||
export function isHumanThinking(text: string | null): boolean {
|
||
if (!text) return false;
|
||
let trimmed = text.trim();
|
||
if (!trimmed) return false;
|
||
|
||
// Strip markdown code blocks if wrapped: ```json ... ``` or ``` ... ```
|
||
if (trimmed.startsWith('```')) {
|
||
const match = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i);
|
||
if (match) {
|
||
trimmed = match[1].trim();
|
||
}
|
||
}
|
||
|
||
// Direct object or array start
|
||
if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
|
||
return false;
|
||
}
|
||
|
||
// Try parsing as JSON if it looks like JSON structures
|
||
try {
|
||
const parsed = JSON.parse(trimmed);
|
||
if (typeof parsed === 'object' && parsed !== null) {
|
||
return false;
|
||
}
|
||
} catch {
|
||
// Not valid standalone JSON, continue checks
|
||
}
|
||
|
||
// Check for common JSON object detection / grounding / action / tool patterns
|
||
if (/^\s*\{.*"label"\s*:.*"results"\s*:/s.test(trimmed) ||
|
||
/^\s*\{.*"point"\s*:.*"label"\s*:/s.test(trimmed) ||
|
||
/^\s*\{.*"action"\s*:/s.test(trimmed) ||
|
||
/^\s*\{.*"name"\s*:/s.test(trimmed) ||
|
||
/^\s*\{\s*"[^"]+"\s*:\s*/.test(trimmed) ||
|
||
/^\s*\[\s*\{\s*"[^"]+"\s*:\s*/.test(trimmed)) {
|
||
return false;
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
/**
|
||
* Clean up raw error messages for display in the UI
|
||
*/
|
||
export function cleanErrorMessage(rawError: any): string {
|
||
if (typeof rawError === 'object' && rawError !== null) {
|
||
if (rawError.message && typeof rawError.message === 'string') {
|
||
return cleanErrorMessage(rawError.message);
|
||
}
|
||
if (rawError.error?.message && typeof rawError.error?.message === 'string') {
|
||
return cleanErrorMessage(rawError.error.message);
|
||
}
|
||
if (rawError.detail && typeof rawError.detail === 'string') {
|
||
return cleanErrorMessage(rawError.detail);
|
||
}
|
||
try {
|
||
return JSON.stringify(rawError);
|
||
} catch {
|
||
return String(rawError);
|
||
}
|
||
}
|
||
|
||
const errorStr = String(rawError).trim();
|
||
if (!errorStr) return "未知错误";
|
||
|
||
// 1. Try regex extraction for "message": "..."
|
||
const doubleQuoteMsgMatch = errorStr.match(/"message"\s*:\s*"((?:[^"\\]|\\.)*)"/i);
|
||
if (doubleQuoteMsgMatch && doubleQuoteMsgMatch[1]) {
|
||
const unescaped = doubleQuoteMsgMatch[1].replace(/\\"/g, '"').replace(/\\n/g, ' ').trim();
|
||
if (unescaped) return unescaped;
|
||
}
|
||
|
||
// 2. Try regex extraction for 'message': '...'
|
||
const singleQuoteMsgMatch = errorStr.match(/'message'\s*:\s*['"]((?:[^'\\]|\\.)*)['"]/i);
|
||
if (singleQuoteMsgMatch && singleQuoteMsgMatch[1]) {
|
||
const unescaped = singleQuoteMsgMatch[1].replace(/\\'/g, "'").replace(/\\n/g, ' ').trim();
|
||
if (unescaped) return unescaped;
|
||
}
|
||
|
||
// 3. Try parsing JSON substrings inside the error string
|
||
const jsonMatch = errorStr.match(/\{[\s\S]*\}/);
|
||
if (jsonMatch) {
|
||
try {
|
||
const normalizedJsonStr = jsonMatch[0]
|
||
.replace(/'/g, '"')
|
||
.replace(/True/g, 'true')
|
||
.replace(/False/g, 'false')
|
||
.replace(/None/g, 'null');
|
||
const parsed = JSON.parse(normalizedJsonStr);
|
||
if (parsed?.message) return String(parsed.message);
|
||
if (parsed?.error?.message) return String(parsed.error.message);
|
||
} catch {
|
||
// Ignore
|
||
}
|
||
}
|
||
|
||
// 4. Fallback: truncate at trace dump headers or return clean message
|
||
let fallback = errorStr;
|
||
if (fallback.includes('=== Source Location Trace')) {
|
||
fallback = fallback.split('=== Source Location Trace')[0];
|
||
}
|
||
if (fallback.includes('[type.googleapis.com')) {
|
||
fallback = fallback.split('[type.googleapis.com')[0];
|
||
}
|
||
fallback = fallback
|
||
.replace(/^Pre-execution validation failed:\s*/i, '')
|
||
.replace(/^Pixel-level validation failed:\s*/i, '')
|
||
.replace(/^Execution error:\s*/i, '')
|
||
.replace(/^ServerError:\s*/i, '')
|
||
// LLM wrappers may add this prefix more than once. It is context, not the
|
||
// actual provider reason, and an empty prefix must not be shown as though
|
||
// it were a useful error message.
|
||
.replace(/^(?:LLM\s+(?:Request\s+)?Error\s*:\s*)+/i, '')
|
||
.trim();
|
||
|
||
return fallback || "未知错误";
|
||
}
|