feat: introduce MCP server, cross-platform runtime, and Showcase UI reorganization
- Showcase UI Reorganization: * Move web UI dashboard from root directory into apps/showcase_ui/ for cleaner project structure * Add 'artemis ui' CLI command (artemis/interfaces/cli/commands/ui.py) and launcher scripts for the interactive showcase console - Standalone MCP Server & Notifiers (mcp_server/): * Implement dedicated MCP server package with mobile automation tools (device_state, inspect_trace, task_manager, task_runner) * Add multi-channel notifiers (AgentAPI, Desktop notifications, Webhooks, File output, and Composite routing) * Introduce background task runner and trace store for autonomous mobile execution monitoring - Cross-Platform Runtime & Toolchain Support: * Add artemis/platform/ with OS-specific implementations for Linux, macOS (Darwin), and Windows * Add artemis/runtime/supervisor.py for resilient process and agent runtime supervision * Add artemis/toolchain/ with dependency resolution and SDK descriptors * Add cross-platform launcher scripts (start.sh, start.bat) and installer scripts (scripts/install_deps.*) - Core Enhancements & Diagnostics: * Enhance coordinate normalization, OCR API fallback, and perception pipeline * Add credential validation (credentials_validator.py) and system diagnostics * Update bilingual documentation (README.md, README_CN.md) and architecture diagrams - Test Suite Expansion: * Add integration tests for cloud gateway, remote ADB, and Artemis login * Add unit tests covering core diagnostics, platform detection, toolchain, credentials validator, optional OCR, and MCP tools
This commit is contained in:
somew1nd committed
1 parent
78e478f502
commit
9a5aaf99b7
249 files changed
+37226
-9339
No files matched your search
@@ -0,0 +1,112 @@
|
||||
import {
|
||||
getStepPreImageUrl,
|
||||
getStepPostImageUrl
|
||||
} from './action-formatter.util';
|
||||
|
||||
describe('action-formatter.util screenshot chaining', () => {
|
||||
it('should chain pre/post images correctly in FailureAnalyzer multi-step recovery', () => {
|
||||
const stepData = {
|
||||
step_id: 'step-06',
|
||||
pre_image_name: '01a0e8d8cddbeb6a352e3bef1afbee0f0476a05b0fc62abb78e59c7b6aa16079',
|
||||
post_image_name: '484361b1ab4b340415e01042e3496f0edc0549c837c897ece242835d12d7d71d',
|
||||
action_taken: {
|
||||
action: 'tap',
|
||||
coordinates: [540, 720],
|
||||
timestamp: 1787077538.0
|
||||
},
|
||||
last_execution_result: {
|
||||
status: 'success',
|
||||
repair_status: 'fixed',
|
||||
execution: [
|
||||
{
|
||||
attempts: ['Pre-execution validation failed']
|
||||
}
|
||||
]
|
||||
},
|
||||
generic_tools: [
|
||||
{
|
||||
trace_id: 'tool-save-note',
|
||||
name: 'save_note',
|
||||
type: 'tool',
|
||||
timestamp: 1787077537.0,
|
||||
payload: { args: { title: 'Note 1' } }
|
||||
},
|
||||
{
|
||||
trace_id: 'agent-failure-analyzer',
|
||||
name: 'failure_analyzer',
|
||||
agent_name: 'failure_analyzer',
|
||||
type: 'agent',
|
||||
timestamp: 1787077540.0,
|
||||
payload: {
|
||||
args: {
|
||||
pre_screenshot_name: '01a0e8d8cddbeb6a352e3bef1afbee0f0476a05b0fc62abb78e59c7b6aa16079',
|
||||
post_screenshot_name: 'fbdf17fa7a970d2c37aa52ae4336119e76a7d13ea457811cf8aaab19b3f53843'
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
trace_id: 'action-click-1',
|
||||
name: 'click',
|
||||
type: 'action',
|
||||
agent_name: 'failure_analyzer',
|
||||
timestamp: 1787077607.0,
|
||||
payload: {
|
||||
args: { target: '[500, 300]' },
|
||||
result: {
|
||||
outcome: 'Clicked successfully',
|
||||
post_image_name: '18bd2ac49e64976f1e74e828ddae76e54da1587044b1d3fb01317dd202ff7c67'
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
trace_id: 'action-click-seq-2',
|
||||
name: 'click_sequence',
|
||||
type: 'action',
|
||||
agent_name: 'failure_analyzer',
|
||||
timestamp: 1787077631.0,
|
||||
payload: {
|
||||
args: { sequence: '[[500, 300], [876, 360]]' },
|
||||
result: {
|
||||
outcome: 'Sequence clicked successfully',
|
||||
post_image_name: '484361b1ab4b340415e01042e3496f0edc0549c837c897ece242835d12d7d71d'
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
const action1 = stepData.generic_tools[2]; // click
|
||||
const action2 = stepData.generic_tools[3]; // click_sequence
|
||||
|
||||
// Action 1: Pre should be the failed state screenshot (fbdf17fa...), Post should be Action 1's post (18bd2ac4...)
|
||||
const act1Pre = getStepPreImageUrl(stepData, action1);
|
||||
const act1Post = getStepPostImageUrl(stepData, action1);
|
||||
expect(act1Pre).toBe('/images/fbdf17fa7a970d2c37aa52ae4336119e76a7d13ea457811cf8aaab19b3f53843');
|
||||
expect(act1Post).toBe('/images/18bd2ac49e64976f1e74e828ddae76e54da1587044b1d3fb01317dd202ff7c67');
|
||||
|
||||
// Action 2: "下一次的决策前是上一次的动作后" -> Pre MUST be Action 1's post (18bd2ac4...), Post should be Action 2's post (484361b1...)
|
||||
const act2Pre = getStepPreImageUrl(stepData, action2);
|
||||
const act2Post = getStepPostImageUrl(stepData, action2);
|
||||
expect(act2Pre).toBe('/images/18bd2ac49e64976f1e74e828ddae76e54da1587044b1d3fb01317dd202ff7c67');
|
||||
expect(act2Post).toBe('/images/484361b1ab4b340415e01042e3496f0edc0549c837c897ece242835d12d7d71d');
|
||||
});
|
||||
|
||||
it('should handle standard single-action step correctly', () => {
|
||||
const stepData = {
|
||||
step_id: 'step-01',
|
||||
pre_image_name: 'pre123',
|
||||
post_image_name: 'post456',
|
||||
action_taken: {
|
||||
action: 'tap',
|
||||
coordinates: [100, 200],
|
||||
timestamp: 1000
|
||||
}
|
||||
};
|
||||
|
||||
const preUrl = getStepPreImageUrl(stepData, stepData.action_taken);
|
||||
const postUrl = getStepPostImageUrl(stepData, stepData.action_taken);
|
||||
|
||||
expect(preUrl).toBe('/images/pre123');
|
||||
expect(postUrl).toBe('/images/post456');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,767 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ActionParam } from '../core/models/stream.model';
|
||||
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util';
|
||||
import { cleanErrorMessage } from './tool-formatter.util';
|
||||
|
||||
/**
|
||||
* Safely extract action object from possible array or nested structure
|
||||
*/
|
||||
export function getActionObject(action: any): any {
|
||||
if (!action) return null;
|
||||
if (Array.isArray(action)) {
|
||||
return action.length > 0 ? action[0] : null;
|
||||
}
|
||||
return action;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the action is a user interaction action on Android
|
||||
*/
|
||||
export function isAndroidAction(action: any): boolean {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return false;
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
return [
|
||||
'tap', 'click', 'click_sequence', 'input', 'input_text', 'swipe', 'scroll', 'press_key',
|
||||
'press_home', 'press_back', 'launch_app', 'manage_app', 'open_app',
|
||||
'focus_and_input_text', 'focus_and_clear_text', 'long_press', 'long_press_on',
|
||||
'wait_for_delay', 'wait'
|
||||
].includes(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the action or tool is a task completion / submission report
|
||||
*/
|
||||
export function isReportStatusAction(action: any): boolean {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return false;
|
||||
const name = (act.name || act.action || '').toLowerCase().replace(/^(_)?(self\.)?exec_/, '');
|
||||
return name === 'report_task_status' || name === 'report_status' || name === 'submit_task_status';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get status of report_task_status (completed / failed)
|
||||
*/
|
||||
export function getReportStatusValue(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return 'completed';
|
||||
const args = act.args && typeof act.args === 'object' ? act.args : (act.payload?.args || act.payload || act);
|
||||
const status = args.status || act.status;
|
||||
if (typeof status === 'string') {
|
||||
return status.toLowerCase();
|
||||
}
|
||||
return 'completed';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get explanation of report_task_status
|
||||
*/
|
||||
export function getReportStatusExplanation(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return '';
|
||||
const args = act.args && typeof act.args === 'object' ? act.args : (act.payload?.args || act.payload || act);
|
||||
return args.explanation || args.reason || args.summary || act.explanation || act.reason || act.summary || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Material Symbol icon name for Android action
|
||||
*/
|
||||
export function getActionIcon(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return 'ads_click';
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
switch (name) {
|
||||
case 'tap':
|
||||
case 'click':
|
||||
case 'click_sequence':
|
||||
return 'ads_click';
|
||||
case 'input':
|
||||
case 'input_text':
|
||||
case 'focus_and_input_text':
|
||||
return 'keyboard';
|
||||
case 'focus_and_clear_text':
|
||||
case 'clear_text':
|
||||
return 'backspace';
|
||||
case 'swipe':
|
||||
case 'scroll':
|
||||
return 'swipe';
|
||||
case 'drag':
|
||||
case 'drag_and_drop':
|
||||
return 'drag_indicator';
|
||||
case 'press_key':
|
||||
case 'press_home':
|
||||
case 'press_back':
|
||||
return 'keyboard_tab';
|
||||
case 'launch_app':
|
||||
case 'manage_app':
|
||||
case 'open_app':
|
||||
return 'open_in_new';
|
||||
case 'long_press':
|
||||
case 'long_press_on':
|
||||
return 'touch_app';
|
||||
case 'wait_for_delay':
|
||||
case 'delay':
|
||||
case 'wait':
|
||||
return 'hourglass_empty';
|
||||
default:
|
||||
return 'settings';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get human-readable title for Android actions
|
||||
*/
|
||||
export function getActionTitle(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return 'Action';
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
switch (name) {
|
||||
case 'tap':
|
||||
case 'click':
|
||||
case 'tap_element':
|
||||
case 'click_element':
|
||||
return 'Tapping Element';
|
||||
case 'input':
|
||||
case 'input_text':
|
||||
case 'focus_and_input_text':
|
||||
return 'Entering Text';
|
||||
case 'focus_and_clear_text':
|
||||
case 'clear_text':
|
||||
return 'Clearing Text';
|
||||
case 'swipe':
|
||||
case 'scroll': {
|
||||
const actObj = getActionObject(action);
|
||||
const args = actObj?.args && typeof actObj.args === 'object' ? actObj.args : {};
|
||||
const dir = actObj?.direction || actObj?.gesture || args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '') || (typeof actObj?.action === 'string' && actObj.action !== name ? actObj.action : '');
|
||||
if (dir && isPureDirectionString(dir)) {
|
||||
return `Swiping Screen (${String(dir).toUpperCase()})`;
|
||||
}
|
||||
return 'Swiping Screen';
|
||||
}
|
||||
case 'drag':
|
||||
case 'drag_and_drop':
|
||||
return 'Dragging Screen';
|
||||
case 'press_key':
|
||||
case 'press_home':
|
||||
case 'press_back':
|
||||
return 'Pressing Hardware Key';
|
||||
case 'launch_app':
|
||||
case 'open_app':
|
||||
return 'Launching Application';
|
||||
case 'stop_app':
|
||||
case 'close_app':
|
||||
return 'Stopping Application';
|
||||
case 'manage_app': {
|
||||
const actObj = getActionObject(action);
|
||||
const actStr = (actObj?.action || actObj?.args?.action || '').toLowerCase();
|
||||
if (actStr === 'launch') return 'Launching Application';
|
||||
if (actStr === 'stop' || actStr === 'close') return 'Stopping Application';
|
||||
return 'Managing Application';
|
||||
}
|
||||
case 'wait_for_delay':
|
||||
case 'delay':
|
||||
case 'wait':
|
||||
return 'Waiting for Delay';
|
||||
case 'long_press':
|
||||
case 'long_press_on':
|
||||
return 'Long Pressing Element';
|
||||
case 'click_sequence':
|
||||
return 'Clicking Sequence';
|
||||
default:
|
||||
return name.replace(/_/g, ' ').replace(/\b\w/g, (c: string) => c.toUpperCase());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get target text or element description for Android action
|
||||
*/
|
||||
export function getActionTargetText(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return '';
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
const args = act.args && typeof act.args === 'object' ? act.args : {};
|
||||
|
||||
if (name.includes('app')) {
|
||||
const rawApp = act.app_name || act.package_name || act.app || args.app_name || args.package_name || args.app || '';
|
||||
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : '';
|
||||
const rawAction = act.action || args.action || '';
|
||||
const actionStr = rawAction ? (String(rawAction).charAt(0).toUpperCase() + String(rawAction).slice(1).toLowerCase()) : '';
|
||||
if (actionStr && app) {
|
||||
return `${actionStr} ${app}`;
|
||||
}
|
||||
return app || actionStr || '';
|
||||
}
|
||||
if (name.includes('delay') || name.includes('wait')) {
|
||||
const ms = act.time_in_ms || act.delay_ms || act.delay_seconds || args.time_in_ms || args.delay_ms || args.delay_seconds || args.duration;
|
||||
if (ms) return `${ms}ms`;
|
||||
}
|
||||
return act.target_text || act.target_class || act.element_id || args.target_text || args.element_id || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get input label for Android action
|
||||
*/
|
||||
export function getActionInputLabel(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return 'Input';
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
if (name.includes('delay') || name.includes('wait')) {
|
||||
return 'Duration';
|
||||
}
|
||||
if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') {
|
||||
const actObj = getActionObject(action);
|
||||
const args = actObj?.args && typeof actObj.args === 'object' ? actObj.args : {};
|
||||
const dir = actObj?.direction || actObj?.gesture || args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '') || (typeof actObj?.action === 'string' && actObj.action !== name ? actObj.action : '');
|
||||
if (dir && isPureDirectionString(dir)) {
|
||||
return 'Direction';
|
||||
}
|
||||
return 'Input';
|
||||
}
|
||||
if (name === 'press_key' || name.includes('key')) {
|
||||
return 'Key';
|
||||
}
|
||||
if (name === 'input_text' || name.includes('input')) {
|
||||
return 'Input Text';
|
||||
}
|
||||
return 'Input';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get input text or value for Android action
|
||||
*/
|
||||
export function getActionInputText(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return '';
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
const args = act.args && typeof act.args === 'object' ? act.args : {};
|
||||
|
||||
if (name === 'press_key' || name.includes('key')) {
|
||||
return act.key || act.keycode || args.key || args.keycode || '';
|
||||
}
|
||||
if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') {
|
||||
const dir = act.direction || act.gesture || args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '') || (typeof act.action === 'string' && act.action !== name ? act.action : '');
|
||||
if (dir && isPureDirectionString(dir)) {
|
||||
return String(dir).toUpperCase();
|
||||
}
|
||||
return '';
|
||||
}
|
||||
if (name.includes('delay') || name.includes('wait')) {
|
||||
const ms = act.time_in_ms || act.delay_ms || act.delay_seconds || args.time_in_ms || args.delay_ms || args.delay_seconds || args.duration;
|
||||
if (ms) return `${ms}ms`;
|
||||
}
|
||||
return act.text || act.input_text || args.text || args.input_text || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get coordinates string representation for Android action
|
||||
*/
|
||||
export function getActionCoords(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return '';
|
||||
const args = act.args && typeof act.args === 'object' ? act.args : {};
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
|
||||
// Check sequence first if action is sequence or sequence arg is present
|
||||
const isSequenceAction = name === 'click_sequence' || name === 'tap_sequence' || Boolean(act.sequence || args.sequence || act.normalized_sequence || args.normalized_sequence);
|
||||
const rawSeq = act.normalized_sequence || args.normalized_sequence || act.sequence || args.sequence || act.targets || args.targets || (Array.isArray(act.coordinates) && Array.isArray(act.coordinates[0]) ? act.coordinates : null) || (Array.isArray(args.coordinates) && Array.isArray(args.coordinates[0]) ? args.coordinates : null);
|
||||
const seqPoints = parseSequenceCoordinates(rawSeq);
|
||||
if (seqPoints && seqPoints.length > 0 && (isSequenceAction || seqPoints.length > 1)) {
|
||||
return seqPoints.map(pt => `[${pt[0]}, ${pt[1]}]`).join(' → ');
|
||||
}
|
||||
|
||||
// Check normalized start and end first
|
||||
const normStart = extractNumbersFromCoordinateValue(act.normalized_start_coordinates || args.normalized_start_coordinates);
|
||||
const normEnd = extractNumbersFromCoordinateValue(act.normalized_end_coordinates || args.normalized_end_coordinates);
|
||||
if (normStart && normEnd && normStart.length === 2 && normEnd.length === 2) {
|
||||
return `[${normStart[0]}, ${normStart[1]}] → [${normEnd[0]}, ${normEnd[1]}]`;
|
||||
}
|
||||
|
||||
const startCoords = extractNumbersFromCoordinateValue(act.start_coordinates || act.start_point || act.start || act.from || args.start_coordinates || args.start_point || args.start || args.from);
|
||||
const endCoords = extractNumbersFromCoordinateValue(act.end_coordinates || act.end_point || act.end || act.to || args.end_coordinates || args.end_point || args.end || args.to);
|
||||
if (startCoords && endCoords && startCoords.length === 2 && endCoords.length === 2) {
|
||||
return `[${startCoords[0]}, ${startCoords[1]}] → [${endCoords[0]}, ${endCoords[1]}]`;
|
||||
}
|
||||
|
||||
const normCoords = extractNumbersFromCoordinateValue(act.normalized_coordinates || args.normalized_coordinates);
|
||||
const coords = normCoords ||
|
||||
extractNumbersFromCoordinateValue(act.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(args.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(act.coords) ||
|
||||
extractNumbersFromCoordinateValue(args.coords) ||
|
||||
extractNumbersFromCoordinateValue(act.target) ||
|
||||
extractNumbersFromCoordinateValue(args.target) ||
|
||||
extractNumbersFromCoordinateValue(args.action) ||
|
||||
extractNumbersFromCoordinateValue(act.action) ||
|
||||
extractNumbersFromCoordinateValue(args.gesture) ||
|
||||
extractNumbersFromCoordinateValue(act.gesture);
|
||||
|
||||
if (coords && Array.isArray(coords)) {
|
||||
if (coords.length === 4) {
|
||||
return `[${coords[0]}, ${coords[1]}] → [${coords[2]}, ${coords[3]}]`;
|
||||
}
|
||||
if (coords.length === 2) {
|
||||
return `[${coords[0]}, ${coords[1]}]`;
|
||||
}
|
||||
return coords.join(', ');
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the action execution failed or encountered an execution failure/interception
|
||||
*/
|
||||
export function isActionFailed(action: any, stepData?: any): boolean {
|
||||
const act = getActionObject(action);
|
||||
if (act) {
|
||||
if (act.status === 'failed' || act.status === 'error' || act.success === false) return true;
|
||||
}
|
||||
if (stepData) {
|
||||
if (stepData.status === 'failed' || stepData.status === 'error') return true;
|
||||
if (stepData.last_execution_result) {
|
||||
const res = typeof stepData.last_execution_result === 'string'
|
||||
? (() => { try { return JSON.parse(stepData.last_execution_result); } catch { return null; } })()
|
||||
: stepData.last_execution_result;
|
||||
|
||||
if (res && typeof res === 'object') {
|
||||
if (res.status === 'failed' || res.status === 'error' || res.success === false || res.is_successful === false) {
|
||||
return true;
|
||||
}
|
||||
if (res.repair_status === 'fixed' || res.repair_status === 'failed' || res.repair_status === 'cannot_fix') {
|
||||
return true;
|
||||
}
|
||||
if (Array.isArray(res.execution) && res.execution.length > 0) {
|
||||
const firstExec = res.execution[0];
|
||||
if (firstExec && (firstExec.status === 'failed' || firstExec.status === 'error' || firstExec.error || (Array.isArray(firstExec.attempts) && firstExec.attempts.length > 0))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get failure message or reason for an action
|
||||
*/
|
||||
export function getActionErrorMessage(action: any, stepData?: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (act && (act.error || act.message || act.failure_reason)) {
|
||||
return cleanErrorMessage(act.error || act.message || act.failure_reason);
|
||||
}
|
||||
if (stepData && stepData.last_execution_result) {
|
||||
const res = typeof stepData.last_execution_result === 'string'
|
||||
? (() => { try { return JSON.parse(stepData.last_execution_result); } catch { return null; } })()
|
||||
: stepData.last_execution_result;
|
||||
|
||||
if (res && typeof res === 'object') {
|
||||
if (Array.isArray(res.execution) && res.execution.length > 0) {
|
||||
const firstExec = res.execution[0];
|
||||
if (firstExec) {
|
||||
if (Array.isArray(firstExec.attempts) && firstExec.attempts.length > 0) {
|
||||
return cleanErrorMessage(firstExec.attempts[0]);
|
||||
}
|
||||
if (firstExec.error || firstExec.failure_reason || firstExec.message) {
|
||||
return cleanErrorMessage(firstExec.error || firstExec.failure_reason || firstExec.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (res.error || res.message || res.failure_reason) {
|
||||
return cleanErrorMessage(res.error || res.message || res.failure_reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
return 'Action Failed';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get bounds string representation for Android action
|
||||
*/
|
||||
export function getActionBounds(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return '';
|
||||
const bounds = act.target_bounds || act.bounds || (act.args && (act.args.target_bounds || act.args.bounds));
|
||||
if (Array.isArray(bounds)) {
|
||||
return bounds.join(', ');
|
||||
}
|
||||
return bounds ? String(bounds) : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get resource id for Android action target
|
||||
*/
|
||||
export function getActionResourceId(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return '';
|
||||
return act.target_resource_id || act.resource_id || (act.args && (act.args.target_resource_id || act.args.resource_id)) || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get class name for Android action target
|
||||
*/
|
||||
export function getActionClass(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return '';
|
||||
return act.target_class || act.class_name || (act.args && (act.args.target_class || act.args.class_name)) || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract extra parameter key-value pairs for Android actions
|
||||
*/
|
||||
export function extractActionExtraParams(action: any, cache?: WeakMap<any, ActionParam[]>): ActionParam[] {
|
||||
const act = getActionObject(action);
|
||||
if (!act || typeof act !== 'object') return [];
|
||||
if (cache && cache.has(act)) {
|
||||
return cache.get(act)!;
|
||||
}
|
||||
|
||||
const standardKeys = new Set([
|
||||
'action', 'name', 'type', 'target_text', 'text', 'input_text', 'target',
|
||||
'coordinates', 'coords', 'target_bounds', 'bounds', 'target_resource_id',
|
||||
'resource_id', 'target_class', 'class_name', 'normalized_coordinates',
|
||||
'normalized_start_coordinates', 'normalized_end_coordinates',
|
||||
'start_coordinates', 'end_coordinates', 'start', 'end', 'from', 'to',
|
||||
'pre_image_name', 'post_image_name', 'pre_screenshot', 'post_screenshot',
|
||||
'before_screenshot', 'after_screenshot', 'status', 'success', 'timestamp',
|
||||
'created_at', 'start_time', 'execution_id', 'controller', 'agent', 'session_id', 'step_id',
|
||||
'app_name', 'package_name', 'app', 'key', 'keycode', 'time_in_ms', 'delay_ms', 'delay_seconds', 'duration',
|
||||
'args', 'kwargs', 'parameters', 'extra_params', 'direction', 'gesture'
|
||||
]);
|
||||
|
||||
const result: ActionParam[] = [];
|
||||
|
||||
const rawArgs = act.args || act.Args || act.kwargs || act.parameters;
|
||||
let parsedArgs: any = {};
|
||||
if (rawArgs) {
|
||||
if (typeof rawArgs === 'object') {
|
||||
parsedArgs = rawArgs;
|
||||
} else if (typeof rawArgs === 'string') {
|
||||
try {
|
||||
parsedArgs = JSON.parse(rawArgs);
|
||||
} catch {
|
||||
if (!rawArgs.includes('<') && !rawArgs.includes('object at')) {
|
||||
parsedArgs = { details: rawArgs };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const mergedObj = { ...act, ...parsedArgs };
|
||||
|
||||
// Explicitly check duration if available and not already formatted
|
||||
const dur = mergedObj.duration || mergedObj.duration_ms;
|
||||
if (dur !== undefined && dur !== null && dur !== '') {
|
||||
result.push({ key: 'Duration', value: typeof dur === 'number' ? `${dur}ms` : String(dur) });
|
||||
}
|
||||
|
||||
for (const [k, v] of Object.entries(mergedObj)) {
|
||||
const lowerK = k.toLowerCase();
|
||||
if (standardKeys.has(lowerK)) continue;
|
||||
if (v === null || v === undefined || v === '') continue;
|
||||
|
||||
let valStr = String(v);
|
||||
if (valStr.includes('object at 0x') || valStr.startsWith('<artemis.') || valStr.includes('<controller')) continue;
|
||||
|
||||
if (typeof v === 'object') {
|
||||
try { valStr = JSON.stringify(v); } catch { valStr = String(v); }
|
||||
}
|
||||
|
||||
let prettyKey = k.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
|
||||
if (lowerK === 'time_in_ms' || lowerK === 'delay_ms') prettyKey = 'Delay';
|
||||
|
||||
result.push({ key: prettyKey, value: valStr });
|
||||
}
|
||||
|
||||
if (cache) {
|
||||
cache.set(act, result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format image URL or path safely
|
||||
*/
|
||||
export function formatImageUrl(candidate: any): string | null {
|
||||
if (!candidate || candidate === 'None' || candidate === 'null' || candidate === 'undefined') return null;
|
||||
if (typeof candidate !== 'string') return null;
|
||||
const trimmed = candidate.trim();
|
||||
if (!trimmed || trimmed === 'None' || trimmed === 'null' || trimmed === 'undefined') return null;
|
||||
|
||||
if (trimmed.startsWith('data:') || trimmed.startsWith('http://') || trimmed.startsWith('https://')) {
|
||||
return trimmed;
|
||||
}
|
||||
if (trimmed.startsWith('/local_file') || trimmed.startsWith('/images/') || trimmed.startsWith('/api/images/')) {
|
||||
return trimmed;
|
||||
}
|
||||
if (trimmed.startsWith('file://')) {
|
||||
return `/local_file?path=${encodeURIComponent(trimmed)}`;
|
||||
}
|
||||
if (trimmed.startsWith('/')) {
|
||||
return `/local_file?path=${encodeURIComponent('file://' + trimmed)}`;
|
||||
}
|
||||
return `/images/${trimmed}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to check if an item is a generic tool trace rather than the step's primary action
|
||||
*/
|
||||
export function isGenericToolTrace(actionData: any, stepData?: any): boolean {
|
||||
if (!actionData) return false;
|
||||
if (actionData === stepData?.action_taken) return false;
|
||||
if (Array.isArray(stepData?.action_taken)) {
|
||||
if (stepData.action_taken === actionData || stepData.action_taken.includes(actionData)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (actionData.is_primary_step_action) return false;
|
||||
return Boolean(actionData.trace_id || actionData.payload || actionData.type === 'tool' || (actionData.type === 'action' && actionData.trace_id));
|
||||
}
|
||||
|
||||
interface ResolvedImageEntry {
|
||||
pre: string | null;
|
||||
post: string | null;
|
||||
hasExplicitPost: boolean;
|
||||
}
|
||||
|
||||
const stepImageMapCache = new WeakMap<any, { key: string; map: Map<any, ResolvedImageEntry> }>();
|
||||
|
||||
function safeParseJson(val: any): any {
|
||||
if (typeof val === 'string' && (val.trim().startsWith('{') || val.trim().startsWith('['))) {
|
||||
try {
|
||||
return JSON.parse(val);
|
||||
} catch {
|
||||
return val;
|
||||
}
|
||||
}
|
||||
return val;
|
||||
}
|
||||
|
||||
export function extractItemPreImage(item: any): string | null {
|
||||
const act = getActionObject(item);
|
||||
if (!act) return null;
|
||||
const payload = safeParseJson(act.payload);
|
||||
const result = safeParseJson(act.result || payload?.result);
|
||||
const args = safeParseJson(act.args || payload?.args);
|
||||
|
||||
const candidate =
|
||||
result?.pre_image_name || result?.pre_screenshot_name || result?.pre_screenshot ||
|
||||
payload?.pre_image_name || payload?.pre_screenshot_name || payload?.pre_screenshot ||
|
||||
args?.pre_image_name || args?.pre_screenshot_name || args?.pre_screenshot ||
|
||||
act.pre_image_name || act.pre_screenshot_name || act.pre_screenshot || act.before_screenshot || act.screenshot;
|
||||
|
||||
return candidate ? String(candidate) : null;
|
||||
}
|
||||
|
||||
export function extractItemPostImage(item: any): string | null {
|
||||
const act = getActionObject(item);
|
||||
if (!act) return null;
|
||||
const payload = safeParseJson(act.payload);
|
||||
const result = safeParseJson(act.result || payload?.result);
|
||||
const args = safeParseJson(act.args || payload?.args);
|
||||
|
||||
const candidate =
|
||||
result?.post_image_name || result?.post_screenshot_name || result?.post_screenshot ||
|
||||
payload?.post_image_name || payload?.post_screenshot_name || payload?.post_screenshot ||
|
||||
args?.post_image_name || args?.post_screenshot_name || args?.post_screenshot ||
|
||||
act.post_image_name || act.post_screenshot_name || act.post_screenshot || act.after_screenshot;
|
||||
|
||||
return candidate ? String(candidate) : null;
|
||||
}
|
||||
|
||||
function getStepFailureScreenshot(stepData: any): string | null {
|
||||
if (!stepData) return null;
|
||||
if (stepData.failed_screenshot) return String(stepData.failed_screenshot);
|
||||
if (stepData.failure_screenshot) return String(stepData.failure_screenshot);
|
||||
if (stepData.failed_post_image_name) return String(stepData.failed_post_image_name);
|
||||
|
||||
if (Array.isArray(stepData.generic_tools)) {
|
||||
for (const tool of stepData.generic_tools) {
|
||||
const act = getActionObject(tool);
|
||||
const payload = safeParseJson(act?.payload);
|
||||
const args = safeParseJson(act?.args || payload?.args);
|
||||
const name = String(act?.name || act?.agent_name || '').toLowerCase();
|
||||
if (args?.post_screenshot_name) return String(args.post_screenshot_name);
|
||||
if (args?.post_screenshot && (name.includes('failure') || name.includes('validator') || name.includes('diagnos'))) {
|
||||
return String(args.post_screenshot);
|
||||
}
|
||||
if (act?.post_screenshot_name && (name.includes('failure') || name.includes('validator'))) {
|
||||
return String(act.post_screenshot_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function getEventTimestamp(obj: any): number {
|
||||
if (!obj) return 0;
|
||||
const ts = obj.timestamp ?? obj.start_time ?? obj.created_at;
|
||||
if (typeof ts === 'number') {
|
||||
return ts < 1e11 ? ts * 1000 : ts;
|
||||
}
|
||||
if (typeof ts === 'string') {
|
||||
const parsed = new Date(ts).getTime();
|
||||
if (!isNaN(parsed) && parsed > 0) return parsed;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry> {
|
||||
const emptyMap = new Map<any, ResolvedImageEntry>();
|
||||
if (!stepData || typeof stepData !== 'object') return emptyMap;
|
||||
|
||||
const toolsLen = Array.isArray(stepData.generic_tools) ? stepData.generic_tools.length : 0;
|
||||
const cacheKey = `${stepData.step_id || ''}_${toolsLen}_${stepData.post_image_name || ''}_${stepData.pre_image_name || ''}`;
|
||||
|
||||
const cached = stepImageMapCache.get(stepData);
|
||||
if (cached && cached.key === cacheKey) {
|
||||
return cached.map;
|
||||
}
|
||||
|
||||
const map = new Map<any, ResolvedImageEntry>();
|
||||
const events: Array<{ item: any; act: any; timestamp: number; isPrimary: boolean }> = [];
|
||||
|
||||
if (stepData.action_taken) {
|
||||
events.push({
|
||||
item: stepData.action_taken,
|
||||
act: getActionObject(stepData.action_taken),
|
||||
timestamp: getEventTimestamp(stepData.action_taken),
|
||||
isPrimary: true
|
||||
});
|
||||
}
|
||||
|
||||
if (Array.isArray(stepData.generic_tools)) {
|
||||
stepData.generic_tools.forEach((t: any) => {
|
||||
if (t) {
|
||||
events.push({
|
||||
item: t,
|
||||
act: getActionObject(t),
|
||||
timestamp: getEventTimestamp(t),
|
||||
isPrimary: false
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
events.sort((a, b) => a.timestamp - b.timestamp);
|
||||
|
||||
const failureShot = getStepFailureScreenshot(stepData);
|
||||
const hadPrimaryFailure = Boolean(failureShot) || (stepData.action_taken && isActionFailed(stepData.action_taken, stepData));
|
||||
const hasSubsequentActions = events.some(e => !e.isPrimary && (isAndroidAction(e.act) || Boolean(extractItemPostImage(e.act))));
|
||||
|
||||
let lastActionItem: any = null;
|
||||
for (let i = events.length - 1; i >= 0; i--) {
|
||||
const e = events[i];
|
||||
if (isAndroidAction(e.act) || extractItemPostImage(e.act) || e.isPrimary) {
|
||||
lastActionItem = e.item;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let currentPreImage = stepData.pre_image_name || stepData.pre_screenshot || null;
|
||||
let switchedToFailureShot = false;
|
||||
|
||||
for (const e of events) {
|
||||
const act = e.act;
|
||||
const explicitPre = extractItemPreImage(act);
|
||||
const explicitPost = extractItemPostImage(act);
|
||||
|
||||
const agentName = String(
|
||||
act?.agent_name || act?.agent || safeParseJson(act?.payload)?.agent_name || act?.name || ''
|
||||
).toLowerCase();
|
||||
const isRecoveryOrValidator = agentName.includes('failure') || agentName.includes('analyzer') || agentName.includes('validator') || agentName.includes('diagnos');
|
||||
|
||||
if (failureShot && !switchedToFailureShot && (isRecoveryOrValidator || (!e.isPrimary && hadPrimaryFailure))) {
|
||||
currentPreImage = failureShot;
|
||||
switchedToFailureShot = true;
|
||||
}
|
||||
|
||||
const resolvedPre = explicitPre || currentPreImage;
|
||||
let resolvedPost: string | null = null;
|
||||
let hasExplicitPost = false;
|
||||
|
||||
if (explicitPost) {
|
||||
resolvedPost = explicitPost;
|
||||
hasExplicitPost = true;
|
||||
} else if (e.isPrimary && !hadPrimaryFailure && !hasSubsequentActions) {
|
||||
resolvedPost = stepData.post_image_name || stepData.post_screenshot || null;
|
||||
} else if (e.item === lastActionItem && !hadPrimaryFailure) {
|
||||
resolvedPost = stepData.post_image_name || stepData.post_screenshot || null;
|
||||
} else if (e.item === lastActionItem && isRecoveryOrValidator) {
|
||||
resolvedPost = stepData.post_image_name || stepData.post_screenshot || null;
|
||||
}
|
||||
|
||||
// "下一次的决策前是上一次的动作后"
|
||||
if (resolvedPost) {
|
||||
currentPreImage = resolvedPost;
|
||||
}
|
||||
|
||||
const entry: ResolvedImageEntry = { pre: resolvedPre, post: resolvedPost, hasExplicitPost };
|
||||
map.set(e.item, entry);
|
||||
if (act && act !== e.item) map.set(act, entry);
|
||||
if (act?.trace_id) map.set(act.trace_id, entry);
|
||||
}
|
||||
|
||||
stepImageMapCache.set(stepData, { key: cacheKey, map });
|
||||
return map;
|
||||
}
|
||||
|
||||
function lookupResolvedImages(stepData: any, actionData?: any): ResolvedImageEntry {
|
||||
const map = resolveStepImageMap(stepData);
|
||||
if (!actionData) {
|
||||
return {
|
||||
pre: stepData?.pre_image_name || stepData?.pre_screenshot || null,
|
||||
post: stepData?.post_image_name || stepData?.post_screenshot || null,
|
||||
hasExplicitPost: Boolean(stepData?.post_image_name || stepData?.post_screenshot)
|
||||
};
|
||||
}
|
||||
|
||||
const act = getActionObject(actionData);
|
||||
|
||||
if (map.has(actionData)) return map.get(actionData)!;
|
||||
if (act && map.has(act)) return map.get(act)!;
|
||||
if (act?.trace_id && map.has(act.trace_id)) return map.get(act.trace_id)!;
|
||||
|
||||
const explicitPre = extractItemPreImage(act);
|
||||
const explicitPost = extractItemPostImage(act);
|
||||
const pre = explicitPre || stepData?.pre_image_name || stepData?.pre_screenshot || null;
|
||||
const post = explicitPost || (isGenericToolTrace(actionData, stepData) ? null : (stepData?.post_image_name || stepData?.post_screenshot || null));
|
||||
return { pre, post, hasExplicitPost: Boolean(explicitPost) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get pre-action screenshot URL
|
||||
*/
|
||||
export function getStepPreImageUrl(stepData: any, actionData?: any): string | null {
|
||||
const resolved = lookupResolvedImages(stepData, actionData);
|
||||
return formatImageUrl(resolved.pre);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get post-action screenshot URL
|
||||
*/
|
||||
export function getStepPostImageUrl(stepData: any, actionData?: any): string | null {
|
||||
const resolved = lookupResolvedImages(stepData, actionData);
|
||||
const postUrl = formatImageUrl(resolved.post);
|
||||
const preUrl = formatImageUrl(resolved.pre);
|
||||
|
||||
if (postUrl && preUrl && postUrl === preUrl && !resolved.hasExplicitPost) {
|
||||
return null;
|
||||
}
|
||||
return postUrl;
|
||||
}
|
||||
@@ -0,0 +1,630 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Helper to parse any coordinate representation into an array of numbers.
|
||||
* Handles:
|
||||
* - [x, y] or [x1, y1, x2, y2]
|
||||
* - "[[x1, y1], [x2, y2]]" or [[x1, y1], [x2, y2]]
|
||||
* - "[920, 290, 920, 180]" or "920 290 920 180" or "920, 290, 920, 180"
|
||||
* - "(920, 290, 920, 180)"
|
||||
*/
|
||||
export function extractNumbersFromCoordinateValue(val: any): number[] | null {
|
||||
if (val === null || val === undefined) return null;
|
||||
|
||||
if (Array.isArray(val)) {
|
||||
if (val.length === 4 && val.every(v => typeof v === 'number' && !isNaN(v))) {
|
||||
return val;
|
||||
}
|
||||
if (val.length === 2 && val.every(v => typeof v === 'number' && !isNaN(v))) {
|
||||
return val;
|
||||
}
|
||||
if (val.length === 2 && Array.isArray(val[0]) && Array.isArray(val[1])) {
|
||||
const p1 = extractNumbersFromCoordinateValue(val[0]);
|
||||
const p2 = extractNumbersFromCoordinateValue(val[1]);
|
||||
if (p1 && p2 && p1.length === 2 && p2.length === 2) {
|
||||
return [p1[0], p1[1], p2[0], p2[1]];
|
||||
}
|
||||
}
|
||||
const flattened: number[] = [];
|
||||
for (const item of val) {
|
||||
if (typeof item === 'number' && !isNaN(item)) {
|
||||
flattened.push(item);
|
||||
} else if (typeof item === 'string') {
|
||||
const matches = item.match(/-?\d+(?:\.\d+)?/g);
|
||||
if (matches) {
|
||||
matches.forEach(m => flattened.push(Number(m)));
|
||||
}
|
||||
} else if (Array.isArray(item)) {
|
||||
const sub = extractNumbersFromCoordinateValue(item);
|
||||
if (sub) flattened.push(...sub);
|
||||
}
|
||||
}
|
||||
if (flattened.length === 4 || flattened.length === 2) {
|
||||
return flattened;
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof val === 'string') {
|
||||
const trimmed = val.trim();
|
||||
if (!trimmed) return null;
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
const res = extractNumbersFromCoordinateValue(parsed);
|
||||
if (res) return res;
|
||||
} catch {}
|
||||
|
||||
const matches = trimmed.match(/-?\d+(?:\.\d+)?/g);
|
||||
if (matches && (matches.length === 4 || matches.length === 2)) {
|
||||
return matches.map(m => Number(m));
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse sequence of coordinates (e.g. [[x1, y1], [x2, y2]] or stringified format)
|
||||
*/
|
||||
export function parseSequenceCoordinates(seq: any): number[][] | null {
|
||||
if (!seq) return null;
|
||||
if (typeof seq === 'string') {
|
||||
const trimmed = seq.trim();
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
const res = parseSequenceCoordinates(parsed);
|
||||
if (res && res.length > 0) return res;
|
||||
} catch {}
|
||||
|
||||
const pairs: number[][] = [];
|
||||
const pairRegex = /\[\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\]/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = pairRegex.exec(trimmed)) !== null) {
|
||||
pairs.push([Number(m[1]), Number(m[2])]);
|
||||
}
|
||||
if (pairs.length > 0) return pairs;
|
||||
}
|
||||
|
||||
if (Array.isArray(seq)) {
|
||||
const points: number[][] = [];
|
||||
for (const item of seq) {
|
||||
const p = extractNumbersFromCoordinateValue(item);
|
||||
if (p && p.length === 2) {
|
||||
points.push([p[0], p[1]]);
|
||||
} else if (Array.isArray(item) && item.length >= 2 && typeof item[0] === 'number' && typeof item[1] === 'number') {
|
||||
points.push([item[0], item[1]]);
|
||||
}
|
||||
}
|
||||
if (points.length > 0) return points;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function isPureDirectionString(str: any): boolean {
|
||||
if (typeof str !== 'string') return false;
|
||||
const s = str.trim().toLowerCase();
|
||||
if (/\d/.test(s)) return false;
|
||||
return [
|
||||
'up', 'down', 'left', 'right',
|
||||
'swipe up', 'swipe down', 'swipe left', 'swipe right',
|
||||
'scroll up', 'scroll down', 'scroll left', 'scroll right',
|
||||
'top', 'bottom'
|
||||
].includes(s);
|
||||
}
|
||||
|
||||
/**
|
||||
* Draw action coordinates (points, click sequence, swipe lines) onto the image overlay element
|
||||
*/
|
||||
export function drawActionCoordinatesOnOverlay(
|
||||
img: HTMLImageElement,
|
||||
overlay: HTMLElement,
|
||||
actionObj: any
|
||||
): void {
|
||||
if (!img || !overlay || !actionObj) return;
|
||||
overlay.innerHTML = '';
|
||||
|
||||
const actions = Array.isArray(actionObj) ? actionObj : [actionObj];
|
||||
const naturalWidth = img.naturalWidth || 1080;
|
||||
const naturalHeight = img.naturalHeight || 2400;
|
||||
|
||||
actions.forEach(act => {
|
||||
if (!act || typeof act !== 'object') return;
|
||||
const actionName = (act.action || act.name || 'tap').toLowerCase();
|
||||
|
||||
const payload = (act.payload && typeof act.payload === 'object') ? act.payload : {};
|
||||
const args = (payload.args && typeof payload.args === 'object') ? payload.args : (act.args || {});
|
||||
|
||||
// Filter out non-touch / system actions that do not operate via touch coordinates
|
||||
const nonDrawingActions = [
|
||||
'manage_app', 'launch_app', 'open_app', 'stop_app', 'close_app',
|
||||
'press_key', 'press_home', 'press_back', 'wait_for_delay', 'wait', 'delay',
|
||||
'report_task_status', 'report_failure_analysis'
|
||||
];
|
||||
if (nonDrawingActions.includes(actionName)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. Multi-point click sequence (High priority for click_sequence / tap_sequence)
|
||||
const isSequenceAction = actionName.includes('sequence') || Boolean(act.sequence || args.sequence || act.normalized_sequence || args.normalized_sequence);
|
||||
const rawSeq = act.normalized_sequence || args.normalized_sequence || act.sequence || args.sequence || act.targets || args.targets || (Array.isArray(act.coordinates) && Array.isArray(act.coordinates[0]) ? act.coordinates : null) || (Array.isArray(args.coordinates) && Array.isArray(args.coordinates[0]) ? args.coordinates : null);
|
||||
const seqPoints = parseSequenceCoordinates(rawSeq);
|
||||
|
||||
if (seqPoints && seqPoints.length > 0 && (isSequenceAction || seqPoints.length > 1)) {
|
||||
const isNormSeq = Boolean(act.normalized_sequence || args.normalized_sequence) || seqPoints.every(p => p[0] <= 1000 && p[1] <= 1000);
|
||||
// Draw connecting lines between consecutive points
|
||||
for (let i = 0; i < seqPoints.length - 1; i++) {
|
||||
createSequenceConnector(
|
||||
seqPoints[i][0], seqPoints[i][1],
|
||||
seqPoints[i + 1][0], seqPoints[i + 1][1],
|
||||
naturalWidth, naturalHeight,
|
||||
overlay,
|
||||
isNormSeq
|
||||
);
|
||||
}
|
||||
// Draw numbered markers for each point in the sequence
|
||||
seqPoints.forEach((pt, idx) => {
|
||||
createPointMarker(
|
||||
pt[0], pt[1],
|
||||
actionName,
|
||||
naturalWidth, naturalHeight,
|
||||
overlay,
|
||||
isNormSeq,
|
||||
idx + 1,
|
||||
seqPoints.length
|
||||
);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine if this is a swipe/scroll/drag gesture
|
||||
const isSwipeLike = actionName.includes('swipe') || actionName.includes('scroll') || actionName.includes('drag') || actionName.includes('slide');
|
||||
|
||||
// 2. Check separate start and end coordinate objects
|
||||
const normStart = extractNumbersFromCoordinateValue(act.normalized_start_coordinates || args.normalized_start_coordinates);
|
||||
const normEnd = extractNumbersFromCoordinateValue(act.normalized_end_coordinates || args.normalized_end_coordinates);
|
||||
const startCoords = normStart || extractNumbersFromCoordinateValue(act.start_coordinates || act.start_point || act.start || act.from ||
|
||||
args.start_coordinates || args.start_point || args.start || args.from);
|
||||
const endCoords = normEnd || extractNumbersFromCoordinateValue(act.end_coordinates || act.end_point || act.end || act.to ||
|
||||
args.end_coordinates || args.end_point || args.end || args.to);
|
||||
|
||||
if (startCoords && endCoords && startCoords.length >= 2 && endCoords.length >= 2) {
|
||||
createLineMarker(
|
||||
startCoords[0], startCoords[1],
|
||||
endCoords[0], endCoords[1],
|
||||
naturalWidth, naturalHeight,
|
||||
overlay,
|
||||
Boolean(normStart && normEnd),
|
||||
actionName
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// 3. Check 4-element coordinate array in coordinates, target, action, gesture, etc.
|
||||
const normCoords = extractNumbersFromCoordinateValue(act.normalized_coordinates || args.normalized_coordinates);
|
||||
const isNorm = Boolean(normCoords);
|
||||
const rawCoords = normCoords ||
|
||||
extractNumbersFromCoordinateValue(act.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(args.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(act.coords) ||
|
||||
extractNumbersFromCoordinateValue(args.coords) ||
|
||||
extractNumbersFromCoordinateValue(act.target) ||
|
||||
extractNumbersFromCoordinateValue(args.target) ||
|
||||
(isSwipeLike ? extractNumbersFromCoordinateValue(args.action) : null) ||
|
||||
(isSwipeLike ? extractNumbersFromCoordinateValue(act.action) : null) ||
|
||||
(isSwipeLike ? extractNumbersFromCoordinateValue(args.gesture) : null) ||
|
||||
(isSwipeLike ? extractNumbersFromCoordinateValue(act.gesture) : null);
|
||||
|
||||
if (rawCoords && Array.isArray(rawCoords)) {
|
||||
if (rawCoords.length === 4) {
|
||||
createLineMarker(
|
||||
rawCoords[0], rawCoords[1], rawCoords[2], rawCoords[3],
|
||||
naturalWidth, naturalHeight,
|
||||
overlay,
|
||||
isNorm,
|
||||
actionName
|
||||
);
|
||||
return;
|
||||
} else if (rawCoords.length === 2 && !isSwipeLike) {
|
||||
createPointMarker(rawCoords[0], rawCoords[1], actionName, naturalWidth, naturalHeight, overlay, isNorm);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Check for directional swipe/scroll/drag gestures ONLY when action is swipe-like
|
||||
if (isSwipeLike) {
|
||||
const rawDir = (typeof args.action === 'string' && isPureDirectionString(args.action) ? args.action : null) ||
|
||||
(typeof act.action === 'string' && act.action !== actionName && isPureDirectionString(act.action) ? act.action : null) ||
|
||||
(isPureDirectionString(args.direction) ? args.direction : null) ||
|
||||
(isPureDirectionString(act.direction) ? act.direction : null) ||
|
||||
(isPureDirectionString(args.gesture) ? args.gesture : null) ||
|
||||
(isPureDirectionString(act.gesture) ? act.gesture : null) || '';
|
||||
const dirStr = String(rawDir).toLowerCase().trim();
|
||||
|
||||
if (dirStr) {
|
||||
let sx = 500, sy = 750, ex = 500, ey = 250;
|
||||
let label = 'SWIPE';
|
||||
|
||||
if (dirStr.includes('up') || dirStr.includes('top') || actionName.includes('up')) {
|
||||
sx = 500; sy = 750; ex = 500; ey = 250;
|
||||
label = 'SWIPE UP';
|
||||
} else if (dirStr.includes('down') || dirStr.includes('bottom') || actionName.includes('down')) {
|
||||
sx = 500; sy = 250; ex = 500; ey = 750;
|
||||
label = 'SWIPE DOWN';
|
||||
} else if (dirStr.includes('left') || actionName.includes('left')) {
|
||||
sx = 800; sy = 500; ex = 200; ey = 500;
|
||||
label = 'SWIPE LEFT';
|
||||
} else if (dirStr.includes('right') || actionName.includes('right')) {
|
||||
sx = 200; sy = 500; ex = 800; ey = 500;
|
||||
label = 'SWIPE RIGHT';
|
||||
}
|
||||
|
||||
createLineMarker(sx, sy, ex, ey, naturalWidth, naturalHeight, overlay, true, label);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Default tap/click coordinate fallback
|
||||
if (rawCoords && Array.isArray(rawCoords) && rawCoords.length === 2) {
|
||||
createPointMarker(rawCoords[0], rawCoords[1], actionName, naturalWidth, naturalHeight, overlay, isNorm);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create and position an animated red dot point marker with edge-aware label
|
||||
*/
|
||||
export function createPointMarker(
|
||||
x: number,
|
||||
y: number,
|
||||
actionType: string,
|
||||
naturalWidth: number,
|
||||
naturalHeight: number,
|
||||
overlay: HTMLElement,
|
||||
isNormalized: boolean = false,
|
||||
sequenceIndex?: number,
|
||||
totalSequence?: number
|
||||
): void {
|
||||
let pctX: number;
|
||||
let pctY: number;
|
||||
|
||||
// Accurate 0-1000 scale resolution
|
||||
if (isNormalized || (!isNormalized && x <= 1000 && y <= 1000)) {
|
||||
pctX = x / 10;
|
||||
pctY = y / 10;
|
||||
} else {
|
||||
pctX = (x / (naturalWidth || 1080)) * 100;
|
||||
pctY = (y / (naturalHeight || 2400)) * 100;
|
||||
}
|
||||
|
||||
// Safety clamp within container boundaries
|
||||
const clampedPctX = Math.max(1.5, Math.min(98.5, pctX));
|
||||
const clampedPctY = Math.max(1.5, Math.min(98.5, pctY));
|
||||
|
||||
const marker = document.createElement('div');
|
||||
marker.className = 'action-point-marker';
|
||||
marker.style.position = 'absolute';
|
||||
marker.style.left = `${clampedPctX}%`;
|
||||
marker.style.top = `${clampedPctY}%`;
|
||||
marker.style.transform = 'translate(-50%, -50%)';
|
||||
marker.style.pointerEvents = 'none';
|
||||
marker.style.zIndex = '10';
|
||||
|
||||
const dotSize = sequenceIndex !== undefined ? 20 : 14;
|
||||
const dot = document.createElement('div');
|
||||
dot.style.width = `${dotSize}px`;
|
||||
dot.style.height = `${dotSize}px`;
|
||||
dot.style.borderRadius = '50%';
|
||||
dot.style.backgroundColor = '#ef4444';
|
||||
dot.style.border = '2px solid white';
|
||||
dot.style.boxShadow = '0 0 6px rgba(0,0,0,0.5)';
|
||||
if (sequenceIndex !== undefined) {
|
||||
dot.style.display = 'flex';
|
||||
dot.style.alignItems = 'center';
|
||||
dot.style.justifyContent = 'center';
|
||||
dot.style.fontSize = '11px';
|
||||
dot.style.fontWeight = 'bold';
|
||||
dot.style.color = 'white';
|
||||
dot.innerText = String(sequenceIndex);
|
||||
}
|
||||
|
||||
const pulseSize = dotSize * 2;
|
||||
const pulse = document.createElement('div');
|
||||
pulse.style.position = 'absolute';
|
||||
pulse.style.top = `-${dotSize / 2 + 2}px`;
|
||||
pulse.style.left = `-${dotSize / 2 + 2}px`;
|
||||
pulse.style.width = `${pulseSize}px`;
|
||||
pulse.style.height = `${pulseSize}px`;
|
||||
pulse.style.borderRadius = '50%';
|
||||
pulse.style.border = '2px solid #ef4444';
|
||||
pulse.style.animation = 'pulse-animation 1.5s infinite ease-out';
|
||||
pulse.style.pointerEvents = 'none';
|
||||
|
||||
const label = document.createElement('div');
|
||||
if (sequenceIndex !== undefined) {
|
||||
label.innerText = totalSequence && totalSequence > 1 ? `#${sequenceIndex} TAP` : actionType.toUpperCase();
|
||||
} else {
|
||||
label.innerText = actionType.toUpperCase();
|
||||
}
|
||||
label.style.position = 'absolute';
|
||||
label.style.backgroundColor = 'rgba(15, 23, 42, 0.9)';
|
||||
label.style.color = 'white';
|
||||
label.style.padding = '1.5px 5px';
|
||||
label.style.borderRadius = '3px';
|
||||
label.style.fontSize = '9px';
|
||||
label.style.fontWeight = 'bold';
|
||||
label.style.letterSpacing = '0.5px';
|
||||
label.style.whiteSpace = 'nowrap';
|
||||
label.style.boxShadow = '0 1px 4px rgba(0,0,0,0.4)';
|
||||
label.style.border = '1px solid rgba(255,255,255,0.2)';
|
||||
|
||||
// Smart edge-aware vertical positioning
|
||||
if (clampedPctY > 85) {
|
||||
label.style.bottom = `${dotSize + 2}px`;
|
||||
label.style.top = 'auto';
|
||||
} else {
|
||||
label.style.top = `${dotSize + 2}px`;
|
||||
label.style.bottom = 'auto';
|
||||
}
|
||||
|
||||
// Smart edge-aware horizontal positioning
|
||||
if (clampedPctX > 82) {
|
||||
label.style.right = '0px';
|
||||
label.style.left = 'auto';
|
||||
label.style.transform = 'none';
|
||||
} else if (clampedPctX < 18) {
|
||||
label.style.left = '0px';
|
||||
label.style.right = 'auto';
|
||||
label.style.transform = 'none';
|
||||
} else {
|
||||
label.style.left = '50%';
|
||||
label.style.right = 'auto';
|
||||
label.style.transform = 'translateX(-50%)';
|
||||
}
|
||||
|
||||
marker.appendChild(pulse);
|
||||
marker.appendChild(dot);
|
||||
marker.appendChild(label);
|
||||
overlay.appendChild(marker);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to draw connecting line/arrow between points in a click sequence
|
||||
*/
|
||||
export function createSequenceConnector(
|
||||
x1: number,
|
||||
y1: number,
|
||||
x2: number,
|
||||
y2: number,
|
||||
naturalWidth: number,
|
||||
naturalHeight: number,
|
||||
overlay: HTMLElement,
|
||||
isNormalized: boolean = false
|
||||
): void {
|
||||
const isNormCoord = isNormalized || (x1 <= 1000 && y1 <= 1000 && x2 <= 1000 && y2 <= 1000);
|
||||
const pctX1 = isNormCoord ? x1 / 10 : (x1 / (naturalWidth || 1080)) * 100;
|
||||
const pctY1 = isNormCoord ? y1 / 10 : (y1 / (naturalHeight || 2400)) * 100;
|
||||
const pctX2 = isNormCoord ? x2 / 10 : (x2 / (naturalWidth || 1080)) * 100;
|
||||
const pctY2 = isNormCoord ? y2 / 10 : (y2 / (naturalHeight || 2400)) * 100;
|
||||
|
||||
const clampedX1 = Math.max(1.5, Math.min(98.5, pctX1));
|
||||
const clampedY1 = Math.max(1.5, Math.min(98.5, pctY1));
|
||||
const clampedX2 = Math.max(1.5, Math.min(98.5, pctX2));
|
||||
const clampedY2 = Math.max(1.5, Math.min(98.5, pctY2));
|
||||
|
||||
let svg = overlay.querySelector('svg');
|
||||
if (!svg) {
|
||||
svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
|
||||
svg.setAttribute('width', '100%');
|
||||
svg.setAttribute('height', '100%');
|
||||
svg.style.position = 'absolute';
|
||||
svg.style.top = '0';
|
||||
svg.style.left = '0';
|
||||
svg.style.pointerEvents = 'none';
|
||||
svg.style.overflow = 'visible';
|
||||
svg.style.zIndex = '5';
|
||||
overlay.appendChild(svg);
|
||||
}
|
||||
|
||||
let defs = svg.querySelector('defs');
|
||||
if (!defs) {
|
||||
defs = document.createElementNS('http://www.w3.org/2000/svg', 'defs');
|
||||
svg.insertBefore(defs, svg.firstChild);
|
||||
}
|
||||
|
||||
const markerId = 'seq-arrow-' + Math.random().toString(36).substring(2, 9);
|
||||
const marker = document.createElementNS('http://www.w3.org/2000/svg', 'marker');
|
||||
marker.setAttribute('id', markerId);
|
||||
marker.setAttribute('viewBox', '0 0 12 12');
|
||||
marker.setAttribute('refX', '8');
|
||||
marker.setAttribute('refY', '6');
|
||||
marker.setAttribute('markerWidth', '6');
|
||||
marker.setAttribute('markerHeight', '6');
|
||||
marker.setAttribute('orient', 'auto');
|
||||
|
||||
const path = document.createElementNS('http://www.w3.org/2000/svg', 'path');
|
||||
path.setAttribute('d', 'M 1 2 L 10 6 L 1 10 z');
|
||||
path.setAttribute('fill', '#ef4444');
|
||||
path.setAttribute('stroke', '#ffffff');
|
||||
path.setAttribute('stroke-width', '1');
|
||||
|
||||
marker.appendChild(path);
|
||||
defs.appendChild(marker);
|
||||
|
||||
// Background white contrast line
|
||||
const bgLine = document.createElementNS('http://www.w3.org/2000/svg', 'line');
|
||||
bgLine.setAttribute('x1', `${clampedX1}%`);
|
||||
bgLine.setAttribute('y1', `${clampedY1}%`);
|
||||
bgLine.setAttribute('x2', `${clampedX2}%`);
|
||||
bgLine.setAttribute('y2', `${clampedY2}%`);
|
||||
bgLine.setAttribute('stroke', '#ffffff');
|
||||
bgLine.setAttribute('stroke-width', '4');
|
||||
bgLine.setAttribute('stroke-linecap', 'round');
|
||||
bgLine.setAttribute('opacity', '0.8');
|
||||
|
||||
// Main red dashed sequence line
|
||||
const line = document.createElementNS('http://www.w3.org/2000/svg', 'line');
|
||||
line.setAttribute('x1', `${clampedX1}%`);
|
||||
line.setAttribute('y1', `${clampedY1}%`);
|
||||
line.setAttribute('x2', `${clampedX2}%`);
|
||||
line.setAttribute('y2', `${clampedY2}%`);
|
||||
line.setAttribute('stroke', '#ef4444');
|
||||
line.setAttribute('stroke-width', '2.5');
|
||||
line.setAttribute('stroke-linecap', 'round');
|
||||
line.setAttribute('marker-end', `url(#${markerId})`);
|
||||
line.setAttribute('stroke-dasharray', '4,4');
|
||||
|
||||
svg.appendChild(bgLine);
|
||||
svg.appendChild(line);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create and position an SVG arrow, high-contrast dashed line, and badge for swipes/drags
|
||||
*/
|
||||
export function createLineMarker(
|
||||
x1: number,
|
||||
y1: number,
|
||||
x2: number,
|
||||
y2: number,
|
||||
naturalWidth: number,
|
||||
naturalHeight: number,
|
||||
overlay: HTMLElement,
|
||||
isNormalized: boolean = false,
|
||||
actionType: string = 'swipe'
|
||||
): void {
|
||||
const isNormCoord = isNormalized || (x1 <= 1000 && y1 <= 1000 && x2 <= 1000 && y2 <= 1000);
|
||||
const pctX1 = isNormCoord ? x1 / 10 : (x1 / (naturalWidth || 1080)) * 100;
|
||||
const pctY1 = isNormCoord ? y1 / 10 : (y1 / (naturalHeight || 2400)) * 100;
|
||||
const pctX2 = isNormCoord ? x2 / 10 : (x2 / (naturalWidth || 1080)) * 100;
|
||||
const pctY2 = isNormCoord ? y2 / 10 : (y2 / (naturalHeight || 2400)) * 100;
|
||||
|
||||
const clampedX1 = Math.max(1.5, Math.min(98.5, pctX1));
|
||||
const clampedY1 = Math.max(1.5, Math.min(98.5, pctY1));
|
||||
const clampedX2 = Math.max(1.5, Math.min(98.5, pctX2));
|
||||
const clampedY2 = Math.max(1.5, Math.min(98.5, pctY2));
|
||||
|
||||
let svg = overlay.querySelector('svg');
|
||||
if (!svg) {
|
||||
svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
|
||||
svg.setAttribute('width', '100%');
|
||||
svg.setAttribute('height', '100%');
|
||||
svg.style.position = 'absolute';
|
||||
svg.style.top = '0';
|
||||
svg.style.left = '0';
|
||||
svg.style.pointerEvents = 'none';
|
||||
svg.style.overflow = 'visible';
|
||||
svg.style.zIndex = '5';
|
||||
overlay.appendChild(svg);
|
||||
}
|
||||
|
||||
let defs = svg.querySelector('defs');
|
||||
if (!defs) {
|
||||
defs = document.createElementNS('http://www.w3.org/2000/svg', 'defs');
|
||||
svg.insertBefore(defs, svg.firstChild);
|
||||
}
|
||||
|
||||
const markerId = 'action-arrow-' + Math.random().toString(36).substring(2, 9);
|
||||
const marker = document.createElementNS('http://www.w3.org/2000/svg', 'marker');
|
||||
marker.setAttribute('id', markerId);
|
||||
marker.setAttribute('viewBox', '0 0 12 12');
|
||||
marker.setAttribute('refX', '8');
|
||||
marker.setAttribute('refY', '6');
|
||||
marker.setAttribute('markerWidth', '8');
|
||||
marker.setAttribute('markerHeight', '8');
|
||||
marker.setAttribute('orient', 'auto');
|
||||
|
||||
const path = document.createElementNS('http://www.w3.org/2000/svg', 'path');
|
||||
path.setAttribute('d', 'M 1 2 L 10 6 L 1 10 z');
|
||||
path.setAttribute('fill', '#ef4444');
|
||||
path.setAttribute('stroke', '#ffffff');
|
||||
path.setAttribute('stroke-width', '1.5');
|
||||
path.setAttribute('stroke-linejoin', 'round');
|
||||
|
||||
marker.appendChild(path);
|
||||
defs.appendChild(marker);
|
||||
|
||||
// Background white contrast line
|
||||
const bgLine = document.createElementNS('http://www.w3.org/2000/svg', 'line');
|
||||
bgLine.setAttribute('x1', `${clampedX1}%`);
|
||||
bgLine.setAttribute('y1', `${clampedY1}%`);
|
||||
bgLine.setAttribute('x2', `${clampedX2}%`);
|
||||
bgLine.setAttribute('y2', `${clampedY2}%`);
|
||||
bgLine.setAttribute('stroke', '#ffffff');
|
||||
bgLine.setAttribute('stroke-width', '6');
|
||||
bgLine.setAttribute('stroke-linecap', 'round');
|
||||
bgLine.setAttribute('opacity', '0.9');
|
||||
|
||||
// Main red dashed line
|
||||
const line = document.createElementNS('http://www.w3.org/2000/svg', 'line');
|
||||
line.setAttribute('x1', `${clampedX1}%`);
|
||||
line.setAttribute('y1', `${clampedY1}%`);
|
||||
line.setAttribute('x2', `${clampedX2}%`);
|
||||
line.setAttribute('y2', `${clampedY2}%`);
|
||||
line.setAttribute('stroke', '#ef4444');
|
||||
line.setAttribute('stroke-width', '3.5');
|
||||
line.setAttribute('stroke-linecap', 'round');
|
||||
line.setAttribute('marker-end', `url(#${markerId})`);
|
||||
line.setAttribute('stroke-dasharray', '6,4');
|
||||
|
||||
// Start touch point circle with white border
|
||||
const startCircleOuter = document.createElementNS('http://www.w3.org/2000/svg', 'circle');
|
||||
startCircleOuter.setAttribute('cx', `${clampedX1}%`);
|
||||
startCircleOuter.setAttribute('cy', `${clampedY1}%`);
|
||||
startCircleOuter.setAttribute('r', '6.5');
|
||||
startCircleOuter.setAttribute('fill', '#ef4444');
|
||||
startCircleOuter.setAttribute('stroke', 'white');
|
||||
startCircleOuter.setAttribute('stroke-width', '2');
|
||||
|
||||
const startCircleInner = document.createElementNS('http://www.w3.org/2000/svg', 'circle');
|
||||
startCircleInner.setAttribute('cx', `${clampedX1}%`);
|
||||
startCircleInner.setAttribute('cy', `${clampedY1}%`);
|
||||
startCircleInner.setAttribute('r', '2.5');
|
||||
startCircleInner.setAttribute('fill', 'white');
|
||||
|
||||
svg.appendChild(bgLine);
|
||||
svg.appendChild(line);
|
||||
svg.appendChild(startCircleOuter);
|
||||
svg.appendChild(startCircleInner);
|
||||
|
||||
// Floating Action badge at midpoint of swipe
|
||||
const midX = (clampedX1 + clampedX2) / 2;
|
||||
const midY = (clampedY1 + clampedY2) / 2;
|
||||
|
||||
const label = document.createElement('div');
|
||||
label.className = 'action-point-marker action-line-label';
|
||||
label.style.position = 'absolute';
|
||||
label.style.left = `${midX}%`;
|
||||
label.style.top = `${midY}%`;
|
||||
label.style.transform = 'translate(-50%, -50%)';
|
||||
label.style.pointerEvents = 'none';
|
||||
label.style.zIndex = '10';
|
||||
|
||||
const badge = document.createElement('div');
|
||||
badge.innerText = actionType.toUpperCase();
|
||||
badge.style.backgroundColor = 'rgba(15, 23, 42, 0.92)';
|
||||
badge.style.color = 'white';
|
||||
badge.style.padding = '2px 6px';
|
||||
badge.style.borderRadius = '4px';
|
||||
badge.style.fontSize = '9px';
|
||||
badge.style.fontWeight = 'bold';
|
||||
badge.style.letterSpacing = '0.5px';
|
||||
badge.style.whiteSpace = 'nowrap';
|
||||
badge.style.boxShadow = '0 2px 6px rgba(0,0,0,0.5)';
|
||||
badge.style.border = '1px solid rgba(255,255,255,0.3)';
|
||||
|
||||
label.appendChild(badge);
|
||||
overlay.appendChild(label);
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote } from '../core/models/markdown.model';
|
||||
import { CheckerResult } from '../core/models/stream.model';
|
||||
|
||||
/**
|
||||
* Split text into bold, code, and plain segments
|
||||
*/
|
||||
export function parseMarkdownSegments(text: string): MarkdownSegment[] {
|
||||
if (!text) return [];
|
||||
const parts = text.split(/(\*\*.*?\*\*|`.*?`)/g);
|
||||
return parts
|
||||
.filter(part => part !== '')
|
||||
.map(part => {
|
||||
if (part.startsWith('**') && part.endsWith('**')) {
|
||||
return {
|
||||
text: part.slice(2, -2),
|
||||
bold: true,
|
||||
code: false
|
||||
};
|
||||
} else if (part.startsWith('`') && part.endsWith('`')) {
|
||||
return {
|
||||
text: part.slice(1, -1),
|
||||
bold: false,
|
||||
code: true
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
text: part,
|
||||
bold: false,
|
||||
code: false
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse markdown note content into structured lines (checklists, headers, list items)
|
||||
*/
|
||||
export function parseNoteLines(content: string): MarkdownLine[] {
|
||||
if (!content) return [];
|
||||
return content.split('\n').map(line => {
|
||||
const trimmed = line.trim();
|
||||
const indent = line.length - line.trimStart().length;
|
||||
|
||||
// Checkbox [x]
|
||||
if (trimmed.startsWith('- [x] ') || trimmed.startsWith('- [X] ')) {
|
||||
const rawText = trimmed.substring(6);
|
||||
return { type: 'checked', segments: parseMarkdownSegments(rawText), indent };
|
||||
}
|
||||
// Active checkbox [/]
|
||||
if (trimmed.startsWith('- [/] ')) {
|
||||
const rawText = trimmed.substring(6);
|
||||
return { type: 'progress', segments: parseMarkdownSegments(rawText), indent };
|
||||
}
|
||||
// Unchecked checkbox [ ]
|
||||
if (trimmed.startsWith('- [ ] ')) {
|
||||
const rawText = trimmed.substring(6);
|
||||
return { type: 'unchecked', segments: parseMarkdownSegments(rawText), indent };
|
||||
}
|
||||
// Regular list item
|
||||
if (trimmed.startsWith('- ') || trimmed.startsWith('* ')) {
|
||||
const rawText = trimmed.substring(2);
|
||||
return { type: 'list-item', segments: parseMarkdownSegments(rawText), indent };
|
||||
}
|
||||
// Headers
|
||||
if (trimmed.startsWith('# ')) {
|
||||
const rawText = trimmed.substring(2);
|
||||
return { type: 'h1', segments: parseMarkdownSegments(rawText), indent };
|
||||
}
|
||||
if (trimmed.startsWith('## ')) {
|
||||
const rawText = trimmed.substring(3);
|
||||
return { type: 'h2', segments: parseMarkdownSegments(rawText), indent };
|
||||
}
|
||||
if (trimmed.startsWith('### ')) {
|
||||
const rawText = trimmed.substring(4);
|
||||
return { type: 'h3', segments: parseMarkdownSegments(rawText), indent };
|
||||
}
|
||||
// Default line
|
||||
return trimmed ? { type: 'text', segments: parseMarkdownSegments(line), indent } : { type: 'empty', segments: [], indent: 0 };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Group parsed markdown lines into title, milestones (checklists), and other lines
|
||||
*/
|
||||
export function parseNote(content: string): ParsedNote {
|
||||
const lines = parseNoteLines(content);
|
||||
const milestones: NoteMilestone[] = [];
|
||||
const otherLines: MarkdownLine[] = [];
|
||||
let title: string | null = null;
|
||||
let currentMilestone: (NoteMilestone & { _indent?: number }) | null = null;
|
||||
let milestoneCount = 0;
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.type === 'empty') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((line.type === 'h1' || line.type === 'h2') && !title) {
|
||||
title = line.segments.map(s => s.text).join('');
|
||||
continue;
|
||||
}
|
||||
|
||||
const isChecklistItem = ['checked', 'progress', 'unchecked'].includes(line.type);
|
||||
|
||||
if (isChecklistItem && (!currentMilestone || line.indent <= (currentMilestone._indent ?? 0))) {
|
||||
milestoneCount++;
|
||||
currentMilestone = {
|
||||
index: milestoneCount,
|
||||
type: line.type as 'checked' | 'progress' | 'unchecked',
|
||||
segments: line.segments,
|
||||
subSteps: [],
|
||||
_indent: line.indent
|
||||
};
|
||||
milestones.push(currentMilestone);
|
||||
} else if (currentMilestone) {
|
||||
if (line.indent > (currentMilestone._indent ?? 0)) {
|
||||
currentMilestone.subSteps.push(line);
|
||||
} else {
|
||||
otherLines.push(line);
|
||||
}
|
||||
} else {
|
||||
otherLines.push(line);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
milestones,
|
||||
otherLines
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the Checker JSON response block
|
||||
*/
|
||||
export function extractCheckerResult(text: string): CheckerResult | null {
|
||||
if (!text) return null;
|
||||
let cleanText = text.trim();
|
||||
if (cleanText.includes('```json')) {
|
||||
const parts = cleanText.split('```json');
|
||||
if (parts.length > 1) {
|
||||
cleanText = parts[1].split('```')[0].trim();
|
||||
}
|
||||
} else if (cleanText.includes('```')) {
|
||||
const parts = cleanText.split('```');
|
||||
if (parts.length > 1) {
|
||||
cleanText = parts[1].split('```')[0].trim();
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(cleanText);
|
||||
if (parsed && typeof parsed === 'object' && 'success' in parsed) {
|
||||
return {
|
||||
success: Boolean(parsed.success),
|
||||
reason: parsed.reason || 'No reason provided.'
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to format inline markdown segments (bold, italic, code, strikethrough)
|
||||
*/
|
||||
function formatInlineMarkdown(text: string): string {
|
||||
if (!text) return '';
|
||||
return text
|
||||
// Inline code first (so asterisks inside code aren't formatted)
|
||||
.replace(/`([^`]+)`/g, '<code class="inline-code">$1</code>')
|
||||
// Bold + Italic
|
||||
.replace(/\*\*\*([^*]+)\*\*\*/g, '<strong><em>$1</em></strong>')
|
||||
.replace(/___([^_]+)___/g, '<strong><em>$1</em></strong>')
|
||||
// Bold
|
||||
.replace(/\*\*([^*]+)\*\*/g, '<strong>$1</strong>')
|
||||
.replace(/__([^_]+)__/g, '<strong>$1</strong>')
|
||||
// Italic
|
||||
.replace(/\*([^*]+)\*/g, '<em>$1</em>')
|
||||
.replace(/\b_([^_]+)_\b/g, '<em>$1</em>')
|
||||
// Strikethrough
|
||||
.replace(/~~([^~]+)~~/g, '<del>$1</del>');
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert basic markdown text to safe HTML for agent logs/thinking
|
||||
*/
|
||||
export function renderMarkdownToHtml(text: string): string {
|
||||
if (!text) return '';
|
||||
|
||||
// Clean up internal agent XML-like tags (e.g. <thought>, <short_term_memory>, etc.)
|
||||
const cleanedText = text
|
||||
.replace(/<\/?(thought|short_term_memory|strong_term_memory|reasoning|plan|task_plan|call_tool)[^\n>]*>?/gi, '')
|
||||
.replace(/\r\n/g, '\n')
|
||||
.replace(/\r/g, '\n')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim();
|
||||
|
||||
// Escape HTML first to prevent XSS
|
||||
const escaped = cleanedText
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>');
|
||||
|
||||
const rawLines = escaped.split('\n');
|
||||
const result: string[] = [];
|
||||
let currentListType: 'ul' | 'ol' | null = null;
|
||||
let inCodeBlock = false;
|
||||
let codeBlockLang = '';
|
||||
let codeBlockLines: string[] = [];
|
||||
|
||||
const closeList = () => {
|
||||
if (currentListType) {
|
||||
result.push(`</${currentListType}>`);
|
||||
currentListType = null;
|
||||
}
|
||||
};
|
||||
|
||||
for (let i = 0; i < rawLines.length; i++) {
|
||||
const rawLine = rawLines[i];
|
||||
const trimmed = rawLine.trim();
|
||||
|
||||
// 1. Code blocks: ```lang ... ```
|
||||
if (trimmed.startsWith('```')) {
|
||||
if (inCodeBlock) {
|
||||
// End code block
|
||||
const codeContent = codeBlockLines.join('\n');
|
||||
result.push(`<pre class="code-block"><code class="lang-${codeBlockLang}">${codeContent}</code></pre>`);
|
||||
inCodeBlock = false;
|
||||
codeBlockLines = [];
|
||||
codeBlockLang = '';
|
||||
} else {
|
||||
closeList();
|
||||
inCodeBlock = true;
|
||||
codeBlockLang = trimmed.substring(3).trim();
|
||||
codeBlockLines = [];
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (inCodeBlock) {
|
||||
codeBlockLines.push(rawLine);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 2. Headings: # to ######
|
||||
const headingMatch = trimmed.match(/^(#{1,6})\s+(.*)$/);
|
||||
if (headingMatch) {
|
||||
closeList();
|
||||
const level = headingMatch[1].length;
|
||||
const headingContent = formatInlineMarkdown(headingMatch[2]);
|
||||
result.push(`<h${level}>${headingContent}</h${level}>`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 3. Horizontal Rule: ---, ***, ___
|
||||
if (/^(\-{3,}|\*{3,}|_{3,})$/.test(trimmed)) {
|
||||
closeList();
|
||||
result.push('<hr class="md-hr" />');
|
||||
continue;
|
||||
}
|
||||
|
||||
// 4. Blockquotes: > quote
|
||||
if (trimmed.startsWith('> ') || trimmed.startsWith('>')) {
|
||||
closeList();
|
||||
const quoteContent = formatInlineMarkdown(trimmed.replace(/^>\s?/, ''));
|
||||
result.push(`<blockquote>${quoteContent}</blockquote>`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 5. Unordered List: - item or * item
|
||||
const ulMatch = rawLine.match(/^(\s*)([\*\-])\s+(.*)$/);
|
||||
if (ulMatch) {
|
||||
if (currentListType !== 'ul') {
|
||||
closeList();
|
||||
currentListType = 'ul';
|
||||
result.push('<ul>');
|
||||
}
|
||||
const spaces = ulMatch[1].length;
|
||||
const subLevel = Math.max(0, Math.floor((spaces - 2) / 2));
|
||||
const indent = subLevel * 12;
|
||||
const style = indent ? ` style="margin-left: ${indent}px"` : '';
|
||||
const itemContent = formatInlineMarkdown(ulMatch[3]);
|
||||
result.push(`<li${style}>${itemContent}</li>`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 6. Ordered List: 1. item
|
||||
const olMatch = rawLine.match(/^(\s*)(\d+)\.\s+(.*)$/);
|
||||
if (olMatch) {
|
||||
if (currentListType !== 'ol') {
|
||||
closeList();
|
||||
currentListType = 'ol';
|
||||
result.push('<ol>');
|
||||
}
|
||||
const spaces = olMatch[1].length;
|
||||
const subLevel = Math.max(0, Math.floor((spaces - 2) / 2));
|
||||
const indent = subLevel * 12;
|
||||
const style = indent ? ` style="margin-left: ${indent}px"` : '';
|
||||
const itemContent = formatInlineMarkdown(olMatch[3]);
|
||||
result.push(`<li${style}>${itemContent}</li>`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 7. Regular paragraph / empty line
|
||||
closeList();
|
||||
if (trimmed === '') {
|
||||
result.push('<div class="md-para-gap"></div>');
|
||||
} else {
|
||||
const lineContent = formatInlineMarkdown(rawLine);
|
||||
result.push(`<div>${lineContent}</div>`);
|
||||
}
|
||||
}
|
||||
|
||||
if (inCodeBlock) {
|
||||
const codeContent = codeBlockLines.join('\n');
|
||||
result.push(`<pre class="code-block"><code class="lang-${codeBlockLang}">${codeContent}</code></pre>`);
|
||||
}
|
||||
|
||||
closeList();
|
||||
return result.join('\n');
|
||||
}
|
||||
@@ -0,0 +1,547 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { StepBlock, PhaseBlock, StepEvent } from '../core/models/stream.model';
|
||||
import { isAndroidAction, isReportStatusAction, getReportStatusExplanation, getReportStatusValue } from './action-formatter.util';
|
||||
import { getUniqueGenericTools } from './tool-formatter.util';
|
||||
|
||||
/**
|
||||
* Helper to safely extract milliseconds timestamp
|
||||
*/
|
||||
export function getItemTimestamp(ts: any, fallback: number = 0): number {
|
||||
if (ts !== undefined && ts !== null) {
|
||||
if (typeof ts === 'number') {
|
||||
return ts < 1e11 ? ts * 1000 : ts;
|
||||
}
|
||||
const parsed = new Date(ts).getTime();
|
||||
if (!isNaN(parsed) && parsed > 0) return parsed;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Consolidate raw SSE logs into deduplicated and ordered StepBlocks
|
||||
*/
|
||||
export function consolidateLogsToBlocks(rawLogs: any[]): StepBlock[] {
|
||||
if (!rawLogs || rawLogs.length === 0) return [];
|
||||
|
||||
const blocks: StepBlock[] = [];
|
||||
|
||||
rawLogs.forEach(log => {
|
||||
if (!log) return;
|
||||
|
||||
if (log.type === 'llm_stream') {
|
||||
const execId = log.data?.execution_id || log.id;
|
||||
const stepId = log.data?.step_id;
|
||||
const streamType = log.data?.stream_type || 'text';
|
||||
const text = log.data?.text || '';
|
||||
const isCompleted = log.data?.isCompleted ?? false;
|
||||
|
||||
// Find existing block: match by stepId if available, or by execId, or active incomplete block
|
||||
let existingIndex = -1;
|
||||
if (stepId) {
|
||||
existingIndex = blocks.findIndex(b => b.id === `step-${stepId}` || b.data?.step_id === stepId);
|
||||
}
|
||||
if (existingIndex === -1 && execId) {
|
||||
existingIndex = blocks.findIndex(b => b.data?.execution_id === execId || b.id === `stream-${execId}`);
|
||||
}
|
||||
|
||||
if (existingIndex > -1) {
|
||||
const existing = blocks[existingIndex];
|
||||
const updatedData = { ...existing.data };
|
||||
if (streamType === 'thinking') {
|
||||
updatedData.operator_native_thinking = text;
|
||||
} else {
|
||||
updatedData.operator_raw_thinking = text;
|
||||
}
|
||||
if (stepId && !updatedData.step_id) {
|
||||
updatedData.step_id = stepId;
|
||||
}
|
||||
if (execId && !updatedData.execution_id) {
|
||||
updatedData.execution_id = execId;
|
||||
}
|
||||
updatedData.isCompleted = isCompleted;
|
||||
|
||||
blocks[existingIndex] = {
|
||||
...existing,
|
||||
data: updatedData
|
||||
};
|
||||
} else {
|
||||
const blockId = stepId ? `step-${stepId}` : `stream-${execId}`;
|
||||
const blockData: any = {
|
||||
execution_id: execId,
|
||||
step_id: stepId,
|
||||
isCompleted: isCompleted,
|
||||
generic_tools: []
|
||||
};
|
||||
if (streamType === 'thinking') {
|
||||
blockData.operator_native_thinking = text;
|
||||
} else {
|
||||
blockData.operator_raw_thinking = text;
|
||||
}
|
||||
|
||||
blocks.push({
|
||||
id: blockId,
|
||||
type: stepId ? 'step' : 'llm_stream',
|
||||
timestamp: log.timestamp,
|
||||
data: blockData
|
||||
});
|
||||
}
|
||||
} else if (log.type === 'step_recorded' || log.type === 'step_updated') {
|
||||
const stepId = log.data.step_id || log.data.step_number || 'unknown';
|
||||
// Find if there is an existing block for this step or an unattached stream block from this turn
|
||||
let existingIndex = blocks.findIndex(b => b.id === `step-${stepId}` || b.data?.step_id === stepId);
|
||||
|
||||
// If not found by step_id, check if there is an open stream block from the same execution turn
|
||||
if (existingIndex === -1) {
|
||||
for (let i = blocks.length - 1; i >= 0; i--) {
|
||||
if (blocks[i].type === 'llm_stream' && (!blocks[i].data?.step_id || blocks[i].data?.step_id === stepId)) {
|
||||
existingIndex = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (existingIndex > -1) {
|
||||
const existingBlock = blocks[existingIndex];
|
||||
const existingTools = existingBlock.data.generic_tools || [];
|
||||
const newTools = log.data.generic_tools || [];
|
||||
const mergedTools = [...existingTools];
|
||||
newTools.forEach((nt: any) => {
|
||||
const matchIdx = mergedTools.findIndex(et => et.trace_id && nt.trace_id && et.trace_id === nt.trace_id);
|
||||
if (matchIdx > -1) {
|
||||
mergedTools[matchIdx] = { ...mergedTools[matchIdx], ...nt };
|
||||
} else {
|
||||
mergedTools.push(nt);
|
||||
}
|
||||
});
|
||||
|
||||
const mergedData = {
|
||||
...existingBlock.data,
|
||||
...log.data,
|
||||
operator_native_thinking: log.data.operator_native_thinking || existingBlock.data.operator_native_thinking || undefined,
|
||||
operator_raw_thinking: log.data.operator_raw_thinking || existingBlock.data.operator_raw_thinking || undefined,
|
||||
action_taken: log.data.action_taken || existingBlock.data.action_taken || undefined,
|
||||
last_execution_result: log.data.last_execution_result || existingBlock.data.last_execution_result || undefined,
|
||||
generic_tools: mergedTools,
|
||||
isCompleted: true
|
||||
};
|
||||
|
||||
blocks[existingIndex] = {
|
||||
...existingBlock,
|
||||
id: `step-${stepId}`,
|
||||
type: 'step',
|
||||
timestamp: existingBlock.timestamp || log.timestamp,
|
||||
data: mergedData
|
||||
};
|
||||
} else {
|
||||
blocks.push({
|
||||
id: `step-${stepId}`,
|
||||
type: 'step',
|
||||
timestamp: log.timestamp,
|
||||
data: {
|
||||
...log.data,
|
||||
isCompleted: true,
|
||||
generic_tools: log.data.generic_tools || []
|
||||
}
|
||||
});
|
||||
}
|
||||
} else if (log.type === 'trace_recorded') {
|
||||
const isAction = log.data.type === 'action';
|
||||
const isTool = log.data.type === 'tool' || isAction;
|
||||
const isFailedLLM = log.data.type === 'llm_call' && log.data.status === 'failed';
|
||||
if (!isTool && !isFailedLLM) return;
|
||||
|
||||
let stepId = log.data.step_id;
|
||||
let existingIndex = -1;
|
||||
|
||||
// 1. If this trace_id already exists in some block, update that block directly
|
||||
if (log.data.trace_id) {
|
||||
existingIndex = blocks.findIndex(b =>
|
||||
b.data?.generic_tools?.some((t: any) => t.trace_id === log.data.trace_id)
|
||||
);
|
||||
}
|
||||
|
||||
// 2. If not found by trace_id and stepId is present, find by step_id
|
||||
if (existingIndex === -1 && stepId) {
|
||||
existingIndex = blocks.findIndex(b => b.id === `step-${stepId}` || b.data?.step_id === stepId);
|
||||
}
|
||||
|
||||
// 3. If still not found and no stepId (e.g. pre-planning / initial planning phase)
|
||||
if (existingIndex === -1 && !stepId) {
|
||||
existingIndex = blocks.findIndex(b => b.id === 'step-pre-planning' || b.data?.step_id === 'pre-planning');
|
||||
}
|
||||
|
||||
// 4. Otherwise find by timestamp among existing step blocks
|
||||
if (existingIndex === -1) {
|
||||
const logTime = log.timestamp ? new Date(log.timestamp).getTime() : 0;
|
||||
let maxStepTime = -1;
|
||||
for (let i = 0; i < blocks.length; i++) {
|
||||
const b = blocks[i];
|
||||
const bTime = b.timestamp ? new Date(b.timestamp).getTime() : 0;
|
||||
if (bTime <= logTime && bTime >= maxStepTime) {
|
||||
maxStepTime = bTime;
|
||||
existingIndex = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (existingIndex > -1) {
|
||||
const stepData = blocks[existingIndex].data;
|
||||
const genericTools = stepData.generic_tools ? [...stepData.generic_tools] : [];
|
||||
const existingToolIdx = log.data.trace_id
|
||||
? genericTools.findIndex((t: any) => t.trace_id === log.data.trace_id)
|
||||
: -1;
|
||||
|
||||
if (existingToolIdx > -1) {
|
||||
genericTools[existingToolIdx] = {
|
||||
...genericTools[existingToolIdx],
|
||||
...log.data
|
||||
};
|
||||
} else {
|
||||
genericTools.push(log.data);
|
||||
}
|
||||
|
||||
blocks[existingIndex] = {
|
||||
...blocks[existingIndex],
|
||||
data: {
|
||||
...stepData,
|
||||
generic_tools: genericTools
|
||||
}
|
||||
};
|
||||
} else {
|
||||
const fallbackStepId = stepId || 'pre-planning';
|
||||
blocks.push({
|
||||
id: `step-${fallbackStepId}`,
|
||||
type: 'step',
|
||||
timestamp: log.timestamp,
|
||||
data: {
|
||||
step_id: fallbackStepId,
|
||||
isCompleted: true,
|
||||
generic_tools: [log.data]
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
blocks.sort((a, b) => {
|
||||
const timeA = a.timestamp ? new Date(a.timestamp).getTime() : 0;
|
||||
const timeB = b.timestamp ? new Date(b.timestamp).getTime() : 0;
|
||||
if (Math.abs(timeA - timeB) > 50) {
|
||||
return timeA - timeB;
|
||||
}
|
||||
// Order by step_number if available (0 is planning phase)
|
||||
const getStepOrder = (block: StepBlock): number => {
|
||||
if (block.data?.step_number !== undefined && block.data.step_number !== null) {
|
||||
return Number(block.data.step_number);
|
||||
}
|
||||
if (block.data?.step_id === 'pre-planning' || block.data?.step_type === 'planning') {
|
||||
return 0;
|
||||
}
|
||||
return 99999;
|
||||
};
|
||||
|
||||
const stepNumA = getStepOrder(a);
|
||||
const stepNumB = getStepOrder(b);
|
||||
if (stepNumA !== stepNumB && stepNumA !== 99999 && stepNumB !== 99999) {
|
||||
return stepNumA - stepNumB;
|
||||
}
|
||||
return timeA - timeB;
|
||||
});
|
||||
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Safely extract and aggregate token usage for a step block
|
||||
*/
|
||||
export function extractBlockTokens(block: StepBlock): { total: number; prompt: number; completion: number } {
|
||||
if (!block || !block.data) return { total: 0, prompt: 0, completion: 0 };
|
||||
const data = block.data;
|
||||
|
||||
// 1. Direct total_tokens or token_usage object
|
||||
if (data.total_tokens !== undefined && data.total_tokens !== null && Number(data.total_tokens) > 0) {
|
||||
const pr = data.token_usage?.prompt_tokens || data.token_usage?.input_tokens || 0;
|
||||
const co = data.token_usage?.completion_tokens || data.token_usage?.output_tokens || 0;
|
||||
return { total: Number(data.total_tokens), prompt: Number(pr), completion: Number(co) };
|
||||
}
|
||||
if (data.token_usage && typeof data.token_usage === 'object') {
|
||||
const pr = data.token_usage.prompt_tokens || data.token_usage.input_tokens || 0;
|
||||
const co = data.token_usage.completion_tokens || data.token_usage.output_tokens || 0;
|
||||
const to = data.token_usage.total_tokens || (pr + co);
|
||||
if (to > 0) {
|
||||
return { total: Number(to), prompt: Number(pr), completion: Number(co) };
|
||||
}
|
||||
}
|
||||
if (data.extra_metadata?.token_usage && typeof data.extra_metadata.token_usage === 'object') {
|
||||
const u = data.extra_metadata.token_usage;
|
||||
const pr = u.prompt_tokens || u.prompt_token_count || u.input_tokens || 0;
|
||||
const co = u.completion_tokens || u.candidates_token_count || u.output_tokens || 0;
|
||||
const to = u.total_tokens || u.total_token_count || (pr + co);
|
||||
if (to > 0) {
|
||||
return { total: Number(to), prompt: Number(pr), completion: Number(co) };
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Generic tools traces (e.g. llm_call traces or tool payloads)
|
||||
let sumPrompt = 0;
|
||||
let sumCompletion = 0;
|
||||
let sumTotal = 0;
|
||||
if (data.generic_tools && Array.isArray(data.generic_tools)) {
|
||||
data.generic_tools.forEach((t: any) => {
|
||||
if (t && t.payload) {
|
||||
const u = t.payload.token_usage || t.payload.usage_metadata;
|
||||
if (u) {
|
||||
const pr = u.prompt_tokens || u.prompt_token_count || u.input_tokens || 0;
|
||||
const co = u.completion_tokens || u.candidates_token_count || u.output_tokens || 0;
|
||||
const to = u.total_tokens || u.total_token_count || (pr + co);
|
||||
sumPrompt += Number(pr);
|
||||
sumCompletion += Number(co);
|
||||
sumTotal += Number(to);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
if (sumTotal > 0) {
|
||||
return { total: sumTotal, prompt: sumPrompt, completion: sumCompletion };
|
||||
}
|
||||
|
||||
// 3. Fallback estimate from thoughts/messages/actions text if no tokens stored
|
||||
let textLen = 0;
|
||||
let numImages = 0;
|
||||
if (data.operator_native_thinking) textLen += data.operator_native_thinking.length;
|
||||
if (data.operator_raw_thinking) textLen += data.operator_raw_thinking.length;
|
||||
if (data.generic_tools && Array.isArray(data.generic_tools)) {
|
||||
data.generic_tools.forEach((t: any) => {
|
||||
if (t.payload) {
|
||||
const payloadStr = typeof t.payload === 'string' ? t.payload : JSON.stringify(t.payload);
|
||||
if (payloadStr.includes('data:image/') || payloadStr.includes('base64,')) {
|
||||
numImages += 1;
|
||||
} else {
|
||||
textLen += payloadStr.length;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
if (textLen > 20 || numImages > 0) {
|
||||
const estimated = Math.round(textLen / 4) + (numImages * 258);
|
||||
return { total: estimated, prompt: 0, completion: estimated };
|
||||
}
|
||||
|
||||
return { total: 0, prompt: 0, completion: 0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Format token count with clean units (e.g. "842 tokens", "76.4k tokens", "1.2M tokens")
|
||||
*/
|
||||
export function formatTokenCount(tokens?: number): string {
|
||||
if (!tokens || tokens <= 0) return '0 tokens';
|
||||
if (tokens < 1000) {
|
||||
return `${tokens} tokens`;
|
||||
}
|
||||
if (tokens < 1_000_000) {
|
||||
const inK = (tokens / 1000).toFixed(1).replace(/\.0$/, '');
|
||||
return `${inK}k tokens`;
|
||||
}
|
||||
const inM = (tokens / 1_000_000).toFixed(2).replace(/\.?0+$/, '');
|
||||
return `${inM}M tokens`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Group step blocks into chronological execution phases (segmented per work unit)
|
||||
*/
|
||||
export function groupBlocksToPhases(blocks: StepBlock[], sessionStartTime: number): PhaseBlock[] {
|
||||
if (!blocks || blocks.length === 0) return [];
|
||||
const phases: PhaseBlock[] = [];
|
||||
|
||||
// Base start timestamp in ms
|
||||
let previousEndTime = sessionStartTime ? sessionStartTime * 1000 : 0;
|
||||
|
||||
blocks.forEach((block, index) => {
|
||||
const blockTime = block.timestamp ? new Date(block.timestamp).getTime() : Date.now();
|
||||
|
||||
if (previousEndTime === 0 && blockTime > 0) {
|
||||
previousEndTime = blockTime;
|
||||
}
|
||||
|
||||
let durationSeconds = 0;
|
||||
if (block.data?.duration && typeof block.data.duration === 'number' && block.data.duration > 0) {
|
||||
durationSeconds = Math.max(1, Math.round(block.data.duration));
|
||||
} else if (blockTime >= previousEndTime && previousEndTime > 0) {
|
||||
durationSeconds = Math.max(1, Math.round((blockTime - previousEndTime) / 1000));
|
||||
} else {
|
||||
durationSeconds = 1;
|
||||
}
|
||||
|
||||
const tokensInfo = extractBlockTokens(block);
|
||||
|
||||
phases.push({
|
||||
id: `phase-${block.id || index}`,
|
||||
durationSeconds,
|
||||
tokens: tokensInfo.total > 0 ? tokensInfo.total : undefined,
|
||||
promptTokens: tokensInfo.prompt > 0 ? tokensInfo.prompt : undefined,
|
||||
completionTokens: tokensInfo.completion > 0 ? tokensInfo.completion : undefined,
|
||||
blocks: [block]
|
||||
});
|
||||
|
||||
previousEndTime = blockTime;
|
||||
});
|
||||
|
||||
return phases;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all actions and tools inside a step card, sorted in true chronological sequence by timestamp
|
||||
*/
|
||||
export function getSortedStepEvents(
|
||||
stepData: any,
|
||||
cache?: WeakMap<any, { length: number; actionTs: any; events: StepEvent[] }>
|
||||
): StepEvent[] {
|
||||
if (!stepData || typeof stepData !== 'object') return [];
|
||||
const toolsLen = stepData.generic_tools ? stepData.generic_tools.length : 0;
|
||||
const actionTs = stepData.action_taken ? (stepData.action_taken.timestamp || stepData.action_taken.start_time || stepData.action_taken.created_at) : null;
|
||||
|
||||
if (cache) {
|
||||
const cached = cache.get(stepData);
|
||||
if (cached && cached.length === toolsLen && cached.actionTs === actionTs) {
|
||||
return cached.events;
|
||||
}
|
||||
}
|
||||
|
||||
const events: Array<{ type: 'action' | 'tool'; data: any; timestamp: number }> = [];
|
||||
const fallbackTime = getItemTimestamp(stepData.timestamp, 0);
|
||||
|
||||
// 1. Add action_taken if present and valid
|
||||
if (stepData.action_taken && (isAndroidAction(stepData.action_taken) || isReportStatusAction(stepData.action_taken))) {
|
||||
const actTime = getItemTimestamp(
|
||||
stepData.action_taken.timestamp ?? stepData.action_taken.start_time ?? stepData.action_taken.created_at,
|
||||
fallbackTime
|
||||
);
|
||||
events.push({
|
||||
type: 'action',
|
||||
data: stepData.action_taken,
|
||||
timestamp: actTime
|
||||
});
|
||||
}
|
||||
|
||||
// 2. Add generic tools if present
|
||||
const uniqueTools = getUniqueGenericTools(stepData.generic_tools);
|
||||
uniqueTools.forEach((tool: any) => {
|
||||
const toolTime = getItemTimestamp(
|
||||
tool.timestamp ?? tool.start_time ?? tool.created_at,
|
||||
fallbackTime
|
||||
);
|
||||
events.push({
|
||||
type: 'tool',
|
||||
data: tool,
|
||||
timestamp: toolTime
|
||||
});
|
||||
});
|
||||
|
||||
// 3. Sort chronologically by timestamp
|
||||
events.sort((a, b) => a.timestamp - b.timestamp);
|
||||
|
||||
const result: StepEvent[] = events.map(e => ({ type: e.type, data: e.data }));
|
||||
if (cache) {
|
||||
cache.set(stepData, { length: toolsLen, actionTs, events: result });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile human-readable session summary from logs
|
||||
*/
|
||||
export function compileSessionSummary(logs: any[]): string {
|
||||
if (!logs || logs.length === 0) return 'No logs available.';
|
||||
|
||||
// Check for goal completed / checker response / cancellation / report_task_status
|
||||
for (let i = logs.length - 1; i >= 0; i--) {
|
||||
const log = logs[i];
|
||||
if (log.type === 'session_ended') {
|
||||
if (log.data?.status === 'cancelled' || log.data?.was_stopped_manually) {
|
||||
return 'Task stopped manually.';
|
||||
}
|
||||
}
|
||||
if (log.type === 'llm_stream' && log.data?.text) {
|
||||
const text = log.data.text;
|
||||
if (text.includes('```json') && text.includes('"success"')) {
|
||||
return 'Execution completed and verified by checker.';
|
||||
}
|
||||
}
|
||||
if (log.type === 'step_recorded' || log.type === 'step_updated') {
|
||||
const act = log.data?.action_taken;
|
||||
if (act && isReportStatusAction(act)) {
|
||||
const exp = getReportStatusExplanation(act);
|
||||
if (exp) return exp;
|
||||
}
|
||||
if (log.data?.status === 'completed') {
|
||||
return log.data.message || 'Task completed successfully.';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 'Execution session in progress or ended.';
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate total session duration in seconds
|
||||
*/
|
||||
export function computeSessionDuration(sessionStartTime: number, logs: any[]): number {
|
||||
if (!sessionStartTime || !logs || logs.length === 0) return 0;
|
||||
const startMs = sessionStartTime * 1000;
|
||||
const lastLog = logs[logs.length - 1];
|
||||
const endMs = lastLog?.timestamp ? new Date(lastLog.timestamp).getTime() : Date.now();
|
||||
return Math.max(0, Math.round((endMs - startMs) / 1000));
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the planning loader should be visible (while running, waiting for next step/action and not actively typing a stream)
|
||||
*/
|
||||
export function checkPlanningLoader(
|
||||
logs: any[],
|
||||
isRunning: boolean,
|
||||
isCurrentRunningSession: boolean = true
|
||||
): boolean {
|
||||
if (!isRunning || !isCurrentRunningSession) return false;
|
||||
if (!logs || logs.length === 0) return true;
|
||||
|
||||
// If the latest step or event indicates task completion, do not show loader
|
||||
for (let i = logs.length - 1; i >= Math.max(0, logs.length - 3); i--) {
|
||||
const log = logs[i];
|
||||
if (log.type === 'session_ended') return false;
|
||||
if (log.type === 'step_recorded' || log.type === 'step_updated') {
|
||||
const act = log.data?.action_taken;
|
||||
if (act && (act.action === 'report_task_status' || act === 'report_task_status' || isReportStatusAction(act))) {
|
||||
return false;
|
||||
}
|
||||
if (log.data?.status === 'completed' || log.data?.status === 'failed') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If there is an active incomplete stream currently streaming chunks, hide the planning loader
|
||||
const hasActiveStream = logs.some(
|
||||
l => l && l.type === 'llm_stream' && l.data && l.data.isCompleted === false
|
||||
);
|
||||
|
||||
if (hasActiveStream) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,956 @@
|
||||
/**
|
||||
* Copyright 2026 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ActionParam } from '../core/models/stream.model';
|
||||
import { isAndroidAction, isReportStatusAction } from './action-formatter.util';
|
||||
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util';
|
||||
|
||||
/**
|
||||
* Extract arguments from tool payload or direct args field
|
||||
*/
|
||||
export function getToolArgs(tool: any): any {
|
||||
if (!tool) return {};
|
||||
if (tool.payload) {
|
||||
let payloadObj = tool.payload;
|
||||
if (typeof payloadObj === 'string') {
|
||||
try { payloadObj = JSON.parse(payloadObj); } catch {}
|
||||
}
|
||||
if (payloadObj && typeof payloadObj === 'object') {
|
||||
if (payloadObj.args && typeof payloadObj.args === 'object') {
|
||||
return payloadObj.args;
|
||||
}
|
||||
return payloadObj;
|
||||
}
|
||||
}
|
||||
if (tool.args) {
|
||||
let argsObj = tool.args;
|
||||
if (typeof argsObj === 'string') {
|
||||
try { argsObj = JSON.parse(argsObj); } catch {}
|
||||
}
|
||||
if (argsObj && typeof argsObj === 'object') {
|
||||
return argsObj;
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a tool is one of the five note-related tools
|
||||
*/
|
||||
export function isNoteTool(tool: any): boolean {
|
||||
if (!tool || !tool.name) return false;
|
||||
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '');
|
||||
const nameLower = cleanName.toLowerCase();
|
||||
return ['save_note', 'read_note', 'list_notes', 'update_note', 'append_note'].includes(nameLower);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a tool is video analysis related
|
||||
*/
|
||||
export function isVideoTool(tool: any): boolean {
|
||||
if (!tool || !tool.name) return false;
|
||||
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '');
|
||||
const nameLower = cleanName.toLowerCase();
|
||||
return ['video_analyzer', 'video_analyzer_pure'].includes(nameLower);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract target video filename or description for video analysis tools
|
||||
*/
|
||||
export function getVideoToolTarget(tool: any): string | null {
|
||||
if (!tool) return null;
|
||||
const args = getToolArgs(tool);
|
||||
if (args.video_path || args.video_file || args.file_path || args.file) {
|
||||
const p = String(args.video_path || args.video_file || args.file_path || args.file);
|
||||
return p.split('/').pop() || p;
|
||||
}
|
||||
if (args.time_description) {
|
||||
return String(args.time_description);
|
||||
}
|
||||
if (args.start_time !== undefined || args.end_time !== undefined) {
|
||||
const start = args.start_time ?? 0;
|
||||
const end = args.end_time ? `${args.end_time}s` : 'end';
|
||||
return `${start}s - ${end}`;
|
||||
}
|
||||
if (args.purpose) {
|
||||
return String(args.purpose);
|
||||
}
|
||||
return 'screen_record.mp4';
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical list of mobile device automation and ADB command tool names.
|
||||
*/
|
||||
export const DEVICE_ACTION_TOOL_NAMES = new Set([
|
||||
'click', 'click_sequence', 'long_press', 'long_press_on', 'input', 'input_text',
|
||||
'swipe', 'scroll', 'press_key', 'press_home', 'press_back', 'launch_app', 'manage_app', 'open_app',
|
||||
'focus_and_input_text', 'focus_and_clear_text', 'tap', 'wait_for_delay', 'wait',
|
||||
'run_adb_command', 'run_short_adb_command'
|
||||
]);
|
||||
|
||||
/**
|
||||
* Check if a tool represents a real mobile device action or ADB command.
|
||||
*/
|
||||
export function isDeviceActionTool(tool: any): boolean {
|
||||
if (!tool || !tool.name) return false;
|
||||
if (tool.type === 'llm_call') return false;
|
||||
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '').toLowerCase();
|
||||
return DEVICE_ACTION_TOOL_NAMES.has(cleanName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Backward-compatible alias for isDeviceActionTool.
|
||||
*/
|
||||
export function isFailureAnalyzerActionTool(tool: any): boolean {
|
||||
return isDeviceActionTool(tool);
|
||||
}
|
||||
|
||||
/**
|
||||
* Framework-internal infrastructure tools that are not intended for end-user timeline display.
|
||||
*/
|
||||
export function isInternalPlumbingTool(tool: any): boolean {
|
||||
if (!tool || !tool.name) return true;
|
||||
const cleanName = tool.name.replace(/^(_)?(self\.)?exec_/, '').toLowerCase();
|
||||
if (cleanName.includes('safety_net')) return true;
|
||||
if (cleanName === 'report_task_status' || cleanName === 'report_status' || cleanName === 'submit_task_status') {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines whether a tool invocation should be displayed in the timeline.
|
||||
*/
|
||||
export function shouldShowTool(tool: any, stepData?: any): boolean {
|
||||
if (!tool || !tool.name) return false;
|
||||
if (tool.type === 'llm_call') return false;
|
||||
if (tool.type === 'agent') return false;
|
||||
if (isInternalPlumbingTool(tool)) return false;
|
||||
|
||||
// If this tool is an operator/validator atomic action that duplicates stepData.action_taken, skip it
|
||||
if (stepData && stepData.action_taken && isAndroidAction(stepData.action_taken) && isDeviceActionTool(tool)) {
|
||||
const agentName = (tool.agent_name || tool.agent || '').toLowerCase();
|
||||
const isSelfHealingOrSubAgent = agentName.includes('failure') ||
|
||||
agentName.includes('analyzer') ||
|
||||
agentName.includes('diagnos') ||
|
||||
(agentName && agentName !== 'operator' && agentName !== 'flashrunner' && agentName !== 'validator');
|
||||
if (!isSelfHealingOrSubAgent) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts human-readable initiating agent name / role for a tool.
|
||||
*/
|
||||
export function getToolAgentName(tool: any): string | null {
|
||||
if (!tool) return null;
|
||||
const name = (tool.agent_name || tool.agent || '').toLowerCase();
|
||||
if (name.includes('failure') || name.includes('analyzer')) {
|
||||
return null; // Omit self-healing label per user instruction
|
||||
}
|
||||
if (name.includes('outputter')) {
|
||||
return 'Outputter';
|
||||
}
|
||||
if (name.includes('validator')) {
|
||||
return 'Validator';
|
||||
}
|
||||
if (name.includes('diagnos')) {
|
||||
return 'Diagnoser';
|
||||
}
|
||||
if (name.includes('explorer')) {
|
||||
return 'Explorer';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter out nested child wrapper duplicates while preserving independent executions at the same level
|
||||
*/
|
||||
export function getUniqueGenericTools(tools: any[] | undefined): any[] {
|
||||
if (!tools) return [];
|
||||
|
||||
const toolMap = new Map<string, any>();
|
||||
for (const t of tools) {
|
||||
if (t && t.trace_id) {
|
||||
toolMap.set(t.trace_id, t);
|
||||
}
|
||||
}
|
||||
|
||||
const cleanName = (nameStr: string) => (nameStr || '').replace(/^(_)?(self\.)?exec_/, '').toLowerCase().trim();
|
||||
|
||||
const result: any[] = [];
|
||||
const seenTraces = new Set<string>();
|
||||
|
||||
for (const tool of tools) {
|
||||
if (!tool || !tool.name) continue;
|
||||
if (tool.type === 'agent') continue;
|
||||
|
||||
if (tool.type === 'llm_call' && tool.status === 'failed') {
|
||||
result.push(tool);
|
||||
continue;
|
||||
}
|
||||
|
||||
const currClean = cleanName(tool.name);
|
||||
|
||||
// Check if this tool is a nested child execution of a parent tool with the exact same tool name
|
||||
let isNestedChildDuplicate = false;
|
||||
if (tool.parent_trace_id) {
|
||||
const parentTool = toolMap.get(tool.parent_trace_id);
|
||||
if (parentTool && cleanName(parentTool.name) === currClean) {
|
||||
isNestedChildDuplicate = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!isNestedChildDuplicate) {
|
||||
if (tool.trace_id) {
|
||||
if (!seenTraces.has(tool.trace_id)) {
|
||||
seenTraces.add(tool.trace_id);
|
||||
// Always use the latest state from toolMap for this trace_id
|
||||
result.push(toolMap.get(tool.trace_id) || tool);
|
||||
}
|
||||
} else {
|
||||
result.push(tool);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get note key for the tool if it exists
|
||||
*/
|
||||
export function getToolKey(tool: any): string | null {
|
||||
if (!tool || !tool.name) return null;
|
||||
const args = tool.payload?.args || tool.args;
|
||||
const key = args?.key;
|
||||
if (!key) return null;
|
||||
return key.toLowerCase().endsWith('.md') ? key : `${key}.md`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get display label for tools (e.g. for pills or headers)
|
||||
*/
|
||||
export function getToolDisplayLabel(tool: any, isFirstSaveNote: boolean = false): string {
|
||||
if (!tool || !tool.name) return '';
|
||||
const cleanName = tool.name.replace(/^(_)?exec_/, '');
|
||||
const nameLower = cleanName.toLowerCase();
|
||||
const args = getToolArgs(tool);
|
||||
|
||||
switch (nameLower) {
|
||||
case 'manage_app':
|
||||
case 'launch_app': {
|
||||
const rawApp = args.app_name || args.package_name || args.app || '';
|
||||
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : 'Application';
|
||||
const rawAction = args.action ? String(args.action).toLowerCase() : '';
|
||||
const verb = rawAction === 'launch' ? 'Launching' : (rawAction === 'stop' || rawAction === 'close' ? 'Stopping' : 'Managing');
|
||||
return `${verb} "${app}"`;
|
||||
}
|
||||
|
||||
case 'wait_for_delay':
|
||||
case 'wait_delay':
|
||||
case 'wait': {
|
||||
const delay = args.delay_seconds || args.seconds || args.delay || args.duration;
|
||||
return delay ? `Waiting for ${delay} second${Number(delay) > 1 ? 's' : ''}...` : 'Waiting for delay...';
|
||||
}
|
||||
case 'wait_for_text': {
|
||||
const text = args.text || args.target_text || '';
|
||||
return text ? `Waiting for text "${text}" to appear on screen` : 'Waiting for text on screen';
|
||||
}
|
||||
|
||||
case 'input_text':
|
||||
case 'input': {
|
||||
const text = args.text || args.input_text || '';
|
||||
return text ? `Entering text "${text}" into field` : 'Entering text into input field';
|
||||
}
|
||||
case 'focus_and_input_text': {
|
||||
const text = args.text || args.input_text || '';
|
||||
return text ? `Focusing field and typing "${text}"` : 'Focusing field and entering text';
|
||||
}
|
||||
case 'focus_and_clear_text':
|
||||
return 'Focusing and clearing field text';
|
||||
|
||||
case 'click':
|
||||
case 'tap': {
|
||||
const target = args.target_text || args.text || args.query || '';
|
||||
return target ? `Tapping on "${target}"` : 'Tapping on screen element';
|
||||
}
|
||||
case 'click_sequence':
|
||||
return 'Executing click sequence';
|
||||
case 'long_press': {
|
||||
const target = args.target_text || args.text || '';
|
||||
return target ? `Long pressing on "${target}"` : 'Long pressing screen element';
|
||||
}
|
||||
case 'swipe': {
|
||||
const dir = args.action || args.direction || '';
|
||||
return dir ? `Swiping ${String(dir).toUpperCase()} on screen` : 'Swiping screen';
|
||||
}
|
||||
case 'press_key': {
|
||||
const key = args.key || args.keycode || '';
|
||||
return key ? `Pressing key ${String(key).toUpperCase()}` : 'Pressing hardware key';
|
||||
}
|
||||
|
||||
case 'save_note':
|
||||
return isFirstSaveNote ? 'Creating note' : 'Saving note';
|
||||
case 'read_note':
|
||||
return 'Reading note';
|
||||
case 'list_notes':
|
||||
return 'Browsing all saved notes';
|
||||
case 'update_note':
|
||||
return 'Updating note';
|
||||
case 'append_note':
|
||||
return 'Updating note';
|
||||
|
||||
case 'object_detection': {
|
||||
const q = Array.isArray(args.queries) ? args.queries.join(', ') : (args.queries || '');
|
||||
return q ? `Locating on screen: "${q}"` : 'Locating elements on screen';
|
||||
}
|
||||
case 'ask_explorer': {
|
||||
const query = args.query || args.prompt || '';
|
||||
return query ? `Searching on screen: "${query}"` : 'Searching on screen';
|
||||
}
|
||||
case 'report_failure_analysis': {
|
||||
const reason = args.reason || args.analysis || '';
|
||||
return reason ? `Investigating issue: ${reason}` : 'Investigating execution issue';
|
||||
}
|
||||
case 'run_adb_command':
|
||||
case 'run_short_adb_command': {
|
||||
const cmd = args.command || args.cmd || '';
|
||||
return cmd ? `Running command: ${cmd}` : 'Running system command';
|
||||
}
|
||||
case 'search_logs':
|
||||
case 'read_logs': {
|
||||
const q = args.query || args.filter || '';
|
||||
return q ? `Searching logs for "${q}"` : 'Analyzing system logs';
|
||||
}
|
||||
case 'log_analyzer':
|
||||
case 'output_analyzer':
|
||||
return 'Analyzing logs';
|
||||
case 'diagnoser':
|
||||
case 'diagnose':
|
||||
return 'Diagnosing issue';
|
||||
case 'video_analyzer':
|
||||
case 'video_analyzer_pure':
|
||||
return 'Analyzing screen recording';
|
||||
case 'extract_segment_metadata': {
|
||||
const start = args.start_time !== undefined ? `${args.start_time}s` : '';
|
||||
const end = args.end_time !== undefined ? `${args.end_time}s` : '';
|
||||
const range = (start && end) ? ` (${start} - ${end})` : (start ? ` (from ${start})` : '');
|
||||
return `Cropping screen recording segment${range}`;
|
||||
}
|
||||
case 'spawn_sub_agent': {
|
||||
const q = args.specific_query || args.query || args.prompt || '';
|
||||
return q ? `Analyzing recording with sub-agent: "${q}"` : 'Analyzing recording with sub-agent';
|
||||
}
|
||||
case 'analyze_audio_only': {
|
||||
const q = args.specific_query || args.query || '';
|
||||
return q ? `Analyzing audio track: "${q}"` : 'Analyzing recording audio track';
|
||||
}
|
||||
case 'get_step_details':
|
||||
return 'Retrieving step execution details';
|
||||
case 'get_step_screenshot':
|
||||
return 'Inspecting screenshot evidence';
|
||||
case 'search_history_for_text': {
|
||||
const q = args.text || args.query || '';
|
||||
return q ? `Searching execution trace for "${q}"` : 'Searching execution trace';
|
||||
}
|
||||
case 'outputter':
|
||||
case 'output_synthesis':
|
||||
return 'Synthesizing output report';
|
||||
case 'web_search': {
|
||||
const q = args.query || '';
|
||||
return q ? `Searching web for "${q}"` : 'Searching the web';
|
||||
}
|
||||
case 'read_url':
|
||||
return 'Fetching web page';
|
||||
|
||||
default:
|
||||
return `Executing ${cleanName.split('_').map((w: string) => w.charAt(0).toUpperCase() + w.slice(1)).join(' ')}`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Material Symbol icon for tools
|
||||
*/
|
||||
export function getToolIcon(tool: any): string {
|
||||
if (!tool || !tool.name) return 'settings';
|
||||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||||
switch (name) {
|
||||
case 'click':
|
||||
case 'tap':
|
||||
return 'ads_click';
|
||||
case 'click_sequence':
|
||||
case 'long_press':
|
||||
return 'touch_app';
|
||||
case 'input_text':
|
||||
case 'input':
|
||||
return 'keyboard';
|
||||
case 'swipe':
|
||||
return 'swipe';
|
||||
case 'press_key':
|
||||
return 'keyboard_tab';
|
||||
case 'manage_app':
|
||||
return 'open_in_new';
|
||||
case 'wait_for_delay':
|
||||
return 'timer';
|
||||
case 'wait_for_text':
|
||||
return 'hourglass_empty';
|
||||
case 'object_detection':
|
||||
return 'search';
|
||||
case 'ask_explorer':
|
||||
return 'search';
|
||||
case 'report_failure_analysis':
|
||||
return 'assessment';
|
||||
case 'run_adb_command':
|
||||
case 'run_short_adb_command':
|
||||
return 'terminal';
|
||||
case 'web_search':
|
||||
return 'travel_explore';
|
||||
case 'read_url':
|
||||
return 'language';
|
||||
case 'search_logs':
|
||||
case 'read_logs':
|
||||
case 'log_analyzer':
|
||||
case 'output_analyzer':
|
||||
return 'receipt_long';
|
||||
case 'diagnoser':
|
||||
case 'diagnose':
|
||||
return 'medical_services';
|
||||
case 'video_analyzer':
|
||||
case 'video_analyzer_pure':
|
||||
return 'video_camera_back';
|
||||
case 'extract_segment_metadata':
|
||||
return 'crop';
|
||||
case 'spawn_sub_agent':
|
||||
return 'smart_toy';
|
||||
case 'analyze_audio_only':
|
||||
return 'graphic_eq';
|
||||
case 'get_step_details':
|
||||
return 'manage_search';
|
||||
case 'get_step_screenshot':
|
||||
return 'image_search';
|
||||
case 'search_history_for_text':
|
||||
return 'find_in_page';
|
||||
case 'outputter':
|
||||
case 'output_synthesis':
|
||||
return 'assignment_turned_in';
|
||||
default:
|
||||
return 'build';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get formatted title for a tool call card
|
||||
*/
|
||||
export function getToolTitle(tool: any): string {
|
||||
if (!tool || !tool.name) return 'Tool Call';
|
||||
const cleanName = tool.name.replace(/^(_)?exec_/, '');
|
||||
const name = cleanName.toLowerCase();
|
||||
switch (name) {
|
||||
case 'click':
|
||||
case 'tap':
|
||||
return 'Tapping Element';
|
||||
case 'click_sequence':
|
||||
return 'Executing Click Sequence';
|
||||
case 'long_press':
|
||||
return 'Long Pressing Element';
|
||||
case 'input_text':
|
||||
case 'input':
|
||||
return 'Entering Text';
|
||||
case 'swipe':
|
||||
case 'scroll': {
|
||||
const args = getToolArgs(tool);
|
||||
const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '');
|
||||
if (dir && isPureDirectionString(dir)) return `Swiping Screen (${String(dir).toUpperCase()})`;
|
||||
return 'Swiping Screen';
|
||||
}
|
||||
case 'drag':
|
||||
case 'drag_and_drop':
|
||||
return 'Dragging Screen';
|
||||
case 'press_key':
|
||||
return 'Pressing Hardware Key';
|
||||
case 'manage_app':
|
||||
case 'launch_app': {
|
||||
const args = getToolArgs(tool);
|
||||
const rawAction = args.action ? String(args.action).toLowerCase() : '';
|
||||
if (rawAction === 'launch') return 'Launching Application';
|
||||
if (rawAction === 'stop' || rawAction === 'close') return 'Stopping Application';
|
||||
return 'Managing Application';
|
||||
}
|
||||
case 'wait_for_delay':
|
||||
case 'wait_delay':
|
||||
return 'Waiting for Delay';
|
||||
case 'wait_for_text':
|
||||
return 'Waiting for Text';
|
||||
case 'object_detection':
|
||||
return 'Locating Elements';
|
||||
case 'ask_explorer':
|
||||
return 'Searching on Screen';
|
||||
case 'report_failure_analysis':
|
||||
return 'Investigating Issue';
|
||||
case 'run_adb_command':
|
||||
case 'run_short_adb_command':
|
||||
return 'Running System Command';
|
||||
case 'web_search':
|
||||
return 'Web Search';
|
||||
case 'read_url':
|
||||
return 'Fetching Web Page';
|
||||
case 'search_logs':
|
||||
case 'read_logs':
|
||||
return 'Searching Logs';
|
||||
case 'log_analyzer':
|
||||
case 'output_analyzer':
|
||||
return 'Analyzing Logs';
|
||||
case 'diagnoser':
|
||||
case 'diagnose':
|
||||
return 'Diagnosing Issue';
|
||||
case 'video_analyzer':
|
||||
case 'video_analyzer_pure':
|
||||
return 'Analyzing Screen Recording';
|
||||
case 'extract_segment_metadata':
|
||||
return 'Cropping Screen Recording';
|
||||
case 'spawn_sub_agent':
|
||||
return 'Delegating Video Analysis';
|
||||
case 'analyze_audio_only':
|
||||
return 'Analyzing Audio Track';
|
||||
default:
|
||||
return cleanName.replace(/_/g, ' ').replace(/\b\w/g, (c: string) => c.toUpperCase());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get target text description for tools
|
||||
*/
|
||||
export function getToolTargetText(tool: any): string {
|
||||
if (!tool || !tool.name) return '';
|
||||
const args = getToolArgs(tool);
|
||||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||||
if (name === 'manage_app' || name === 'launch_app') {
|
||||
const rawApp = args.app_name || args.package_name || args.app || '';
|
||||
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : '';
|
||||
const rawAct = args.action ? String(args.action).toLowerCase() : '';
|
||||
const act = rawAct ? (rawAct.charAt(0).toUpperCase() + rawAct.slice(1)) : '';
|
||||
if (act && app) {
|
||||
return `${act} ${app}`;
|
||||
}
|
||||
return app || act || '';
|
||||
}
|
||||
if (name === 'swipe' && typeof args.action === 'string' && isPureDirectionString(args.action)) {
|
||||
return args.action;
|
||||
}
|
||||
if (name === 'wait_for_text') {
|
||||
return args.text || '';
|
||||
}
|
||||
if (name === 'object_detection') {
|
||||
if (Array.isArray(args.queries)) return args.queries.join(', ');
|
||||
return args.queries || '';
|
||||
}
|
||||
if (name === 'ask_explorer' || name === 'web_search') {
|
||||
return args.query || args.prompt || '';
|
||||
}
|
||||
if (name === 'read_url') {
|
||||
return args.url || '';
|
||||
}
|
||||
if (name === 'search_logs' || name === 'read_logs') {
|
||||
return args.query || args.filter || '';
|
||||
}
|
||||
if (name === 'report_failure_analysis') {
|
||||
return args.status || args.reason || args.analysis || '';
|
||||
}
|
||||
if (args.target && typeof args.target === 'string') {
|
||||
return args.target;
|
||||
}
|
||||
if (args.target && typeof args.target === 'number') {
|
||||
return `Element #${args.target}`;
|
||||
}
|
||||
if (args.index !== undefined) {
|
||||
return `Element #${args.index}`;
|
||||
}
|
||||
return args.target_text || args.target_class || args.element || args.element_text || (name !== 'input_text' ? args.text : '') || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get input label for tools
|
||||
*/
|
||||
export function getToolInputLabel(tool: any): string {
|
||||
if (!tool || !tool.name) return 'Input';
|
||||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||||
if (name === 'wait_for_delay' || name === 'wait_delay' || name === 'delay' || name === 'wait') {
|
||||
return 'Duration';
|
||||
}
|
||||
if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') {
|
||||
const args = getToolArgs(tool);
|
||||
const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '');
|
||||
if (dir && isPureDirectionString(dir)) {
|
||||
return 'Direction';
|
||||
}
|
||||
return 'Input';
|
||||
}
|
||||
if (name === 'press_key' || name === 'press_home' || name === 'press_back') {
|
||||
return 'Key';
|
||||
}
|
||||
if (name === 'input_text' || name === 'input') {
|
||||
return 'Input Text';
|
||||
}
|
||||
return 'Input';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get input value for tools
|
||||
*/
|
||||
export function getToolInputText(tool: any): string {
|
||||
if (!tool || !tool.name) return '';
|
||||
const args = getToolArgs(tool);
|
||||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||||
if (name === 'press_key') {
|
||||
return args.key || args.keycode || '';
|
||||
}
|
||||
if (name === 'swipe' || name === 'scroll' || name === 'drag' || name === 'drag_and_drop') {
|
||||
const dir = args.direction || args.gesture || (typeof args.action === 'string' ? args.action : '');
|
||||
if (dir && isPureDirectionString(dir)) {
|
||||
return String(dir).toUpperCase();
|
||||
}
|
||||
return '';
|
||||
}
|
||||
if (name === 'wait_for_delay' || name === 'wait_delay') {
|
||||
return args.time_in_ms ? `${args.time_in_ms}ms` : (args.delay_ms ? `${args.delay_ms}ms` : (args.time ? `${args.time}` : ''));
|
||||
}
|
||||
if (name === 'long_press' && args.duration) {
|
||||
return `Duration: ${args.duration}ms`;
|
||||
}
|
||||
if (name === 'input_text' || name === 'input') {
|
||||
return args.text || args.input_text || '';
|
||||
}
|
||||
return args.input_text || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get coordinates string representation for tools
|
||||
*/
|
||||
export function getToolCoords(tool: any): string {
|
||||
if (!tool || !tool.name) return '';
|
||||
const args = getToolArgs(tool);
|
||||
const cleanName = tool.name.replace(/^(_)?exec_/, '');
|
||||
const name = cleanName.toLowerCase();
|
||||
|
||||
// Check sequence first if action is sequence or sequence arg is present
|
||||
const isSequenceAction = name === 'click_sequence' || name === 'tap_sequence' || Boolean(args.sequence || args.normalized_sequence);
|
||||
const rawSeq = args.normalized_sequence || args.sequence || args.targets || (Array.isArray(args.coordinates) && Array.isArray(args.coordinates[0]) ? args.coordinates : null);
|
||||
const seqPoints = parseSequenceCoordinates(rawSeq);
|
||||
if (seqPoints && seqPoints.length > 0 && (isSequenceAction || seqPoints.length > 1)) {
|
||||
return seqPoints.map(pt => `[${pt[0]}, ${pt[1]}]`).join(' → ');
|
||||
}
|
||||
|
||||
// Check normalized start and end first
|
||||
const normStart = extractNumbersFromCoordinateValue(args.normalized_start_coordinates || args.start_coordinates || args.start);
|
||||
const normEnd = extractNumbersFromCoordinateValue(args.normalized_end_coordinates || args.end_coordinates || args.end);
|
||||
if (normStart && normEnd && normStart.length === 2 && normEnd.length === 2) {
|
||||
return `[${normStart[0]}, ${normStart[1]}] → [${normEnd[0]}, ${normEnd[1]}]`;
|
||||
}
|
||||
|
||||
const coords = extractNumbersFromCoordinateValue(args.normalized_coordinates) ||
|
||||
extractNumbersFromCoordinateValue(args.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(args.target) ||
|
||||
extractNumbersFromCoordinateValue(args.action) ||
|
||||
extractNumbersFromCoordinateValue(args.gesture) ||
|
||||
extractNumbersFromCoordinateValue(args.point);
|
||||
|
||||
if (coords && Array.isArray(coords)) {
|
||||
if (coords.length === 4) {
|
||||
return `[${coords[0]}, ${coords[1]}] → [${coords[2]}, ${coords[3]}]`;
|
||||
}
|
||||
if (coords.length === 2) {
|
||||
return `[${coords[0]}, ${coords[1]}]`;
|
||||
}
|
||||
return coords.join(', ');
|
||||
}
|
||||
|
||||
if (args.x !== undefined && args.y !== undefined) {
|
||||
return `[${args.x}, ${args.y}]`;
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get analysis / reasoning text for tools
|
||||
*/
|
||||
export function getToolAnalysisText(tool: any): string {
|
||||
if (!tool || !tool.name) return '';
|
||||
const args = getToolArgs(tool);
|
||||
return args.analysis || args.reasoning || args.summary || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if tool is ADB command
|
||||
*/
|
||||
export function isAdbCommandTool(tool: any): boolean {
|
||||
if (!tool || !tool.name) return false;
|
||||
const name = tool.name.toLowerCase().replace(/^(_)?exec_/, '');
|
||||
return name === 'run_adb_command' || name === 'run_short_adb_command';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get ADB command line string
|
||||
*/
|
||||
export function getAdbCommandLine(tool: any): string {
|
||||
const args = getToolArgs(tool);
|
||||
return args.CommandLine || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get ADB working directory
|
||||
*/
|
||||
export function getAdbCwd(tool: any): string {
|
||||
const args = getToolArgs(tool);
|
||||
return args.Cwd || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get ADB requested terminal id
|
||||
*/
|
||||
export function getAdbTerminalId(tool: any): string {
|
||||
const args = getToolArgs(tool);
|
||||
return args.RequestedTerminalID || '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get fallback generic details for unspecified tools
|
||||
*/
|
||||
export function getToolGenericDetails(tool: any): string {
|
||||
if (!tool) return '';
|
||||
if (isAdbCommandTool(tool)) {
|
||||
return '';
|
||||
}
|
||||
if (getToolTargetText(tool) || getToolInputText(tool) || getToolCoords(tool) || getToolAnalysisText(tool)) {
|
||||
return '';
|
||||
}
|
||||
const args = getToolArgs(tool);
|
||||
if (!args || typeof args !== 'object') return '';
|
||||
const ignoredKeys = new Set(['state', 'controller', 'ctx', 'session_id', 'step_id', 'trace_id', 'parent_trace_id', 'times', 'delay_ms']);
|
||||
const entries = Object.entries(args).filter(([k, v]) => !ignoredKeys.has(k) && v !== undefined && v !== null && v !== '');
|
||||
if (entries.length === 0) return '';
|
||||
return entries.map(([k, v]) => `${k}: ${typeof v === 'object' ? JSON.stringify(v) : v}`).join(', ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if tool execution failed
|
||||
*/
|
||||
export function isToolFailed(tool: any): boolean {
|
||||
if (!tool) return false;
|
||||
if (tool.status === 'failed' || tool.status === 'error') return true;
|
||||
const args = getToolArgs(tool);
|
||||
if (args.status === 'failed' || args.status === 'error' || args.status === 'cannot_fix') return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get error message for a failed tool
|
||||
*/
|
||||
export function getToolErrorMessage(tool: any): string {
|
||||
if (!tool) return 'Tool Failed';
|
||||
const args = getToolArgs(tool);
|
||||
if (args.status === 'cannot_fix') return 'Status: cannot_fix';
|
||||
if (args.error || args.message || args.failure_reason) {
|
||||
return args.error || args.message || args.failure_reason;
|
||||
}
|
||||
if (tool.error || tool.message) return tool.error || tool.message;
|
||||
return 'Action Failed';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract extra parameters for generic tools
|
||||
*/
|
||||
export function extractToolExtraParams(toolData: any, cache?: WeakMap<any, ActionParam[]>): ActionParam[] {
|
||||
if (!toolData) return [];
|
||||
const payload = toolData.payload || toolData.args || toolData;
|
||||
if (!payload || typeof payload !== 'object') return [];
|
||||
if (cache && cache.has(payload)) {
|
||||
return cache.get(payload)!;
|
||||
}
|
||||
|
||||
const standardKeys = new Set([
|
||||
'action', 'name', 'type', 'target_text', 'text', 'input_text', 'target',
|
||||
'coordinates', 'coords', 'target_bounds', 'bounds', 'target_resource_id',
|
||||
'resource_id', 'target_class', 'class_name', 'normalized_coordinates',
|
||||
'pre_image_name', 'post_image_name', 'pre_screenshot', 'post_screenshot',
|
||||
'before_screenshot', 'after_screenshot', 'status', 'success', 'timestamp',
|
||||
'created_at', 'start_time', 'execution_id', 'controller', 'agent', 'session_id', 'step_id',
|
||||
'app_name', 'package_name', 'app', 'key', 'keycode', 'time_in_ms', 'delay_ms', 'delay_seconds', 'duration',
|
||||
'args', 'kwargs', 'parameters', 'extra_params', 'payload', 'trace_id', 'result', 'error', 'analysis', 'details', 'command', 'cwd', 'terminal_id'
|
||||
]);
|
||||
|
||||
const result: ActionParam[] = [];
|
||||
|
||||
const rawArgs = toolData.args || toolData.Args || toolData.kwargs || toolData.parameters || toolData.payload;
|
||||
let parsedArgs: any = {};
|
||||
if (rawArgs) {
|
||||
if (typeof rawArgs === 'object') {
|
||||
parsedArgs = rawArgs;
|
||||
} else if (typeof rawArgs === 'string') {
|
||||
try {
|
||||
parsedArgs = JSON.parse(rawArgs);
|
||||
} catch {
|
||||
if (!rawArgs.includes('<') && !rawArgs.includes('object at')) {
|
||||
parsedArgs = { details: rawArgs };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const mergedObj = { ...toolData, ...(typeof payload === 'object' ? payload : {}), ...parsedArgs };
|
||||
|
||||
for (const [k, v] of Object.entries(mergedObj)) {
|
||||
const lowerK = k.toLowerCase();
|
||||
if (standardKeys.has(lowerK)) continue;
|
||||
if (v === null || v === undefined || v === '') continue;
|
||||
|
||||
let valStr = String(v);
|
||||
if (valStr.includes('object at 0x') || valStr.startsWith('<artemis.') || valStr.includes('<controller') || valStr.includes('<android_world')) continue;
|
||||
|
||||
if (typeof v === 'object') {
|
||||
try { valStr = JSON.stringify(v); } catch { valStr = String(v); }
|
||||
}
|
||||
|
||||
let prettyKey = k.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
|
||||
if (lowerK === 'time_in_ms' || lowerK === 'delay_ms') prettyKey = 'Delay';
|
||||
|
||||
result.push({ key: prettyKey, value: valStr });
|
||||
}
|
||||
|
||||
if (cache) {
|
||||
cache.set(payload, result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if text is genuine human thinking rather than raw JSON payload output
|
||||
*/
|
||||
export function isHumanThinking(text: string | null): boolean {
|
||||
if (!text) return false;
|
||||
let trimmed = text.trim();
|
||||
if (!trimmed) return false;
|
||||
|
||||
// Strip markdown code blocks if wrapped: ```json ... ``` or ``` ... ```
|
||||
if (trimmed.startsWith('```')) {
|
||||
const match = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i);
|
||||
if (match) {
|
||||
trimmed = match[1].trim();
|
||||
}
|
||||
}
|
||||
|
||||
// Direct object or array start
|
||||
if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Try parsing as JSON if it looks like JSON structures
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
if (typeof parsed === 'object' && parsed !== null) {
|
||||
return false;
|
||||
}
|
||||
} catch {
|
||||
// Not valid standalone JSON, continue checks
|
||||
}
|
||||
|
||||
// Check for common JSON object detection / grounding / action / tool patterns
|
||||
if (/^\s*\{.*"label"\s*:.*"results"\s*:/s.test(trimmed) ||
|
||||
/^\s*\{.*"point"\s*:.*"label"\s*:/s.test(trimmed) ||
|
||||
/^\s*\{.*"action"\s*:/s.test(trimmed) ||
|
||||
/^\s*\{.*"name"\s*:/s.test(trimmed) ||
|
||||
/^\s*\{\s*"[^"]+"\s*:\s*/.test(trimmed) ||
|
||||
/^\s*\[\s*\{\s*"[^"]+"\s*:\s*/.test(trimmed)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up raw error messages for display in the UI
|
||||
*/
|
||||
export function cleanErrorMessage(rawError: any): string {
|
||||
if (typeof rawError === 'object' && rawError !== null) {
|
||||
if (rawError.message && typeof rawError.message === 'string') {
|
||||
return cleanErrorMessage(rawError.message);
|
||||
}
|
||||
if (rawError.error?.message && typeof rawError.error?.message === 'string') {
|
||||
return cleanErrorMessage(rawError.error.message);
|
||||
}
|
||||
if (rawError.detail && typeof rawError.detail === 'string') {
|
||||
return cleanErrorMessage(rawError.detail);
|
||||
}
|
||||
try {
|
||||
return JSON.stringify(rawError);
|
||||
} catch {
|
||||
return String(rawError);
|
||||
}
|
||||
}
|
||||
|
||||
const errorStr = String(rawError).trim();
|
||||
if (!errorStr) return 'Unknown error';
|
||||
|
||||
// 1. Try regex extraction for "message": "..."
|
||||
const doubleQuoteMsgMatch = errorStr.match(/"message"\s*:\s*"((?:[^"\\]|\\.)*)"/i);
|
||||
if (doubleQuoteMsgMatch && doubleQuoteMsgMatch[1]) {
|
||||
const unescaped = doubleQuoteMsgMatch[1].replace(/\\"/g, '"').replace(/\\n/g, ' ').trim();
|
||||
if (unescaped) return unescaped;
|
||||
}
|
||||
|
||||
// 2. Try regex extraction for 'message': '...'
|
||||
const singleQuoteMsgMatch = errorStr.match(/'message'\s*:\s*['"]((?:[^'\\]|\\.)*)['"]/i);
|
||||
if (singleQuoteMsgMatch && singleQuoteMsgMatch[1]) {
|
||||
const unescaped = singleQuoteMsgMatch[1].replace(/\\'/g, "'").replace(/\\n/g, ' ').trim();
|
||||
if (unescaped) return unescaped;
|
||||
}
|
||||
|
||||
// 3. Try parsing JSON substrings inside the error string
|
||||
const jsonMatch = errorStr.match(/\{[\s\S]*\}/);
|
||||
if (jsonMatch) {
|
||||
try {
|
||||
const normalizedJsonStr = jsonMatch[0]
|
||||
.replace(/'/g, '"')
|
||||
.replace(/True/g, 'true')
|
||||
.replace(/False/g, 'false')
|
||||
.replace(/None/g, 'null');
|
||||
const parsed = JSON.parse(normalizedJsonStr);
|
||||
if (parsed?.message) return String(parsed.message);
|
||||
if (parsed?.error?.message) return String(parsed.error.message);
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Fallback: truncate at trace dump headers or return clean message
|
||||
let fallback = errorStr;
|
||||
if (fallback.includes('=== Source Location Trace')) {
|
||||
fallback = fallback.split('=== Source Location Trace')[0];
|
||||
}
|
||||
if (fallback.includes('[type.googleapis.com')) {
|
||||
fallback = fallback.split('[type.googleapis.com')[0];
|
||||
}
|
||||
fallback = fallback
|
||||
.replace(/^Pre-execution validation failed:\s*/i, '')
|
||||
.replace(/^Pixel-level validation failed:\s*/i, '')
|
||||
.replace(/^Execution error:\s*/i, '')
|
||||
.replace(/^ServerError:\s*/i, '')
|
||||
.trim();
|
||||
|
||||
return fallback || errorStr;
|
||||
}
|
||||
Reference in new issue
Block a user