feat(plugins): add source-aware evidence and recording timelines
This commit is contained in:
1 parent
72ff342605
commit
ef7af9988e
61 files changed
+3847
-108
No files matched your search
@@ -2,13 +2,109 @@ import {
|
||||
getStepPreImageUrl,
|
||||
getStepPostImageUrl,
|
||||
extractStepReplayFrames,
|
||||
getActionTitle,
|
||||
getActionCoords,
|
||||
getActionTargetText,
|
||||
isActionFailed,
|
||||
getActionErrorMessage,
|
||||
extractActionExtraParams
|
||||
} from './action-formatter.util';
|
||||
|
||||
describe('plugin replay marker descriptions', () => {
|
||||
it('does not parse digits in real hashed click and observation names as coordinates', () => {
|
||||
const click = { action: 'plug_96c522c2e2_click', plugin_id: 'open-computer-use', args: { app: 'Demo' } };
|
||||
const observe = { action: 'plug_5f6ec6ac87_get_app_state', plugin_id: 'open-computer-use', args: { app: 'Demo' } };
|
||||
expect(getActionTitle(click)).toBe('点击');
|
||||
expect(getActionTitle(observe)).toBe('观察电脑窗口');
|
||||
expect(getActionCoords(click)).toBe('');
|
||||
expect(getActionCoords(observe)).toBe('');
|
||||
expect(getActionTargetText(observe)).toBe('Demo');
|
||||
expect(getActionCoords({ ...click, args: { app: 'Demo', x: 100, y: 200 } })).toBe('[100, 200]');
|
||||
});
|
||||
|
||||
it('uses the original tool name from the trace rather than its dynamic binding name', () => {
|
||||
const trace = { name: 'plug_96c522c2e2_click', type: 'tool',
|
||||
payload: { plugin_id: 'desktop', tool_name: 'double_click', args: { x: 70, y: 80 } } };
|
||||
expect(getActionTitle(trace)).toBe('双击');
|
||||
expect(getActionCoords(trace)).toBe('[70, 80]');
|
||||
});
|
||||
|
||||
it('counts a Flash action and its tool trace as one frame using their shared call id', () => {
|
||||
const callId = 'tool-call-1';
|
||||
const action = { action: 'plug_96c522c2e2_click', call_id: callId, plugin_id: 'open-computer-use',
|
||||
source_id: 'computer-window', args: { app: 'Demo', x: 100, y: 200 } };
|
||||
const trace = { type: 'tool', name: action.action, trace_id: 'different-trace-uuid', timestamp: 109,
|
||||
payload: { plugin_id: action.plugin_id, call_id: callId, source_id: action.source_id,
|
||||
args: action.args, post_image_name: 'computer-shot' } };
|
||||
const frames = extractStepReplayFrames([{ step_id: 'step-4', step_number: 4, timestamp: 109,
|
||||
action_taken: action, generic_tools: [trace], post_image_name: 'computer-shot' }]);
|
||||
expect(frames.length).toBe(1);
|
||||
expect(frames[0].sourceId).toBe('computer-window');
|
||||
expect(frames[0].postImageUrl).toBe('/images/computer-shot');
|
||||
expect(frames[0].actionText).toBe('点击 ([100, 200])');
|
||||
expect(frames[0].rawStepNumber).toBe(4);
|
||||
});
|
||||
|
||||
it('preserves repeated same-source actions when they have different call ids', () => {
|
||||
const trace = (callId: string, timestamp: number) => ({ type: 'tool', name: 'plug_96c522c2e2_click',
|
||||
trace_id: `trace-${callId}`, timestamp, payload: { plugin_id: 'desktop', call_id: callId,
|
||||
source_id: 'computer-window', args: { x: 100, y: 200 }, post_image_name: `shot-${callId}` } });
|
||||
const first = trace('one', 101);
|
||||
const second = trace('two', 102);
|
||||
const frames = extractStepReplayFrames([{ step_id: 'multi', step_number: 1, timestamp: 101,
|
||||
action_taken: first, generic_tools: [first, second] }]);
|
||||
expect(frames.length).toBe(2);
|
||||
expect(frames.map(frame => frame.timestamp)).toEqual([101, 102]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('action-formatter.util screenshot chaining', () => {
|
||||
for (const update of [
|
||||
{ pre_image_name: null },
|
||||
{ pre_screenshot: null },
|
||||
{ last_execution_result: { pre_image_valid: false } }
|
||||
]) {
|
||||
it(`clears invalidated before-image aliases in merged step updates: ${JSON.stringify(update)}`, () => {
|
||||
const frames = extractStepReplayFrames([
|
||||
{ type: 'step_recorded', data: {
|
||||
step_id: 'cross-window', step_number: 1, timestamp: 100,
|
||||
pre_image_name: 'window-a', pre_screenshot: 'legacy-window-a',
|
||||
action_taken: { action: 'click' }
|
||||
} },
|
||||
{ type: 'step_updated', data: {
|
||||
step_id: 'cross-window', post_image_name: 'window-b', ...update
|
||||
} },
|
||||
{ type: 'step_updated', data: { step_id: 'cross-window', summary: 'Completed' } }
|
||||
]);
|
||||
expect(frames.length).toBe(1);
|
||||
expect(frames[0].preImageUrl).toBeNull();
|
||||
expect(frames[0].postImageUrl).toBe('/images/window-b');
|
||||
expect(frames[0].imageUrl).toBe('/images/window-b');
|
||||
});
|
||||
}
|
||||
|
||||
it('retains a valid before-image when an update omits image fields', () => {
|
||||
const frames = extractStepReplayFrames([
|
||||
{ step_id: 'same-window', step_number: 1, timestamp: 100,
|
||||
pre_image_name: 'before', action_taken: { action: 'click' } },
|
||||
{ step_id: 'same-window', summary: 'Completed', post_image_name: 'after' }
|
||||
]);
|
||||
expect(frames[0].preImageUrl).toBe('/images/before');
|
||||
expect(frames[0].postImageUrl).toBe('/images/after');
|
||||
});
|
||||
|
||||
it('keeps distinct windows from the same plugin as distinct evidence and replay frames', () => {
|
||||
const first = { type: 'tool', trace_id: 'w1', name: 'plug_click', timestamp: 101,
|
||||
payload: { plugin_id: 'desktop', source_id: 'window-1', post_image_name: 'one' } };
|
||||
const second = { type: 'tool', trace_id: 'w2', name: 'plug_click', timestamp: 102,
|
||||
payload: { plugin_id: 'desktop', source_id: 'window-2', post_image_name: 'two' } };
|
||||
const step = { step_id: 'multi', step_number: 1, timestamp: 101, action_taken: first, generic_tools: [first, second] };
|
||||
expect(getStepPostImageUrl(step, first)).toBe('/images/one');
|
||||
expect(getStepPostImageUrl(step, second)).toBe('/images/two');
|
||||
const frames = extractStepReplayFrames([step]);
|
||||
expect(frames.map(frame => frame.sourceId)).toEqual(['window-1', 'window-2']);
|
||||
expect(frames.map(frame => frame.postImageUrl)).toEqual(['/images/one', '/images/two']);
|
||||
});
|
||||
it('keeps plugin observations separate from the phone image chain', () => {
|
||||
const desktop = { type: 'tool', trace_id: 'desktop', name: 'plug_observe', timestamp: 1001,
|
||||
payload: { plugin_id: 'desktop-control', post_image_name: 'desktop-screen' } };
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
import { ActionParam, StepReplayFrame } from '../core/models/stream.model';
|
||||
import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates, unwrapTraceAction } from './image-overlay.util';
|
||||
import { cleanErrorMessage, joinTargetDescriptions } from './tool-formatter.util';
|
||||
import { getPluginToolView } from './plugin-tool-view.util';
|
||||
|
||||
/**
|
||||
* Safely parse JSON string if it looks like an object/array, otherwise return original value
|
||||
@@ -166,6 +167,8 @@ export function getActionIcon(action: any): string {
|
||||
export function getActionTitle(action: any): string {
|
||||
const act = getActionObject(action);
|
||||
if (!act) return "操作";
|
||||
const plugin = getPluginToolView(act);
|
||||
if (plugin) return plugin.title;
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
switch (name) {
|
||||
case 'tap':
|
||||
@@ -233,6 +236,11 @@ export function getActionTargetText(action: any): string {
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
const args = act.args && typeof act.args === 'object' ? act.args : {};
|
||||
|
||||
if (getPluginToolView(act)) {
|
||||
return act.target_text || act.target_description || args.target_text || args.target_description
|
||||
|| act.app_name || act.app || args.app_name || args.app || '';
|
||||
}
|
||||
|
||||
if (name.includes('app')) {
|
||||
const rawApp = act.app_name || act.package_name || act.app || args.app_name || args.package_name || args.app || '';
|
||||
const app = rawApp ? (rawApp.charAt(0).toUpperCase() + rawApp.slice(1)) : '';
|
||||
@@ -321,6 +329,7 @@ export function getActionCoords(action: any): string {
|
||||
if (!act) return '';
|
||||
const args = act.args && typeof act.args === 'object' ? act.args : {};
|
||||
const name = (act.name || act.action || '').toLowerCase();
|
||||
const plugin = getPluginToolView(act) !== null;
|
||||
|
||||
// Check sequence first if action is sequence or sequence arg is present
|
||||
const isSequenceAction = name === 'click_sequence' || name === 'tap_sequence' || Boolean(act.sequence || args.sequence || act.normalized_sequence || args.normalized_sequence);
|
||||
@@ -344,15 +353,18 @@ export function getActionCoords(action: any): string {
|
||||
}
|
||||
|
||||
const normCoords = extractNumbersFromCoordinateValue(act.normalized_coordinates || args.normalized_coordinates);
|
||||
const point = [act.x ?? args.x, act.y ?? args.y];
|
||||
const explicitPoint = point.every(value => typeof value === 'number' && Number.isFinite(value)) ? point : null;
|
||||
const coords = normCoords ||
|
||||
explicitPoint ||
|
||||
extractNumbersFromCoordinateValue(act.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(args.coordinates) ||
|
||||
extractNumbersFromCoordinateValue(act.coords) ||
|
||||
extractNumbersFromCoordinateValue(args.coords) ||
|
||||
extractNumbersFromCoordinateValue(act.target) ||
|
||||
extractNumbersFromCoordinateValue(args.target) ||
|
||||
extractNumbersFromCoordinateValue(args.action) ||
|
||||
extractNumbersFromCoordinateValue(act.action) ||
|
||||
extractNumbersFromCoordinateValue(plugin ? null : args.action) ||
|
||||
extractNumbersFromCoordinateValue(plugin ? null : act.action) ||
|
||||
extractNumbersFromCoordinateValue(args.gesture) ||
|
||||
extractNumbersFromCoordinateValue(act.gesture);
|
||||
|
||||
@@ -613,6 +625,29 @@ function pluginSource(item: any): string | null {
|
||||
return act?.plugin_id || payload?.plugin_id || payload?.action?.plugin_id || result?.plugin_id || null;
|
||||
}
|
||||
|
||||
function toolCallIdentity(item: any): string {
|
||||
const act = getActionObject(item);
|
||||
const payload = safeParseJson(act?.payload);
|
||||
const result = safeParseJson(act?.result || payload?.result);
|
||||
return String(act?.call_id || payload?.call_id || result?.call_id || act?.trace_id || '');
|
||||
}
|
||||
|
||||
export function actionEvidenceSource(item: any): { id: string; label: string } {
|
||||
const act = getActionObject(item);
|
||||
const payload = safeParseJson(act?.payload) || {};
|
||||
const result = safeParseJson(act?.result || payload?.result) || {};
|
||||
const source = act?.source || payload?.source || payload?.action?.source || result?.source;
|
||||
const plugin = pluginSource(item);
|
||||
const android = !item || isAndroidAction(act) || isReportStatusAction(act);
|
||||
const recording = act?.recording || result?.recording;
|
||||
const recordingLabel = recording?.status === 'recording' ? (recording.scope === 'screen' ? ' · 屏幕录制中' : ' · 窗口录制中') : recording?.status === 'failed' ? ' · 录像异常' : '';
|
||||
const labels: Record<string, string> = { computer: '电脑', android: '安卓', terminal: '终端', api: '接口', file: '文件', other: '插件' };
|
||||
return {
|
||||
id: act?.source_id || payload?.source_id || payload?.action?.source_id || result?.source_id || (plugin ? `plugin:${plugin}` : android ? 'android' : 'auxiliary'),
|
||||
label: (source ? (labels[source.kind] || '插件') + (source.label ? ` · ${source.label}` : '') : (plugin ? `插件 · ${plugin}` : android ? '安卓' : '辅助')) + recordingLabel
|
||||
};
|
||||
}
|
||||
|
||||
export function extractItemPreImage(item: any): string | null {
|
||||
const act = getActionObject(item);
|
||||
if (!act) return null;
|
||||
@@ -792,7 +827,7 @@ export function resolveStepImageMap(stepData: any): Map<any, ResolvedImageEntry>
|
||||
}
|
||||
|
||||
// Ensure stepData.action_taken cross-resolves to the action images
|
||||
if (stepData.action_taken && lastActionItem && pluginSource(stepData.action_taken) === pluginSource(lastActionItem)) {
|
||||
if (stepData.action_taken && lastActionItem && actionEvidenceSource(stepData.action_taken).id === actionEvidenceSource(lastActionItem).id) {
|
||||
const lastEntry = map.get(lastActionItem);
|
||||
if (lastEntry) {
|
||||
map.set(stepData.action_taken, lastEntry);
|
||||
@@ -905,19 +940,23 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
|
||||
|
||||
const mergedStepNum = (stepNum !== null) ? stepNum : existing.step_number;
|
||||
const mergedStepId = existing.step_id || step.step_id;
|
||||
// Explicit invalidation must clear both aliases, otherwise an old window
|
||||
// screenshot can reappear through the legacy pre_screenshot fallback.
|
||||
const clearPreImage = safeParseJson(step.last_execution_result)?.pre_image_valid === false ||
|
||||
(!preferExistingAction && (step.pre_image_name === null || step.pre_screenshot === null));
|
||||
|
||||
const merged = {
|
||||
...existing,
|
||||
...step,
|
||||
step_id: mergedStepId,
|
||||
step_number: mergedStepNum,
|
||||
pre_image_name: preferExistingAction
|
||||
pre_image_name: clearPreImage ? null : preferExistingAction
|
||||
? existing.pre_image_name
|
||||
: (step.pre_image_name || existing.pre_image_name),
|
||||
post_image_name: preferExistingAction
|
||||
? (existing.post_image_name || step.post_image_name)
|
||||
: (step.post_image_name || existing.post_image_name),
|
||||
pre_screenshot: preferExistingAction
|
||||
pre_screenshot: clearPreImage ? null : preferExistingAction
|
||||
? existing.pre_screenshot
|
||||
: (step.pre_screenshot || existing.pre_screenshot),
|
||||
post_screenshot: preferExistingAction
|
||||
@@ -978,6 +1017,20 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
|
||||
primaryImg
|
||||
});
|
||||
}
|
||||
// A Pro step may contain actions from several targets. Each tool retains its
|
||||
// own frame and timestamp instead of borrowing the step's last phone image.
|
||||
const seen = new Set<string>([toolCallIdentity(act)]);
|
||||
for (const tool of stepData.generic_tools || []) {
|
||||
if (!pluginSource(tool)) continue;
|
||||
const key = toolCallIdentity(tool);
|
||||
if (tool === act || (key && seen.has(key))) continue;
|
||||
if (key) seen.add(key);
|
||||
const toolPre = getStepPreImageUrl(stepData, tool);
|
||||
const toolPost = getStepPostImageUrl(stepData, tool);
|
||||
if (!toolPre && !toolPost) continue;
|
||||
visualCandidates.push({ stepData: { ...stepData, timestamp: tool.timestamp || stepData.timestamp }, act: tool,
|
||||
preUrl: toolPre, postUrl: toolPost, primaryImg: toolPost || toolPre! });
|
||||
}
|
||||
}
|
||||
|
||||
// Sort strictly by original step_number ascending, then by timestamp
|
||||
@@ -1004,6 +1057,8 @@ export function extractStepReplayFrames(logsOrSteps: any[]): StepReplayFrame[] {
|
||||
: stepNum;
|
||||
|
||||
frames.push({
|
||||
sourceId: actionEvidenceSource(act).id,
|
||||
sourceLabel: actionEvidenceSource(act).label,
|
||||
index: i,
|
||||
stepNumber: stepNum,
|
||||
rawStepNumber: rawStepNum,
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import { actionEvidenceSource, getStepPostImageUrl, getStepPreImageUrl } from './action-formatter.util';
|
||||
import { isDeviceActionTool } from './tool-formatter.util';
|
||||
import { getPluginToolView } from './plugin-tool-view.util';
|
||||
|
||||
describe('plugin evidence timeline cards', () => {
|
||||
const observation = () => ({
|
||||
name: 'plug_abcdef1234_get_app_state', type: 'tool', status: 'success',
|
||||
payload: {
|
||||
plugin_id: 'desktop', tool_name: 'get_app_state', effect: 'read', observation: true,
|
||||
args: { app: 'Demo' }, source_id: 'plugin:desktop:window-1',
|
||||
source: { kind: 'computer', label: 'Demo' }, post_image_name: 'desktop-screen',
|
||||
pre_image_valid: false,
|
||||
result: { plugin_id: 'desktop', execution: 'completed', side_effect: 'none',
|
||||
content: [{ type: 'text', text: '当前窗口已截图' }] }
|
||||
}
|
||||
});
|
||||
|
||||
it('renders real dynamic plugin observations with computer evidence, not Android actions', () => {
|
||||
const tool = observation();
|
||||
expect(isDeviceActionTool(tool)).toBeFalse();
|
||||
expect(getPluginToolView(tool)).toEqual(jasmine.objectContaining({
|
||||
title: '观察电脑窗口', kind: '辅助观察', auxiliary: true,
|
||||
status: '已执行', result: '当前窗口已截图', args: [{ key: 'app', value: 'Demo' }]
|
||||
}));
|
||||
expect(actionEvidenceSource(tool)).toEqual({ id: 'plugin:desktop:window-1', label: '电脑 · Demo' });
|
||||
const step = { pre_image_name: 'phone-before', post_image_name: 'phone-after', generic_tools: [tool] };
|
||||
expect(getStepPreImageUrl(step, tool)).toBeNull();
|
||||
expect(getStepPostImageUrl(step, tool)).toContain('desktop-screen');
|
||||
});
|
||||
|
||||
it('uses the declared write contract even before a tool result arrives', () => {
|
||||
const view = getPluginToolView({ name: 'plug_abcdef1234_click', type: 'tool', status: 'running',
|
||||
payload: { plugin_id: 'desktop', tool_name: 'click', effect: 'write', observation: false,
|
||||
args: { x: 30, y: 40 } } });
|
||||
expect(view).toEqual(jasmine.objectContaining({ title: '点击', kind: '插件操作', auxiliary: false, status: '执行中' }));
|
||||
});
|
||||
|
||||
it('keeps read-only file calls folded without labelling them as screen observations', () => {
|
||||
const view = getPluginToolView({ name: 'plug_abcdef1234_read_file',
|
||||
payload: { plugin_id: 'files', effect: 'read', observation: false, args: { path: 'result.txt' } } });
|
||||
expect(view).toEqual(jasmine.objectContaining({ kind: '辅助读取', auxiliary: true, status: '待返回' }));
|
||||
});
|
||||
|
||||
it('uses stored legacy results and keeps unavailable outcomes uncertain', () => {
|
||||
const view = getPluginToolView({ name: 'plug_abcdef1234_click', type: 'tool', status: 'failed',
|
||||
payload: JSON.stringify({ plugin_id: 'desktop', args: { x: 30 },
|
||||
result: { tool: 'click', execution: 'unknown', side_effect: 'possible', message: '连接中断,需重新观察' } }) });
|
||||
expect(view).toEqual(jasmine.objectContaining({ kind: '插件操作', auxiliary: false,
|
||||
status: '结果不确定', result: '连接中断,需重新观察', args: [{ key: 'x', value: '30' }] }));
|
||||
});
|
||||
|
||||
it('does not guess write intent from a dynamic tool name with missing metadata', () => {
|
||||
expect(getPluginToolView({ name: 'plug_abcdef1234_click', status: 'running' }))
|
||||
.toEqual(jasmine.objectContaining({ kind: '插件调用', auxiliary: true, args: [] }));
|
||||
});
|
||||
|
||||
it('does not treat ordinary Android or model tools as plugin calls', () => {
|
||||
expect(getPluginToolView({ name: 'click', type: 'tool', payload: { args: { x: 30 } } })).toBeNull();
|
||||
expect(getPluginToolView({ name: 'plug_abcdef1234_click', type: 'llm_call' })).toBeNull();
|
||||
});
|
||||
|
||||
it('reflects a completed receipt even when stream updates reuse the tool object', () => {
|
||||
const tool: any = { name: 'plug_abcdef1234_click', status: 'running',
|
||||
payload: { plugin_id: 'desktop', effect: 'write', args: {} } };
|
||||
expect(getPluginToolView(tool)?.status).toBe('执行中');
|
||||
tool.status = 'success';
|
||||
tool.payload.result = { execution: 'completed', message: '已点击' };
|
||||
expect(getPluginToolView(tool)?.status).toBe('已执行');
|
||||
expect(getPluginToolView(tool)?.result).toBe('已点击');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,58 @@
|
||||
export interface PluginToolView {
|
||||
title: string;
|
||||
kind: string;
|
||||
auxiliary: boolean;
|
||||
result: string;
|
||||
status: string;
|
||||
args: Array<{ key: string; value: string }>;
|
||||
}
|
||||
|
||||
function objectValue(value: any): Record<string, any> {
|
||||
if (typeof value === 'string') {
|
||||
try { value = JSON.parse(value); } catch { return {}; }
|
||||
}
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
||||
}
|
||||
|
||||
/** Display plugin calls from their declared contract, never from Android tool names. */
|
||||
export function getPluginToolView(tool: any): PluginToolView | null {
|
||||
if (!tool || typeof tool !== 'object' || tool.type === 'llm_call') return null;
|
||||
const payload = objectValue(tool.payload);
|
||||
const result = objectValue(payload['result'] ?? tool.result);
|
||||
const pluginId = payload['plugin_id'] || result['plugin_id'] || tool.plugin_id;
|
||||
if (!pluginId && !/^plug_[a-f0-9]{10}_/.test(tool.name || tool.action || '')) {
|
||||
return null;
|
||||
}
|
||||
const name = String(payload['tool_name'] || result['tool'] || tool.name || tool.action || '插件调用')
|
||||
.replace(/^plug_[a-f0-9]{10}_/, '');
|
||||
const effect = payload['effect'] || (result['side_effect'] === 'none' ? 'read' :
|
||||
['possible', 'confirmed'].includes(result['side_effect']) ? 'write' : '');
|
||||
const auxiliary = effect !== 'write';
|
||||
const titles: Record<string, string> = {
|
||||
get_app_state: '观察电脑窗口', list_apps: '列出电脑应用',
|
||||
click: '点击', double_click: '双击', drag: '拖动', scroll: '滚动',
|
||||
type_text: '输入文本', press_key: '按键', screenshot: '截取屏幕',
|
||||
save_screenshot: '保存截图', launch_app: '打开应用'
|
||||
};
|
||||
const content = Array.isArray(result['content']) ? result['content'] : [];
|
||||
const text = content.filter((item: any) => item?.type === 'text' && typeof item.text === 'string')
|
||||
.map((item: any) => item.text).join('\n');
|
||||
const state = result['execution'];
|
||||
const status = tool.status === 'running' ? '执行中' :
|
||||
result['status'] === 'uncertain' || state === 'unknown' ? '结果不确定' :
|
||||
tool.status === 'failed' || tool.status === 'error' || result['status'] === 'error' ? '失败' :
|
||||
state === 'partial' ? '部分完成' : state === 'not_started' ? '未执行' :
|
||||
state === 'completed' || tool.status === 'success' ? '已执行' : '待返回';
|
||||
const view: PluginToolView = {
|
||||
title: titles[name] || name.replace(/_/g, ' '),
|
||||
kind: payload['observation'] === true ? '辅助观察' : effect === 'read' ? '辅助读取' :
|
||||
effect === 'write' ? '插件操作' : '插件调用',
|
||||
auxiliary,
|
||||
result: [result['message'], text].filter(Boolean).join('\n') || String(tool.error || ''),
|
||||
status,
|
||||
args: Object.entries(objectValue(payload['args'] ?? tool.args)).map(([key, value]) => ({
|
||||
key, value: typeof value === 'string' ? value : JSON.stringify(value)
|
||||
}))
|
||||
};
|
||||
return view;
|
||||
}
|
||||
@@ -47,6 +47,19 @@ export interface SegmentLocation {
|
||||
|
||||
const clamp = (value: number, min: number, max: number): number => Math.max(min, Math.min(max, value));
|
||||
|
||||
/** A real task-time seek must never silently jump across an unrecorded interval. */
|
||||
export function locateCoveredSessionTime(segments: readonly TimelineSegment[], seconds: number): SegmentLocation | null {
|
||||
if (!Number.isFinite(seconds)) return null;
|
||||
const index = segments.findIndex(s => {
|
||||
const start = s.offset_ms === undefined ? s.start : s.offset_ms / 1000;
|
||||
return seconds >= start && seconds < start + s.duration;
|
||||
});
|
||||
if (index < 0) return null;
|
||||
const s = segments[index];
|
||||
const start = s.offset_ms === undefined ? s.start : s.offset_ms / 1000;
|
||||
return { index, localTime: seconds - start, timelineTime: seconds };
|
||||
}
|
||||
|
||||
/** True when every segment carries the session-relative v2 fields. */
|
||||
export function hasSessionOffsets(segments: readonly TimelineSegment[]): boolean {
|
||||
return segments.length > 0 && segments.every(
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { StepReplayFrame } from '../core/models/stream.model';
|
||||
import { hasSessionOffsets, sessionTimeToTimelineTime, TimelineSegment } from './recording-timeline.util';
|
||||
import { hasSessionOffsets, locateCoveredSessionTime, sessionTimeToTimelineTime, TimelineSegment } from './recording-timeline.util';
|
||||
|
||||
export interface VideoStepMarker {
|
||||
time: number;
|
||||
@@ -13,7 +13,8 @@ export function buildVideoStepMarkers(
|
||||
frames: readonly StepReplayFrame[],
|
||||
sessionStart: number | undefined,
|
||||
segments: readonly TimelineSegment[],
|
||||
duration: number
|
||||
duration: number,
|
||||
sessionTimeline = false
|
||||
): VideoStepMarker[] {
|
||||
const start = Number(sessionStart);
|
||||
if (!Number.isFinite(start) || !Number.isFinite(duration) || duration <= 0) return [];
|
||||
@@ -31,7 +32,8 @@ export function buildVideoStepMarkers(
|
||||
const sessionTime = timestamp - start;
|
||||
// Do not create misleading ticks at the start/end for steps outside the recording.
|
||||
if (sessionTime < firstOffset || sessionTime > lastOffset) return [];
|
||||
const time = sessionTimeToTimelineTime(segments, sessionTime);
|
||||
if (sessionTimeline && !locateCoveredSessionTime(segments, sessionTime)) return [];
|
||||
const time = sessionTimeline ? sessionTime : sessionTimeToTimelineTime(segments, sessionTime);
|
||||
if (time < 0 || time > duration) return [];
|
||||
return [{ time, frame }];
|
||||
}).sort((a, b) => a.time - b.time);
|
||||
|
||||
Reference in new issue
Block a user