From 6a69a595ef355323f7012f70adfe04c4b1f76f7a Mon Sep 17 00:00:00 2001 From: chenmingxuan Date: Mon, 2 Nov 2026 22:33:40 +0800 Subject: [PATCH] fix(ui): localize native Android action displays --- .../app/utils/action-formatter.util.spec.ts | 37 +++++++ .../src/app/utils/action-formatter.util.ts | 11 +- .../app/utils/native-action-labels.util.ts | 102 ++++++++++++++++++ .../src/app/utils/tool-formatter.util.spec.ts | 26 +++++ .../src/app/utils/tool-formatter.util.ts | 19 +++- 5 files changed, 192 insertions(+), 3 deletions(-) create mode 100644 apps/showcase_ui/src/app/utils/native-action-labels.util.ts diff --git a/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts b/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts index 522c6e5..2c07edb 100644 --- a/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts +++ b/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts @@ -621,3 +621,40 @@ describe('extractActionExtraParams', () => { expect(sequence.map(p => p.key)).toContain('Interval Ms'); }); }); + +describe('native action Chinese presentation', () => { + it('renders a pinch trace in Chinese without modifying the recorded gesture', () => { + const action = { + name: 'pinch', args: { + center: [360, 1040], start_span: 100, end_span: 400, + duration_ms: 1000, coordinate_space: 'pixel', target_description: 'timeline' + } + }; + const original = JSON.stringify(action); + expect(getActionTitle(action)).toBe('双指缩放/旋转'); + expect(extractActionExtraParams(action)).toContain({ key: '坐标空间', value: '像素坐标' }); + expect(extractActionExtraParams(action)).toContain({ key: '起始指间距', value: '100' }); + expect(JSON.stringify(action)).toBe(original); + }); + + it('translates sequence members while preserving literal target text and IDs', () => { + const params = extractActionExtraParams({ action: 'action_sequence', steps: [ + { action: 'control_action', args: { + selector: { text: 'visible', resource_id: 'demo:id/set_text' }, + action: 'replace_selection', text: 'set_text' + } }, + { action: 'wait_until', state: 'stable', stable_ms: 500 }, + { action: 'click', args: { target: [100, 200] } }, + { action: 'wait', time_in_ms: 100 } + ] }); + const steps = JSON.parse(params.find(param => param.key === '动作步骤')!.value); + expect(steps[0]['操作']).toBe('操作原生控件'); + expect(steps[0]['参数']['操作']).toBe('替换选区文本'); + expect(steps[0]['参数']['控件选择器']['文本']).toBe('visible'); + expect(steps[0]['参数']['控件选择器']['资源 ID']).toBe('demo:id/set_text'); + expect(steps[0]['参数']['文本']).toBe('set_text'); + expect(steps[1]['等待条件']).toBe('持续稳定'); + expect(steps[2]['操作']).toBe('点击元素'); + expect(steps[3]['操作']).toBe('等待'); + }); +}); diff --git a/apps/showcase_ui/src/app/utils/action-formatter.util.ts b/apps/showcase_ui/src/app/utils/action-formatter.util.ts index 226bb35..a7c6549 100644 --- a/apps/showcase_ui/src/app/utils/action-formatter.util.ts +++ b/apps/showcase_ui/src/app/utils/action-formatter.util.ts @@ -18,6 +18,7 @@ import { ActionParam, StepReplayFrame } from '../core/models/stream.model'; import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates, unwrapTraceAction } from './image-overlay.util'; import { cleanErrorMessage, joinTargetDescriptions } from './tool-formatter.util'; import { getPluginToolView } from './plugin-tool-view.util'; +import { formatNativeActionParam, getNativeActionArgs, getNativeActionTitle } from './native-action-labels.util'; /** * Safely parse JSON string if it looks like an object/array, otherwise return original value @@ -170,6 +171,8 @@ export function getActionTitle(action: any): string { const plugin = getPluginToolView(act); if (plugin) return plugin.title; const name = (act.name || act.action || '').toLowerCase(); + const nativeTitle = getNativeActionTitle(name, act.args || act); + if (nativeTitle) return nativeTitle; switch (name) { case 'tap': case 'click': @@ -539,6 +542,8 @@ export function extractActionExtraParams(action: any, cache?: WeakMap> = { + coordinate_space: new Map(Object.entries({ normalized: '归一化坐标', pixel: '像素坐标' })), + state: new Map(Object.entries({ + visible: '可见', absent: '消失', enabled: '可用', disabled: '禁用', + checked: '已勾选', unchecked: '未勾选', selected: '已选中', + unselected: '未选中', text: '文本匹配', value: '数值匹配', stable: '持续稳定' + })), + direction: new Map(Object.entries({ + forward: '向前', backward: '向后', up: '向上', down: '向下', left: '向左', right: '向右' + })), + execution: new Map(Object.entries({ + not_started: '未开始', observed: '已观测', completed: '已完成', + partial: '部分完成', cancelled: '已取消', unknown: '结果未知' + })), + verification: new Map(Object.entries({ pending_observation: '等待观测' })) +}; + +export function getNativeActionTitle(name: string, args: any = {}): string | null { + const clean = String(name).toLowerCase().replace(/^_?(?:self\.)?exec_/, '') + .replace(/^action_sequence\./, ''); + if (clean === 'control_action') { + const request = getNativeActionArgs(args); + return CONTROL_LABELS.get(request.action) ?? ACTION_LABELS.get(clean)!; + } + return ACTION_LABELS.get(clean) ?? null; +} + +export function getNativeActionArgs(args: any): any { + return args?.action === 'control_action' && args.args ? args.args : (args ?? {}); +} + +function localizedValue(value: any, key: string, controlAction = false): any { + if (Array.isArray(value)) return value.map(item => localizedValue(item, key, controlAction)); + if (value && typeof value === 'object') { + return Object.fromEntries(Object.entries(value).map(([field, item]) => [ + PARAM_LABELS.get(field) ?? field, + localizedValue(item, field, field === 'action' ? controlAction : field === 'args' && value.action === 'control_action') + ])); + } + if (typeof value === 'string') { + if (key === 'action') { + return ACTION_LABELS.get(value) ?? (controlAction ? CONTROL_LABELS : STEP_LABELS).get(value) ?? value; + } + if (Object.hasOwn(ENUM_LABELS, key)) return ENUM_LABELS[key].get(value) ?? value; + } + return value; +} + +export function formatNativeActionParam(key: string, value: any): ActionParam { + const localized = localizedValue(value, key); + return { + key: PARAM_LABELS.get(key) ?? key.replace(/_/g, ' ').replace(/\b\w/g, c => c.toUpperCase()), + value: typeof localized === 'object' ? JSON.stringify(localized) : String(localized) + }; +} diff --git a/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts b/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts index efacffc..54d6a15 100644 --- a/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts +++ b/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts @@ -6,6 +6,8 @@ import { getCompressionPhaseLabel, isCompressionWaiting, getToolDisplayLabel, + getToolTitle, + getToolGenericDetails, getToolIcon, getToolTargetText, getUniqueGenericTools, @@ -373,3 +375,27 @@ describe('extractToolExtraParams', () => { expect(keys).toContain('Interval Ms'); }); }); + +describe('native tool Chinese presentation', () => { + it('shares Chinese titles across live tool lines and expanded cards', () => { + for (const name of ['pinch', 'touch_gesture', 'read_control', 'wait_until', 'scroll_to', 'action_sequence']) { + const tool = { name: `self.exec_${name}`, payload: { args: {} } }; + expect(getToolTitle(tool)).toMatch(/[\u4e00-\u9fff]/); + expect(getToolDisplayLabel(tool)).toBe(getToolTitle(tool)); + } + }); + + it('understands nested native sequence calls and parsed payload arguments', () => { + const tool = { name: 'action_sequence.control_action', payload: { args: { + action: 'control_action', args: { action: 'ime_enter', selector: { content_desc: 'Send' } } + } } }; + expect(getToolTitle(tool)).toBe('提交输入法操作'); + expect(getToolDisplayLabel(tool)).toBe('提交输入法操作'); + expect(extractToolExtraParams(tool)).toContain({ key: '控件选择器', value: '{"控件描述":"Send"}' }); + const wait = { name: 'wait_until', payload: JSON.stringify({ args: { + state: 'text', expected_text: 'visible', timeout_ms: 3000 + } }) }; + expect(extractToolExtraParams(wait)).toContain({ key: '等待条件', value: '文本匹配' }); + expect(getToolGenericDetails(wait)).toContain('预期文本：visible'); + }); +}); diff --git a/apps/showcase_ui/src/app/utils/tool-formatter.util.ts b/apps/showcase_ui/src/app/utils/tool-formatter.util.ts index 2390c2e..4709c68 100644 --- a/apps/showcase_ui/src/app/utils/tool-formatter.util.ts +++ b/apps/showcase_ui/src/app/utils/tool-formatter.util.ts @@ -16,6 +16,7 @@ import { ActionParam } from '../core/models/stream.model'; import { extractNumbersFromCoordinateValue, isPureDirectionString, parseSequenceCoordinates } from './image-overlay.util'; +import { formatNativeActionParam, getNativeActionArgs, getNativeActionTitle } from './native-action-labels.util'; // Tool objects are replaced (not mutated) when a trace is updated, so a // WeakMap keyed on the tool is a safe memo for the parsed-args lookup that @@ -553,6 +554,8 @@ export function getToolDisplayLabel(tool: any, isFirstSaveNote: boolean = false) const cleanName = tool.name.replace(/^(_)?exec_/, ''); const nameLower = cleanName.toLowerCase(); const args = getToolArgs(tool); + const nativeTitle = getNativeActionTitle(tool.name, args); + if (nativeTitle) return nativeTitle; switch (nameLower) { case 'manage_app': @@ -899,6 +902,8 @@ export function getToolIcon(tool: any): string { */ export function getToolTitle(tool: any): string { if (!tool || !tool.name) return "工具调用"; + const nativeTitle = getNativeActionTitle(tool.name, getToolArgs(tool)); + if (nativeTitle) return nativeTitle; const cleanName = tool.name.replace(/^(_)?exec_/, ''); const name = cleanName.toLowerCase(); switch (name) { @@ -1191,6 +1196,9 @@ export function getAdbTerminalId(tool: any): string { */ export function getToolGenericDetails(tool: any): string { if (!tool) return ''; + if (getNativeActionTitle(tool.name || '', getToolArgs(tool))) { + return extractToolExtraParams(tool).map(param => `${param.key}：${param.value}`).join('，'); + } if (isAdbCommandTool(tool)) { return ''; } @@ -1235,7 +1243,9 @@ export function getToolErrorMessage(tool: any): string { */ export function extractToolExtraParams(toolData: any, cache?: WeakMap): ActionParam[] { if (!toolData) return []; - const payload = toolData.payload || toolData.args || toolData; + const nativeArgs = getNativeActionArgs(getToolArgs(toolData)); + const native = getNativeActionTitle(toolData.name || '', nativeArgs) !== null; + const payload = native ? nativeArgs : (toolData.payload || toolData.args || toolData); if (!payload || typeof payload !== 'object') return []; if (cache && cache.has(payload)) { return cache.get(payload)!; @@ -1272,11 +1282,16 @@ export function extractToolExtraParams(toolData: any, cache?: WeakMap