feat(android): add state conditions and measured gesture feedback

Add optional ensure_state, gesture region and final value feedback shared by Flash and Pro. Preserve completed inputs when verification fails and stop repeated ineffective strategies. Regenerate pinned tool schemas for the new optional arguments.
This commit is contained in:
chenmingxuan committed 2026-11-02 23:20:34 +08:00
1 parent 6a69a595ef
commit 4ea7939d9c
20 files changed
+2779 -33

No files matched your search

@@ -623,6 +623,24 @@ describe('extractActionExtraParams', () => {
});
describe('native action Chinese presentation', () => {
it('renders actual state and final value verification in Chinese without changing literal evidence', () => {
const action = { action: 'action_sequence', steps: [{ action: 'control_action', request: {
action: 'click', selector: { resource_id: 'demo:id/player' },
ensure_state: { state: 'content_desc', expected_text: '播放' }
} }], feedback: {
selector: { resource_id: 'demo:id/clock' }, value_format: 'clock', target_value: 63737, tolerance: 5
}, value_feedback: { after_value: 63737, residual: 0, within_tolerance: true }, verification: 'value_matched' };
const original = JSON.stringify(action);
const params = extractActionExtraParams(action);
const steps = JSON.parse(params.find(p => p.key === '动作步骤')!.value);
expect(steps[0]['原生请求']['操作']).toBe('原生点击');
expect(steps[0]['原生请求']['确保控件状态']['等待条件']).toBe('控件描述匹配');
expect(steps[0]['原生请求']['确保控件状态']['预期文本']).toBe('播放');
expect(JSON.parse(params.find(p => p.key === '实际值核验')!.value)['读数格式']).toBe('时分秒文本');
expect(params).toContain({ key: '验证状态', value: '实际值符合目标' });
expect(JSON.stringify(action)).toBe(original);
});
it('renders a pinch trace in Chinese without modifying the recorded gesture', () => {
const action = {
name: 'pinch', args: {
@@ -8,7 +8,7 @@ const ACTION_LABELS = new Map(Object.entries({
control_action: '操作原生控件',
wait_until: '等待控件条件',
scroll_to: '滚动查找控件',
action_sequence: '动作序列'
action_sequence: '动作序列', drag: '拖动屏幕'
}));
const CONTROL_LABELS = new Map(Object.entries({
@@ -40,6 +40,20 @@ const PARAM_LABELS = new Map(Object.entries({
poll_interval_ms: '检查间隔(毫秒)', stable_ms: '稳定时长(毫秒)',
direction: '滚动方向', max_scrolls: '最多滚动次数', settle_ms: '停稳时长(毫秒)',
require_visible: '可见条件', unless_visible: '跳过条件',
ensure_state: '确保控件状态', verify_timeout_ms: '状态核验超时(毫秒)',
feedback: '实际值核验', value_feedback: '数值反馈', value_format: '读数格式',
before_value: '调整前数值', after_value: '调整后数值', target_value: '目标数值',
before_observation: '调整前观测', after_observation: '调整后观测',
node: '原生节点', samples: '采样次数', stable_for_ms: '已稳定时长(毫秒)',
residual: '实际残差', changed: '读数已变化', improved: '误差已减小',
within_tolerance: '符合允许误差', consecutive_no_progress: '连续无效次数',
region: '手势操作区域', start: '起点', end: '终点',
calibration: '刻度校准', axis: '刻度轴', mode: '拖动方式', anchors: '观测刻度',
position: '刻度位置', indicator_position: '固定指针位置',
min_movement_px: '已测最小有效距离(像素)', movement_px: '计划移动距离(像素)',
pixel_error_budget: '允许像素误差', value_per_pixel: '每像素数值',
rounding_error_value: '取整数值误差', anchor_coordinate_space: '刻度坐标空间',
target_position_pixel: '目标刻度像素位置', request: '原生请求',
steps: '动作步骤', action: '操作', args: '参数',
time_in_ms: '等待时长(毫秒)', target_description: '目标说明',
execution: '执行状态', verification: '验证状态'
@@ -50,7 +64,7 @@ const ENUM_LABELS: Record<string, Map<string, string>> = {
state: new Map(Object.entries({
visible: '可见', absent: '消失', enabled: '可用', disabled: '禁用',
checked: '已勾选', unchecked: '未勾选', selected: '已选中',
unselected: '未选中', text: '文本匹配', value: '数值匹配', stable: '持续稳定'
unselected: '未选中', text: '文本匹配', content_desc: '控件描述匹配', value: '数值匹配', stable: '持续稳定'
})),
direction: new Map(Object.entries({
forward: '向前', backward: '向后', up: '向上', down: '向下', left: '向左', right: '向右'
@@ -59,7 +73,15 @@ const ENUM_LABELS: Record<string, Map<string, string>> = {
not_started: '未开始', observed: '已观测', completed: '已完成',
partial: '部分完成', cancelled: '已取消', unknown: '结果未知'
})),
verification: new Map(Object.entries({ pending_observation: '等待观测' }))
mode: new Map(Object.entries({ thumb: '移动滑块', pan: '平移刻度' })),
value_format: new Map(Object.entries({ range: '原生数值范围', number: '数字文本', clock: '时分秒文本' })),
anchor_coordinate_space: new Map(Object.entries({ normalized: '归一化坐标', pixel: '像素坐标' })),
verification: new Map(Object.entries({
pending_observation: '等待观测', condition_satisfied: '状态条件已满足',
condition_not_met: '状态条件未满足', value_matched: '实际值符合目标',
value_not_reached: '实际值未达目标', unavailable: '无法核验',
strategy_change_required: '需要更换调整策略'
}))
};
export function getNativeActionTitle(name: string, args: any = {}): string | null {
@@ -73,7 +95,7 @@ export function getNativeActionTitle(name: string, args: any = {}): string | nul
}
export function getNativeActionArgs(args: any): any {
return args?.action === 'control_action' && args.args ? args.args : (args ?? {});
return args?.action === 'control_action' ? (args.request ?? args.args ?? args) : (args ?? {});
}
function localizedValue(value: any, key: string, controlAction = false): any {
@@ -81,7 +103,7 @@ function localizedValue(value: any, key: string, controlAction = false): any {
if (value && typeof value === 'object') {
return Object.fromEntries(Object.entries(value).map(([field, item]) => [
PARAM_LABELS.get(field) ?? field,
localizedValue(item, field, field === 'action' ? controlAction : field === 'args' && value.action === 'control_action')
localizedValue(item, field, field === 'action' ? controlAction : (field === 'args' || field === 'request') && value.action === 'control_action')
]));
}
if (typeof value === 'string') {