diff --git a/DESIGN.md b/DESIGN.md index 79bee95..3b92207 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -88,6 +88,8 @@ Responses 消息交给工具、下游解析器或正式裁定前，必须具有 `drag` 支持明确的归一化或原图像素端点，默认 300ms，不自动过冲。可根据当前缩放下的两个真实数值刻度校准 thumb 位置，或将 pan 刻度移至固定指针；返回真实派发端点、比例及端点取整误差。校准假定本次局部刻度线性，换缩放须重新读取锚点；计算回执保留 pending_observation，实际选中值及业务目标仍须通过动作后的观察验证。像素分辨率不足时先放大，不宣称任意时间精度。 +状态动作使用显式 `control_action.ensure_state`：仅以真实原生 checked/selected、文本/描述或数值判断，已满足则跳过，否则写入一次并有界只读核验。缺失状态不当作 false；核验失败保留完成的输入，不自动再切换。drag/pinch/touch_gesture 的可选 region 在首个输入前约束所有触点路径。drag 或整个 action_sequence 的可选 feedback 绑定唯一持久数值读数，在输入前确认可读，整组结束后有界等待稳定并报告实际值、目标、残差和允许误差；连续两次同策略未减小误差则拒绝第三次输入，要求重新观察、缩放或改变有效移动距离。记录仅存于当前设备执行实例，缺失读数不记作零，不共享到新任务。计算精度、实际数值条件和最终业务验收分开；无原生读数的自绘控件仍依靠视觉证据，不能宣称任意精度。详见[原生动作说明](docs/android-native-actions.md)。 + 任务耗时复用现有队列、工作进程控制文件与验收 trace，网页、MCP 和报告使用同一只读投影。分别列出后台排队、等待设备/资源、暂停与示范、执行与整理、最终验收及报告导出；总历时包含这些等待，不代表模型速度。缺少时间记录显示未知，不将旧任务缺失的分项补成零。运行结束后不继续累计执行耗时，报告导出单独计时。 指导接收不保证赶在终态前消费。保留接收时点、执行器消费回执及后续动作记录的区别；`consumed` 仅证明执行器已读取，不能宣称模型已应用或验收已通过。终态仍未领取的指导投影为 `unapplied`，领取后未留下消费回执的为 `unconfirmed`，不改写原始文件。网页详细查询复用原指导入口，旧数组响应保持兼容。 diff --git a/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts b/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts index 2c07edb..db422de 100644 --- a/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts +++ b/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts @@ -623,6 +623,24 @@ describe('extractActionExtraParams', () => { }); describe('native action Chinese presentation', () => { + it('renders actual state and final value verification in Chinese without changing literal evidence', () => { + const action = { action: 'action_sequence', steps: [{ action: 'control_action', request: { + action: 'click', selector: { resource_id: 'demo:id/player' }, + ensure_state: { state: 'content_desc', expected_text: '播放' } + } }], feedback: { + selector: { resource_id: 'demo:id/clock' }, value_format: 'clock', target_value: 63737, tolerance: 5 + }, value_feedback: { after_value: 63737, residual: 0, within_tolerance: true }, verification: 'value_matched' }; + const original = JSON.stringify(action); + const params = extractActionExtraParams(action); + const steps = JSON.parse(params.find(p => p.key === '动作步骤')!.value); + expect(steps[0]['原生请求']['操作']).toBe('原生点击'); + expect(steps[0]['原生请求']['确保控件状态']['等待条件']).toBe('控件描述匹配'); + expect(steps[0]['原生请求']['确保控件状态']['预期文本']).toBe('播放'); + expect(JSON.parse(params.find(p => p.key === '实际值核验')!.value)['读数格式']).toBe('时分秒文本'); + expect(params).toContain({ key: '验证状态', value: '实际值符合目标' }); + expect(JSON.stringify(action)).toBe(original); + }); + it('renders a pinch trace in Chinese without modifying the recorded gesture', () => { const action = { name: 'pinch', args: { diff --git a/apps/showcase_ui/src/app/utils/native-action-labels.util.ts b/apps/showcase_ui/src/app/utils/native-action-labels.util.ts index c3cc16f..9969f94 100644 --- a/apps/showcase_ui/src/app/utils/native-action-labels.util.ts +++ b/apps/showcase_ui/src/app/utils/native-action-labels.util.ts @@ -8,7 +8,7 @@ const ACTION_LABELS = new Map(Object.entries({ control_action: '操作原生控件', wait_until: '等待控件条件', scroll_to: '滚动查找控件', - action_sequence: '动作序列' + action_sequence: '动作序列', drag: '拖动屏幕' })); const CONTROL_LABELS = new Map(Object.entries({ @@ -40,6 +40,20 @@ const PARAM_LABELS = new Map(Object.entries({ poll_interval_ms: '检查间隔（毫秒）', stable_ms: '稳定时长（毫秒）', direction: '滚动方向', max_scrolls: '最多滚动次数', settle_ms: '停稳时长（毫秒）', require_visible: '可见条件', unless_visible: '跳过条件', + ensure_state: '确保控件状态', verify_timeout_ms: '状态核验超时（毫秒）', + feedback: '实际值核验', value_feedback: '数值反馈', value_format: '读数格式', + before_value: '调整前数值', after_value: '调整后数值', target_value: '目标数值', + before_observation: '调整前观测', after_observation: '调整后观测', + node: '原生节点', samples: '采样次数', stable_for_ms: '已稳定时长（毫秒）', + residual: '实际残差', changed: '读数已变化', improved: '误差已减小', + within_tolerance: '符合允许误差', consecutive_no_progress: '连续无效次数', + region: '手势操作区域', start: '起点', end: '终点', + calibration: '刻度校准', axis: '刻度轴', mode: '拖动方式', anchors: '观测刻度', + position: '刻度位置', indicator_position: '固定指针位置', + min_movement_px: '已测最小有效距离（像素）', movement_px: '计划移动距离（像素）', + pixel_error_budget: '允许像素误差', value_per_pixel: '每像素数值', + rounding_error_value: '取整数值误差', anchor_coordinate_space: '刻度坐标空间', + target_position_pixel: '目标刻度像素位置', request: '原生请求', steps: '动作步骤', action: '操作', args: '参数', time_in_ms: '等待时长（毫秒）', target_description: '目标说明', execution: '执行状态', verification: '验证状态' @@ -50,7 +64,7 @@ const ENUM_LABELS: Record> = { state: new Map(Object.entries({ visible: '可见', absent: '消失', enabled: '可用', disabled: '禁用', checked: '已勾选', unchecked: '未勾选', selected: '已选中', - unselected: '未选中', text: '文本匹配', value: '数值匹配', stable: '持续稳定' + unselected: '未选中', text: '文本匹配', content_desc: '控件描述匹配', value: '数值匹配', stable: '持续稳定' })), direction: new Map(Object.entries({ forward: '向前', backward: '向后', up: '向上', down: '向下', left: '向左', right: '向右' @@ -59,7 +73,15 @@ const ENUM_LABELS: Record> = { not_started: '未开始', observed: '已观测', completed: '已完成', partial: '部分完成', cancelled: '已取消', unknown: '结果未知' })), - verification: new Map(Object.entries({ pending_observation: '等待观测' })) + mode: new Map(Object.entries({ thumb: '移动滑块', pan: '平移刻度' })), + value_format: new Map(Object.entries({ range: '原生数值范围', number: '数字文本', clock: '时分秒文本' })), + anchor_coordinate_space: new Map(Object.entries({ normalized: '归一化坐标', pixel: '像素坐标' })), + verification: new Map(Object.entries({ + pending_observation: '等待观测', condition_satisfied: '状态条件已满足', + condition_not_met: '状态条件未满足', value_matched: '实际值符合目标', + value_not_reached: '实际值未达目标', unavailable: '无法核验', + strategy_change_required: '需要更换调整策略' + })) }; export function getNativeActionTitle(name: string, args: any = {}): string | null { @@ -73,7 +95,7 @@ export function getNativeActionTitle(name: string, args: any = {}): string | nul } export function getNativeActionArgs(args: any): any { - return args?.action === 'control_action' && args.args ? args.args : (args ?? {}); + return args?.action === 'control_action' ? (args.request ?? args.args ?? args) : (args ?? {}); } function localizedValue(value: any, key: string, controlAction = false): any { @@ -81,7 +103,7 @@ function localizedValue(value: any, key: string, controlAction = false): any { if (value && typeof value === 'object') { return Object.fromEntries(Object.entries(value).map(([field, item]) => [ PARAM_LABELS.get(field) ?? field, - localizedValue(item, field, field === 'action' ? controlAction : field === 'args' && value.action === 'control_action') + localizedValue(item, field, field === 'action' ? controlAction : (field === 'args' || field === 'request') && value.action === 'control_action') ])); } if (typeof value === 'string') { diff --git a/artemis/agents/flash/flash_runner.md b/artemis/agents/flash/flash_runner.md index c3039d6..f157aea 100644 --- a/artemis/agents/flash/flash_runner.md +++ b/artemis/agents/flash/flash_runner.md @@ -11,6 +11,9 @@ You are an autonomous UI execution agent. Accomplish the user's objective using **Objective: {{ goal }}** +{% if "drag" in available_tools %}- **数值目标的闭环**：先观察真实手势区域，为 drag/pinch/touch_gesture 填 `drag.region`，不要把日期栏、导航栏当作时间轴。若原生可读取持久数值，为 drag 或 action_sequence 绑定 `action_sequence.feedback`（唯一稳定 selector、value_format、target_value、tolerance）；时分秒用 clock，单位为当天秒数。组合“拖动→暂停”时只在整个序列上绑定 feedback，末尾读稳定值，不能在拖动中途的浮层上核验。已验证的同一持久读数在允许误差内就是此数值条件的证据；日期、暂停等其它要求仍分别核对，不能无依据地改称误差无法判断。连续无效反馈会拒绝第三次同策略输入；放大并重新读刻度、改为足够距离或原生设值，不能靠微改坐标绕过。calibration.tolerance 给出几何误差预算；min_movement_px 只填写实际测得的起效距离，不猜固定阈值。没有可读的持久数值时保留视觉证据与不确定项，不捏造 selector 或读数。{% endif %} +{% if "control_action" in available_tools %}- **先状态后动作**：对于暂停、勾选等目标状态，先 read_control 确认其真实原生语义，再用 control_action.ensure_state；已符合时跳过，否则只操作一次并有界核验。可按观察到的 checked/selected、text 或 content_desc 判断；不能假设 selected 就表示暂停。selector 不包含动作会改变的文本/描述。拖动可能恢复播放，把确保暂停放在拖动后同一动作序列中；状态核验失败时重新观察，不重复盲点切换。{% endif %} + --- # 1. COGNITIVE PROTOCOL diff --git a/artemis/agents/validator/tool_declarations.py b/artemis/agents/validator/tool_declarations.py index 2754ed7..76a63e0 100644 --- a/artemis/agents/validator/tool_declarations.py +++ b/artemis/agents/validator/tool_declarations.py @@ -263,7 +263,9 @@ REPORT_TASK_STATUS_TOOL = ToolDeclaration( "properties": { "report_note_key": { "type": "string", - "description": "If the requested final report is already saved with save_note, name that note here. ARTEMIS uses its exact body as the final answer and output.md, instead of generating a second conflicting report. Omit when explanation itself is the complete report.", + "pattern": r"^[\w-]{0,128}$", + "maxLength": 128, + "description": "Copy the exact plain note name ONLY if the requested final report was saved with save_note. Otherwise use the empty string (or omit); never supply placeholders or paths such as /not/requested. ARTEMIS uses the saved note's exact body as the final report.", }, "status": { "type": "string", diff --git a/artemis/mcp/action_parameters.py b/artemis/mcp/action_parameters.py index 21bd94f..0b21727 100644 --- a/artemis/mcp/action_parameters.py +++ b/artemis/mcp/action_parameters.py @@ -12,6 +12,7 @@ class GestureModel(BaseModel): Point = Annotated[list[StrictInt], Field(min_length=2, max_length=2)] +Region = Annotated[list[StrictInt], Field(min_length=4, max_length=4)] Duration = Annotated[int, Field(strict=True, ge=1, le=10000)] CoordinateSpace = Literal["normalized", "pixel"] @@ -25,3 +26,29 @@ def pixel_point(point: list[int], space: CoordinateSpace, width: int, height: in if space == "pixel": return list(point) return [min(size - 1, p * size // 1000) for p, size in zip(point, [width, height], strict=True)] + + +def pixel_region(region: list[int], space: CoordinateSpace, width: int, height: int) -> list[int]: + """Observed rectangle; right/bottom are exclusive, like Android bounds.""" + left, top, right, bottom = region + max_x, max_y = (1000, 1000) if space == "normalized" else (width, height) + if not (0 <= left < right <= max_x and 0 <= top < bottom <= max_y): + raise ValueError( + "region must be a nonempty on-screen [left, top, right, bottom] rectangle." + ) + if space == "pixel": + return list(region) + return [ + left * width // 1000, + top * height // 1000, + right * width // 1000, + bottom * height // 1000, + ] + + +def require_in_region(points: list[list[int]], region: list[int]) -> None: + left, top, right, bottom = region + if any(not (left <= x < right and top <= y < bottom) for x, y in points): + raise ValueError( + "Gesture leaves the observed region; re-observe the actual gesture surface. No input executed." + ) diff --git a/artemis/mcp/action_specs.py b/artemis/mcp/action_specs.py index fba2b98..fab5244 100644 --- a/artemis/mcp/action_specs.py +++ b/artemis/mcp/action_specs.py @@ -70,6 +70,8 @@ from pydantic import BaseModel, Field, create_model from artemis.core.tool_declaration import ToolDeclaration from artemis.mcp.action_types import ActionResult from artemis.mcp.gestures import CoordinateSpace, DragCalibration, GestureStep +from artemis.mcp.action_parameters import Region +from artemis.mcp.value_feedback import ValueFeedback from artemis.mcp.native_actions import NATIVE_REQUESTS, READ_ONLY_ACTIONS __all__ = [ @@ -225,6 +227,8 @@ async def _wire_drag(actuator: Any, a: dict[str, Any]) -> ActionResult: async def _wire_action_sequence(actuator: Any, a: dict[str, Any]) -> ActionResult: + if a.get("feedback") is not None: + return await actuator.action_sequence(a["steps"], feedback=a["feedback"]) return await actuator.action_sequence(a["steps"]) @@ -298,7 +302,13 @@ _SPECS: tuple[ActionSpec, ...] = ( "A prior screenshot does not prove the panel is still open. Use click.unless_visible " "and drag.require_visible with an observed panel resource_id to avoid toggling an " "open panel closed or dragging its underlying content. Never invent resource IDs. " - "Verify actual ticks/handle endpoints before claiming maximum or minimum." + "Verify actual ticks/handle endpoints before claiming maximum or minimum. " + "For a measurable target, bind feedback to its persistent native readout: " + "final feedback is read AFTER ALL members, including any pause, finish. Never use a release-time tooltip. " + "Use control_action.request.ensure_state for observed checked/selected/text/value state, " + "so an already satisfied state is skipped rather than toggled. A playback icon alone " + "does not prove a native boolean state: inspect it first. Feedback blocks the third " + "consecutive ineffective strategy; zoom or increase the meaningful movement, then re-observe." ), params=( ParamSpec( @@ -306,11 +316,20 @@ _SPECS: tuple[ActionSpec, ...] = ( list[GestureStep], "Ordered UI preparation, click, wait, drag, pinch, touch_gesture, control_action and wait_until members. control_action members wrap their native selector/action arguments in request. Inline text is limited to 1024 characters; use a standalone control_action for long edits. Entire encoded sequence stays within the existing 16KiB argument safety bound.", ), + ParamSpec( + "feedback", + ValueFeedback | None, + "Optional actual value/target comparison after the WHOLE sequence. clock uses seconds since midnight; exact bound and stable observation are checked locally, not inferred from dispatch.", + required=False, + ), ), ), wire=WireDialect( description="Execute validated short gesture steps consecutively; stop at the first failure.", - params=(ParamSpec("steps", list[GestureStep]),), + params=( + ParamSpec("steps", list[GestureStep]), + ParamSpec("feedback", ValueFeedback | None, required=False), + ), bind=_wire_action_sequence, ), ), @@ -350,6 +369,18 @@ _SPECS: tuple[ActionSpec, ...] = ( "coordinate_space", CoordinateSpace, required=False, default="normalized" ), ParamSpec("duration_ms", int, required=False, default=300), + ParamSpec( + "region", + Region | None, + "Observed gesture surface rectangle, in coordinate_space. Keep both endpoints inside.", + required=False, + ), + ParamSpec( + "feedback", + ValueFeedback | None, + "Read the persistent actual value after the drag. For drag then pause, put feedback on action_sequence instead.", + required=False, + ), ParamSpec( "calibration", DragCalibration | None, @@ -374,6 +405,8 @@ _SPECS: tuple[ActionSpec, ...] = ( "coordinate_space", CoordinateSpace, required=False, default="normalized" ), ParamSpec("duration_ms", int, required=False, default=300), + ParamSpec("region", Region | None, required=False), + ParamSpec("feedback", ValueFeedback | None, required=False), ParamSpec("calibration", DragCalibration | None, required=False), ParamSpec("require_visible", str | None, required=False), ), @@ -853,6 +886,11 @@ _NATIVE_DESCRIPTIONS = { "control_action": ( "[ACTION] Perform ONE supported native action on a freshly resolved unique control. " "Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. " + "For a desired state use ensure_state (checked/selected/text/content_desc/value), then " + "verify_timeout_ms: skip an already satisfied state, otherwise write ONCE and poll read-only. " + "Copy observed state semantics; do not assume selected means paused. Bind a stable selector " + "excluding text/content_desc that the action changes. A failed postcondition does not undo " + "the completed input and must not cause an automatic second toggle. " "action=set_progress requires a finite in-range value; set_selection requires selection_start/end " "in UTF-16 code units (never split an emoji). set_text and replace_selection require text, " "including empty text; whitespace is preserved. Other actions take no text/value/selection. " diff --git a/artemis/mcp/actuators/adb.py b/artemis/mcp/actuators/adb.py index 44b0273..0ea7adc 100644 --- a/artemis/mcp/actuators/adb.py +++ b/artemis/mcp/actuators/adb.py @@ -108,6 +108,9 @@ class AdbActuator: ): self.ctx = ctx self.controller = controller or UnifiedMobileController(ctx) + from artemis.mcp.value_feedback import ValueFeedbackRunner + + self._value_feedback = ValueFeedbackRunner() # --- Contract -------------------------------------------------------------------- @@ -183,6 +186,8 @@ class AdbActuator: else "possible" ) if receipt.get("success"): + if name in {"pinch", "touch_gesture"} and receipt.get("execution") == "completed": + self._value_feedback.history.clear() return ActionResult.success( name, "Native operation completed. Inspect the receipt and current app state; business verification remains separate.", @@ -224,10 +229,24 @@ class AdbActuator: from artemis.mcp.gestures import DragRequest, plan_drag try: - planned = plan_drag(DragRequest.model_validate(kwargs), *self._dims()) + request = DragRequest.model_validate(kwargs) + planned = plan_drag(request, *self._dims()) except ValueError as exc: return ActionResult.failure("drag", str(exc), code=ActionCode.INVALID_ARGS) - return await self._dispatch_drag(planned) + if request.feedback is not None and "read_control" not in self.capabilities(): + return ActionResult.failure( + "drag", + "Value feedback requires native read_control; no input executed.", + code=ActionCode.UNSUPPORTED, + metadata={"execution": "not_started", "side_effect": "none"}, + ) + return await self._value_feedback.run( + request.feedback, + [planned], + "drag", + self.controller.native_request, + lambda: self._dispatch_drag(planned), + ) async def _dispatch_drag(self, planned: dict) -> ActionResult: metadata = dict(planned) @@ -374,19 +393,18 @@ class AdbActuator: f"Panel target {step['target']} did not become visible within {step['timeout_ms']}ms after one opener tap. No opener retry; re-observe before continuing." ) from exc - async def action_sequence(self, steps: list[Any]) -> ActionResult: + async def action_sequence( + self, steps: list[Any], feedback: dict[str, Any] | None = None + ) -> ActionResult: """Execute once; optional visibility guards read live UI without model turns.""" - import time - from artemis.config.agent import load_agent_config - from artemis.data_engine.trace import TraceSpan from artemis.mcp.gestures import plan_sequence, validate_sequence - from artemis.runtime.task_control import TaskControl, has_pending_instruction try: limit = load_agent_config().pro.execution.max_burst_actions + request = validate_sequence({"steps": steps, "feedback": feedback}, limit) planned = plan_sequence( - validate_sequence({"steps": steps}, limit), + request, *self._dims(), limit=limit, ) @@ -406,6 +424,28 @@ class AdbActuator: code=ActionCode.UNSUPPORTED, metadata={"execution": "not_started", "side_effect": "none"}, ) + if request.feedback is not None and "read_control" not in self.capabilities(): + return ActionResult.failure( + "action_sequence", + "Value feedback requires native read_control; no input executed.", + code=ActionCode.UNSUPPORTED, + metadata={"execution": "not_started", "side_effect": "none"}, + ) + return await self._value_feedback.run( + request.feedback, + planned, + "action_sequence", + self.controller.native_request, + lambda: self._execute_sequence(planned), + ) + + async def _execute_sequence(self, planned: list[dict]) -> ActionResult: + import time + + from artemis.data_engine.trace import TraceSpan + from artemis.mcp.native_actions import NATIVE_ACTIONS + from artemis.runtime.task_control import TaskControl, has_pending_instruction + started = time.monotonic() receipts = [ {**step, "index": i + 1, "execution": "not_started"} for i, step in enumerate(planned) @@ -484,8 +524,12 @@ class AdbActuator: error = None if result.ok else result.message receipt.update(result.metadata) receipt["native_execution"] = result.metadata.get("execution", "unknown") + member_execution = ( + "skipped" + if result.ok and result.metadata.get("skipped") + else receipt["native_execution"] + ) if not result.ok: - member_execution = result.metadata.get("execution", "unknown") failed_code = result.code elif step["action"] == "drag": result = await self._dispatch_drag(step) @@ -503,6 +547,7 @@ class AdbActuator: member_execution if member_execution in {"skipped", "not_started", "cancelled", "observed"} + or receipt.get("native_execution") == "completed" or not error else "unknown" ), @@ -553,7 +598,7 @@ class AdbActuator: ) return ActionResult.success( "action_sequence", - f"Executed {completed} gesture steps consecutively; skipped {skipped} unnecessary UI-opening steps. Verify the intended result in the final screen.", + f"Executed {completed} gesture steps consecutively; skipped {skipped} already satisfied steps. Verify the intended result in the final screen.", metadata=metadata, ) diff --git a/artemis/mcp/actuators/base.py b/artemis/mcp/actuators/base.py index 7508886..3b986a7 100644 --- a/artemis/mcp/actuators/base.py +++ b/artemis/mcp/actuators/base.py @@ -105,7 +105,9 @@ class Actuator(Protocol): async def scroll_to(self, **kwargs: Any) -> ActionResult: ... - async def action_sequence(self, steps: list[Any]) -> ActionResult: ... + async def action_sequence( + self, steps: list[Any], feedback: dict[str, Any] | None = None + ) -> ActionResult: ... async def manage_app(self, action: str, app_name: str) -> ActionResult: ... diff --git a/artemis/mcp/gestures.py b/artemis/mcp/gestures.py index 26e2a1f..6a4d951 100644 --- a/artemis/mcp/gestures.py +++ b/artemis/mcp/gestures.py @@ -14,7 +14,10 @@ from artemis.mcp.action_parameters import ( Duration, GestureModel, Point, + Region, pixel_point, + pixel_region, + require_in_region, ) from artemis.mcp.native_actions import ( ControlActionRequest, @@ -24,6 +27,7 @@ from artemis.mcp.native_actions import ( plan_pinch, plan_touch, ) +from artemis.mcp.value_feedback import ValueFeedback class ScaleAnchor(GestureModel): @@ -43,9 +47,19 @@ class DragCalibration(GestureModel): indicator_position: float | None = Field( default=None, description="Required for pan: the fixed indicator's observed axis position." ) + tolerance: float | None = Field( + default=None, + ge=0, + description="Allowed value error. Reject a plan whose pixel rounding exceeds it; this is geometric error only, not proof of application precision.", + ) + min_movement_px: float | None = Field( + default=None, + gt=0, + description="Previously measured minimum effective drag distance in pixels. If this calibrated move is smaller, refuse input and zoom first. Never guess a device-wide threshold.", + ) -class DragRequest(GestureModel): +class DragGeometry(GestureModel): start: Point target_description: str = Field(min_length=1) end: Point | None = Field( @@ -53,6 +67,10 @@ class DragRequest(GestureModel): ) coordinate_space: CoordinateSpace = "normalized" duration_ms: Duration = 300 + region: Region | None = Field( + default=None, + description="Observed gesture surface rectangle in coordinate_space, excluding date/navigation rails and unrelated overlays.", + ) calibration: DragCalibration | None = Field( default=None, description="Computes the endpoint from observed anchors; then set end=null." ) @@ -72,6 +90,10 @@ class DragRequest(GestureModel): return self +class DragRequest(DragGeometry): + feedback: ValueFeedback | None = None + + class ClickStep(GestureModel): action: Literal["click"] target: Point | Annotated[str, Field(min_length=1, max_length=512)] = Field( @@ -95,7 +117,7 @@ class ClickStep(GestureModel): ) -class DragStep(DragRequest): +class DragStep(DragGeometry): action: Literal["drag"] @@ -170,6 +192,7 @@ GestureStep = ( class SequenceRequest(GestureModel): steps: list[GestureStep] = Field(min_length=1, max_length=10) + feedback: ValueFeedback | None = None def validate_sequence(payload: dict, limit: int) -> SequenceRequest: @@ -183,7 +206,7 @@ def validate_sequence(payload: dict, limit: int) -> SequenceRequest: return SequenceRequest.model_validate(payload) -def plan_drag(request: DragRequest, width: int, height: int) -> dict: +def plan_drag(request: DragGeometry, width: int, height: int) -> dict: start = pixel_point(request.start, request.coordinate_space, width, height) calibration = None if request.calibration is None: @@ -237,8 +260,23 @@ def plan_drag(request: DragRequest, width: int, height: int) -> dict: "rounding_error_value": abs(end[axis] - endpoint) * abs(slope), "verification": "pending_observation", } + calibration["movement_px"] = abs(end[axis] - start[axis]) + if scale.tolerance is not None: + calibration["pixel_error_budget"] = scale.tolerance / abs(slope) + if calibration["rounding_error_value"] > scale.tolerance: + raise ValueError( + "Pixel rounding exceeds the requested tolerance; zoom and re-observe anchors before input." + ) + if scale.min_movement_px is not None and calibration["movement_px"] < scale.min_movement_px: + raise ValueError( + "Planned movement is below the observed minimum effective distance; zoom and re-observe anchors. No input executed." + ) if start == end: raise ValueError("Drag has no pixel movement; zoom further or choose a different endpoint.") + region = None + if request.region is not None: + region = pixel_region(request.region, request.coordinate_space, width, height) + require_in_region([start, end], region) return { "action": "drag", "coordinates": [*start, *end], @@ -250,6 +288,7 @@ def plan_drag(request: DragRequest, width: int, height: int) -> dict: ], "duration_ms": request.duration_ms, "target_description": request.target_description, + **({"region": region} if region is not None else {}), **({"require_visible": request.require_visible} if request.require_visible else {}), **({"calibration": calibration} if calibration else {}), } diff --git a/artemis/mcp/native_actions.py b/artemis/mcp/native_actions.py index fcc7b41..3815550 100644 --- a/artemis/mcp/native_actions.py +++ b/artemis/mcp/native_actions.py @@ -10,7 +10,10 @@ from artemis.mcp.action_parameters import ( Duration, GestureModel, Point, + Region, pixel_point, + pixel_region, + require_in_region, ) ShortText = Annotated[str, Field(min_length=1, max_length=512)] @@ -71,6 +74,11 @@ class ControlActionRequest(ReadControlRequest): text: Annotated[str, StringConstraints(strip_whitespace=False, max_length=65536)] | None = None selection_start: Annotated[int, Field(strict=True, ge=0)] | None = None selection_end: Annotated[int, Field(strict=True, ge=0)] | None = None + ensure_state: "ControlCondition | None" = Field( + default=None, + description="Desired observed state: skip an already satisfied action; otherwise write ONCE and verify by read-only polling. Never infer state from an icon or missing attributes.", + ) + verify_timeout_ms: Annotated[int, Field(strict=True, ge=1, le=3000)] = 1500 @model_validator(mode="after") def action_arguments(self): @@ -106,6 +114,10 @@ class TouchGestureRequest(GestureModel): coordinate_space: CoordinateSpace = "normalized" target_description: ShortText require_visible: ShortText | None = None + region: Region | None = Field( + default=None, + description="Observed gesture surface bounds in coordinate_space; every finger path must stay inside it.", + ) @model_validator(mode="after") def continuous_timeline(self): @@ -130,6 +142,10 @@ class PinchRequest(GestureModel): coordinate_space: CoordinateSpace = "normalized" target_description: ShortText require_visible: ShortText | None = None + region: Region | None = Field( + default=None, + description="Observed zoom surface bounds in coordinate_space; not the whole screen or date rail. Both fingers must remain inside.", + ) @model_validator(mode="after") def meaningful_pinch(self): @@ -149,6 +165,7 @@ class WaitUntilRequest(ReadControlRequest): "selected", "unselected", "text", + "content_desc", "value", "stable", ] @@ -163,8 +180,8 @@ class WaitUntilRequest(ReadControlRequest): @model_validator(mode="after") def expected_arguments(self): - if (self.expected_text is not None) != (self.state == "text"): - raise ValueError("Only state=text requires expected_text (exact match).") + if (self.expected_text is not None) != (self.state in {"text", "content_desc"}): + raise ValueError("Only state=text/content_desc requires expected_text (exact match).") if (self.expected_value is not None) != (self.state == "value"): raise ValueError("Only state=value requires expected_value.") if self.state != "value" and self.tolerance != 0: @@ -174,6 +191,26 @@ class WaitUntilRequest(ReadControlRequest): return self +class ControlCondition(GestureModel): + state: Literal[ + "checked", "unchecked", "selected", "unselected", "text", "content_desc", "value" + ] + expected_text: ( + Annotated[str, StringConstraints(strip_whitespace=False, max_length=4096)] | None + ) = None + expected_value: float | None = None + tolerance: Annotated[float, Field(ge=0)] = 0 + + @model_validator(mode="after") + def expected_arguments(self): + # Reuse the wait predicate contract without duplicating its semantics. + WaitUntilRequest(selector={"class_name": "condition"}, **self.model_dump()) + return self + + +ControlActionRequest.model_rebuild() + + class ScrollToRequest(GestureModel): container: ControlSelector target: ControlSelector @@ -222,6 +259,10 @@ def plan_touch(request: TouchGestureRequest, width: int, height: int) -> dict: } for frame in request.frames ] + region = None + if request.region is not None: + region = pixel_region(request.region, request.coordinate_space, width, height) + require_in_region([p for frame in frames for p in frame["points"]], region) # Reject trajectories that merge two fingers, including between keyframes. for before, after in zip(frames, frames[1:]): for first in range(len(before["points"])): @@ -240,6 +281,7 @@ def plan_touch(request: TouchGestureRequest, width: int, height: int) -> dict: "coordinate_space": "pixel", "screen_size": [width, height], "target_description": request.target_description, + **({"region": region} if region is not None else {}), **({"require_visible": request.require_visible} if request.require_visible else {}), } @@ -271,6 +313,9 @@ def plan_pinch(request: PinchRequest, width: int, height: int) -> dict: coordinate_space="pixel", target_description=request.target_description, require_visible=request.require_visible, + region=pixel_region(request.region, request.coordinate_space, width, height) + if request.region is not None + else None, ), width, height, diff --git a/artemis/mcp/native_executor.py b/artemis/mcp/native_executor.py index 311adc5..127f5c8 100644 --- a/artemis/mcp/native_executor.py +++ b/artemis/mcp/native_executor.py @@ -158,10 +158,10 @@ async def _pause(deadline: float, interval: float) -> None: def _condition(request: WaitUntilRequest, node: dict) -> bool: state = request.state - if state == "text": - if node.get("password") is True or not isinstance(node.get("text"), str): + if state in {"text", "content_desc"}: + if node.get("password") is True or not isinstance(node.get(state), str): raise ValueError("An exact readable native text value is required.") - return node["text"] == request.expected_text + return node[state] == request.expected_text if state == "value": native_range = _range(node) if native_range is None or request.expected_value is None: @@ -268,6 +268,23 @@ async def _control(request: ControlActionRequest, native: NativeCaller) -> dict: if not inspection["success"]: return _result("control_action", inspection) node = inspection["node"] + condition = None + if request.ensure_state is not None: + condition = WaitUntilRequest( + selector=request.selector, + timeout_ms=request.verify_timeout_ms, + **request.ensure_state.model_dump(), + ) + try: + satisfied = _condition(condition, node) + except ValueError as exc: + return _result( + "control_action", _failure("INVALID_STATE", str(exc)), preflight=inspection + ) + if satisfied: + return _result( + "control_action", inspection, skipped=True, verification="condition_satisfied" + ) invalid = _write_guard(node, request.action, request.expected_token) if invalid is None and request.action == "set_progress": native_range = _range(node) @@ -284,9 +301,30 @@ async def _control(request: ControlActionRequest, native: NativeCaller) -> dict: ) if invalid is not None: return _result("control_action", invalid, preflight=inspection, node_before=node) - args = request.model_dump(mode="json", exclude_none=True) + args = request.model_dump( + mode="json", exclude_none=True, exclude={"ensure_state", "verify_timeout_ms"} + ) args["expected_token"] = node["token"] receipt = await _call(native, "control_action", args) + if receipt["success"] and condition is not None: + verification = await _wait(condition, native) + outcome = dict(receipt) + if not verification["success"]: + outcome.update( + success=False, + code=verification["code"], + error="The input completed, but its requested state was not verified. Do not toggle again blindly.", + ) + return _result( + "control_action", + outcome, + native_receipt=receipt, + preflight=inspection, + postcondition=verification, + node_before=node, + node_after=verification.get("node"), + verification="condition_satisfied" if verification["success"] else "condition_not_met", + ) return _result( "control_action", receipt, diff --git a/artemis/mcp/value_feedback.py b/artemis/mcp/value_feedback.py new file mode 100644 index 0000000..20a6e96 --- /dev/null +++ b/artemis/mcp/value_feedback.py @@ -0,0 +1,284 @@ +"""Optional, task-local feedback for numeric adjustments; never retries an input.""" + +import asyncio +import json +import math +import re +from collections.abc import Awaitable, Callable +from asyncio import sleep +from time import monotonic +from typing import Annotated, Literal + +from pydantic import Field, model_validator + +from artemis.mcp.action_parameters import GestureModel +from artemis.mcp.action_types import ActionCode, ActionResult +from artemis.mcp.native_actions import ControlSelector +from artemis.mcp.native_executor import NativeCaller, execute_native + + +class ValueFeedback(GestureModel): + selector: ControlSelector = Field( + description="Unique, observed persistent value readout, not a transient drag tooltip. Copy its stable selector, excluding changing text." + ) + value_format: Literal["range", "number", "clock"] = Field( + description="range reads native range.current; number requires an entire numeric text; clock requires exactly one HH:MM:SS in readable text and returns seconds since midnight." + ) + target_value: float + tolerance: Annotated[float, Field(ge=0)] + stable_ms: Annotated[int, Field(strict=True, ge=100, le=5000)] = 1000 + timeout_ms: Annotated[int, Field(strict=True, ge=200, le=10000)] = 3000 + + @model_validator(mode="after") + def bounded_observation(self): + if self.stable_ms >= self.timeout_ms: + raise ValueError("stable_ms must be smaller than timeout_ms.") + return self + + +def read_value(node: dict, value_format: str) -> float: + if node.get("password") is True or node.get("visible") is not True: + raise ValueError("The value must be visible, readable and non-password.") + if value_format == "range": + native_range = node.get("range") + if not isinstance(native_range, dict): + raise ValueError("No native numeric range; do not guess a value from the gesture.") + values = [native_range.get(k) for k in ("min", "current", "max")] + if any(type(v) not in (int, float) or not math.isfinite(v) for v in values): + raise ValueError("Invalid native numeric range.") + low, current, high = values + if not low <= current <= high: + raise ValueError("Invalid native numeric range bounds.") + return float(current) + text = node.get("text") + if not isinstance(text, str) or len(text) > 4096: + raise ValueError("The selected node has no bounded readable text.") + if value_format == "clock": + matches = re.findall(r"(? str: + """Ignore tiny coordinate jitter; a new scale, axis or meaningful span is a new strategy.""" + strategy = [] + for step in planned: + if step["action"] == "drag": + x1, y1, x2, y2 = step["coordinates"] + dx, dy = x2 - x1, y2 - y1 + calibration = step.get("calibration", {}) + strategy.append( + [ + "drag", + "x" if abs(dx) >= abs(dy) else "y", + 1 if (dx if abs(dx) >= abs(dy) else dy) > 0 else -1, + math.hypot(dx, dy), + step["duration_ms"], + round(calibration.get("value_per_pixel", 0), 8), + step.get("region"), + ] + ) + else: + strategy.append(step) + return json.dumps(strategy, sort_keys=True, ensure_ascii=False) + + +def same_strategy(left: str, right: str) -> bool: + """Compare relative changes; crossing an arbitrary pixel bucket is not a new strategy.""" + before, after = json.loads(left), json.loads(right) + if len(before) != len(after): + return False + for old, new in zip(before, after, strict=True): + if isinstance(old, list) and isinstance(new, list) and old[0] == new[0] == "drag": + if old[1:3] != new[1:3] or old[6] != new[6]: + return False + if any(not 0.5 < new[i] / old[i] < 2 for i in (3, 4)): + return False + if not math.isclose(old[5], new[5], rel_tol=0.05): + return False + elif old != new: + return False + return True + + +class ValueFeedbackRunner: + """Bounded memory owned by one device actuator/task, never persisted or shared.""" + + def __init__(self): + self.history: dict[str, tuple[str, float, int]] = {} + + async def _observe( + self, request: ValueFeedback, native: NativeCaller, *, stable: bool + ) -> tuple[float, dict]: + deadline = monotonic() + request.timeout_ms / 1000 + previous = None + since = monotonic() + samples = 0 + async with asyncio.timeout(request.timeout_ms / 1000): + while monotonic() < deadline: + receipt = await execute_native( + "read_control", + {"selector": request.selector.model_dump(exclude_none=True)}, + 1, + 1, + native, + ) + if monotonic() >= deadline: + raise TimeoutError("The value observation arrived after the feedback deadline.") + if not receipt["success"]: + if stable and receipt.get("code") in { + "NOT_FOUND", + "READ_FAILED", + "UNAVAILABLE", + }: + previous = None + since = monotonic() + await sleep(min(0.1, max(0, deadline - monotonic()))) + continue + raise ValueError( + f"Value read failed: {receipt.get('code')}; no value inferred." + ) + value = read_value(receipt["node"], request.value_format) + samples += 1 + evidence = { + "node": receipt["node"], + "samples": samples, + "stable_for_ms": round((monotonic() - since) * 1000) + if stable and value == previous + else 0, + } + if not stable: + return value, evidence + if value != previous: + previous, since = value, monotonic() + elif monotonic() - since >= request.stable_ms / 1000: + return value, evidence + await sleep(min(0.1, max(0, deadline - monotonic()))) + raise TimeoutError("The value did not settle within the feedback deadline.") + + async def run( + self, + request: ValueFeedback | None, + planned: list[dict], + action: str, + native: NativeCaller, + operation: Callable[[], Awaitable[ActionResult]], + ) -> ActionResult: + if request is None: + return await operation() + key = json.dumps( + [ + request.selector.model_dump(), + request.value_format, + request.target_value, + request.tolerance, + ], + sort_keys=True, + ) + strategy = strategy_key(planned) + try: + before, before_observation = await self._observe(request, native, stable=False) + except (ValueError, TimeoutError) as exc: + self.history.pop(key, None) + return ActionResult.failure( + action, + f"Feedback unavailable; no input executed: {exc}", + code=ActionCode.TARGET_NOT_FOUND, + metadata={ + "execution": "not_started", + "side_effect": "none", + "verification": "unavailable", + }, + ) + previous = self.history.get(key) + stalled = ( + previous[2] + if previous and previous[1] == before and same_strategy(previous[0], strategy) + else 0 + ) + feedback = { + "selector": request.selector.model_dump(exclude_none=True), + "before_value": before, + "before_observation": before_observation, + "target_value": request.target_value, + "tolerance": request.tolerance, + "value_format": request.value_format, + } + if stalled >= 2: + return ActionResult.failure( + action, + "NO_PROGRESS: two adjustments did not reduce the actual residual. No further input executed. Re-observe the gesture surface and scale; zoom, change the meaningful drag span or use a native setter. Do not repeat the same small drag.", + metadata={ + "execution": "not_started", + "side_effect": "none", + "verification": "strategy_change_required", + "value_feedback": {**feedback, "consecutive_no_progress": stalled}, + }, + ) + result = await operation() + if not result.ok: + self.history.pop(key, None) + return result + try: + after, after_observation = await self._observe(request, native, stable=True) + except (ValueError, TimeoutError) as exc: + self.history.pop(key, None) + return result.model_copy( + update={ + "ok": False, + "code": ActionCode.TIMEOUT + if isinstance(exc, TimeoutError) + else ActionCode.TARGET_NOT_FOUND, + "message": f"Input completed; value feedback unavailable: {exc}. Do not replay the input blindly.", + "metadata": { + **result.metadata, + "execution": "completed", + "verification": "unavailable", + "value_feedback": feedback, + }, + } + ) + residual = after - request.target_value + matched = abs(residual) <= request.tolerance + improved = abs(residual) < abs(before - request.target_value) - 1e-9 + stalled = 0 if matched or improved else stalled + 1 + if len(self.history) >= 32: + self.history.pop(next(iter(self.history))) + self.history[key] = (strategy, after, stalled) + feedback.update( + after_observation=after_observation, + after_value=after, + residual=residual, + changed=after != before, + improved=improved, + within_tolerance=matched, + consecutive_no_progress=stalled, + ) + return result.model_copy( + update={ + "ok": matched, + "code": ActionCode.OK if matched else ActionCode.DEVICE_ERROR, + "message": f"Input completed. Observed value={after:g}, target={request.target_value:g}, residual={residual:g}, tolerance={request.tolerance:g}. " + + ( + "Value condition met; complete task acceptance separately." + if matched + else "Value condition not met. Use this observed residual to change strategy; never infer success from dispatch." + ), + "metadata": { + **result.metadata, + "execution": "completed", + "verification": "value_matched" if matched else "value_not_reached", + "value_feedback": feedback, + }, + } + ) diff --git a/docs/android-native-actions.md b/docs/android-native-actions.md index 11bb90a..f16e34d 100644 --- a/docs/android-native-actions.md +++ b/docs/android-native-actions.md @@ -37,6 +37,24 @@ Flash、普通 Pro 和探索共用同一套动作参数、助手执行入口与 ## 时间轴精度 +精确调整可绑定可选的 `feedback`：`selector` 指向原生可读取、唯一且持久的真实选中值，`value_format` 为 `range`（原生范围）、`number`（完整数字文本）或 `clock`（文本中唯一的 HH:MM:SS，换算为当天秒数），并提供同单位的 `target_value` 与 `tolerance`。不要选择录像卡片、视频水印或短暂拖动浮层，也不要在 selector 中包含会变化的读数文本。无法建立真实绑定时继续视觉观测，不能编造控件或把手势派发当作读值。 + +单独拖动把 feedback 放在 `drag` 参数上；“拖动后暂停”等组合把它放在 `action_sequence` 顶层，不放在 drag 成员上。输入前确认可读，整组成功后再等待读数连续稳定 `stable_ms`（默认 1000，范围 100–5000），总观察不超过 `timeout_ms`（默认 3000，上限 10000，必须大于稳定时长）。时钟稳定时长应覆盖其刷新周期。回执含 before_value、after_value、residual、within_tolerance 和连续无效次数；数值条件满足与最终业务验收分开。输入完成但读数失败或未达标仍保留完成回执，不会自动重发。 + +同一执行实例内，同读数、目标和策略连续两次没有减小实际误差，第三次拒绝输入，提示重新观察并放大、改变有效移动距离或使用原生设值。微改坐标不算换策略；新刻度比例、轴向或明显不同的距离可重新尝试。读数改变或任务实例更换后重新判断。此守卫依赖显式 feedback，不声称能够自动理解所有自绘控件。 + +所有手指路径可以通过 `region=[left,top,right,bottom]` 限定到已观察到的手势区域；坐标单位跟随 coordinate_space，右/下边界不包含在区域内。拖动、缩放或多指轨迹离开区域时整组在首个输入前拒绝；区域只验证几何边界，不证明窗口未被遮挡。calibration.tolerance 检查像素取整误差，回执给出每像素数值和允许像素误差；min_movement_px 只能填实测起效距离，过小则拒绝并要求放大。几何误差不包括读图、惯性与 APP 的内部吸附误差。 + +暂停或勾选等状态可用 `control_action.ensure_state`，支持 checked/unchecked、selected/unselected、text/content_desc 精确匹配及 value/tolerance。先 read_control 确认其语义，稳定 selector 不包含会改变的文字/描述。达到状态时跳过；未达到时仅操作一次，再只读等待 verify_timeout_ms（默认 1500，上限 3000）。状态缺失不当作 false，核验失败保留已完成输入，不继续盲点切换。不能假定 selected 就表示暂停。 + +例如，已观察确认资源 ID `player` 的描述为“播放”时表示暂停，可将以下成员放在拖动后： + +```json +{"action":"control_action","request":{"selector":{"resource_id":"player"},"action":"click","ensure_state":{"state":"content_desc","expected_text":"播放"}}} +``` + +示例 ID 和状态含义必须换成实际观察结果。绑定时间反馈示例为 `{"selector":{"resource_id":"observed_current_time"},"value_format":"clock","target_value":63737,"tolerance":5}`，对应目标 17:42:17±5 秒；不用于日期验收或跨日时间差。 + 双指距离变化不一定等于 APP 比例尺变化：先小幅缩放，再读取实际刻度，建立像素与时间的局部对应。平移后等待停稳，读取残差再修正；播放中应先暂停后验收。 如果常驻读数通过标签、节点角色或层级明确绑定到固定指针当前选时，且在暂停或独立停稳观测中持续显示、与刻度一致，可以直接作为秒级证据，无须再要求粗像素插值独立达到同样精度。事件卡片、录像水印和短暂拖动浮层不能代替当前选时。真实冲突、来源不明或缺少稳定证据时继续保留不确定结论；没有可信读数时仍用刻度插值并计入读图误差。 diff --git a/skills/artemis-use/references/native-actions.md b/skills/artemis-use/references/native-actions.md index 7e5ac2d..5bd39b7 100644 --- a/skills/artemis-use/references/native-actions.md +++ b/skills/artemis-use/references/native-actions.md @@ -37,6 +37,24 @@ Flash、普通 Pro 和探索共用同一套动作参数、助手执行入口与 ## 时间轴精度 +精确调整可绑定可选的 `feedback`：`selector` 指向原生可读取、唯一且持久的真实选中值，`value_format` 为 `range`（原生范围）、`number`（完整数字文本）或 `clock`（文本中唯一的 HH:MM:SS，换算为当天秒数），并提供同单位的 `target_value` 与 `tolerance`。不要选择录像卡片、视频水印或短暂拖动浮层，也不要在 selector 中包含会变化的读数文本。无法建立真实绑定时继续视觉观测，不能编造控件或把手势派发当作读值。 + +单独拖动把 feedback 放在 `drag` 参数上；“拖动后暂停”等组合把它放在 `action_sequence` 顶层，不放在 drag 成员上。输入前确认可读，整组成功后再等待读数连续稳定 `stable_ms`（默认 1000，范围 100–5000），总观察不超过 `timeout_ms`（默认 3000，上限 10000，必须大于稳定时长）。时钟稳定时长应覆盖其刷新周期。回执含 before_value、after_value、residual、within_tolerance 和连续无效次数；数值条件满足与最终业务验收分开。输入完成但读数失败或未达标仍保留完成回执，不会自动重发。 + +同一执行实例内，同读数、目标和策略连续两次没有减小实际误差，第三次拒绝输入，提示重新观察并放大、改变有效移动距离或使用原生设值。微改坐标不算换策略；新刻度比例、轴向或明显不同的距离可重新尝试。读数改变或任务实例更换后重新判断。此守卫依赖显式 feedback，不声称能够自动理解所有自绘控件。 + +所有手指路径可以通过 `region=[left,top,right,bottom]` 限定到已观察到的手势区域；坐标单位跟随 coordinate_space，右/下边界不包含在区域内。拖动、缩放或多指轨迹离开区域时整组在首个输入前拒绝；区域只验证几何边界，不证明窗口未被遮挡。calibration.tolerance 检查像素取整误差，回执给出每像素数值和允许像素误差；min_movement_px 只能填实测起效距离，过小则拒绝并要求放大。几何误差不包括读图、惯性与 APP 的内部吸附误差。 + +暂停或勾选等状态可用 `control_action.ensure_state`，支持 checked/unchecked、selected/unselected、text/content_desc 精确匹配及 value/tolerance。先 read_control 确认其语义，稳定 selector 不包含会改变的文字/描述。达到状态时跳过；未达到时仅操作一次，再只读等待 verify_timeout_ms（默认 1500，上限 3000）。状态缺失不当作 false，核验失败保留已完成输入，不继续盲点切换。不能假定 selected 就表示暂停。 + +例如，已观察确认资源 ID `player` 的描述为“播放”时表示暂停，可将以下成员放在拖动后： + +```json +{"action":"control_action","request":{"selector":{"resource_id":"player"},"action":"click","ensure_state":{"state":"content_desc","expected_text":"播放"}}} +``` + +示例 ID 和状态含义必须换成实际观察结果。绑定时间反馈示例为 `{"selector":{"resource_id":"observed_current_time"},"value_format":"clock","target_value":63737,"tolerance":5}`，对应目标 17:42:17±5 秒；不用于日期验收或跨日时间差。 + 双指距离变化不一定等于 APP 比例尺变化：先小幅缩放，再读取实际刻度，建立像素与时间的局部对应。平移后等待停稳，读取残差再修正；播放中应先暂停后验收。 如果常驻读数通过标签、节点角色或层级明确绑定到固定指针当前选时，且在暂停或独立停稳观测中持续显示、与刻度一致，可以直接作为秒级证据，无须再要求粗像素插值独立达到同样精度。事件卡片、录像水印和短暂拖动浮层不能代替当前选时。真实冲突、来源不明或缺少稳定证据时继续保留不确定结论；没有可信读数时仍用刻度插值并计入读图误差。 diff --git a/tests/fixtures/action_surfaces/action_server_manifest.json b/tests/fixtures/action_surfaces/action_server_manifest.json index 3e920f8..084d18b 100644 --- a/tests/fixtures/action_surfaces/action_server_manifest.json +++ b/tests/fixtures/action_surfaces/action_server_manifest.json @@ -103,6 +103,60 @@ "title": "ControlActionStep", "type": "object" }, + "ControlCondition": { + "additionalProperties": false, + "properties": { + "state": { + "enum": [ + "checked", + "unchecked", + "selected", + "unselected", + "text", + "content_desc", + "value" + ], + "title": "State", + "type": "string" + }, + "expected_text": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Text" + }, + "expected_value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Value" + }, + "tolerance": { + "default": 0, + "minimum": 0, + "title": "Tolerance", + "type": "number" + } + }, + "required": [ + "state" + ], + "title": "ControlCondition", + "type": "object" + }, "ControlSelector": { "additionalProperties": false, "description": "Every supplied attribute is an exact AND match against a fresh native node.", @@ -242,6 +296,34 @@ "default": null, "description": "Required for pan: the fixed indicator's observed axis position.", "title": "Indicator Position" + }, + "tolerance": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Allowed value error. Reject a plan whose pixel rounding exceeds it; this is geometric error only, not proof of application precision.", + "title": "Tolerance" + }, + "min_movement_px": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Previously measured minimum effective drag distance in pixels. If this calibrated move is smaller, refuse input and zoom first. Never guess a device-wide threshold.", + "title": "Min Movement Px" } }, "required": [ @@ -304,6 +386,24 @@ "title": "Duration Ms", "type": "integer" }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed gesture surface rectangle in coordinate_space, excluding date/navigation rails and unrelated overlays.", + "title": "Region" + }, "calibration": { "anyOf": [ { @@ -464,6 +564,24 @@ "default": null, "title": "Require Visible" }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed zoom surface bounds in coordinate_space; not the whole screen or date rail. Both fingers must remain inside.", + "title": "Region" + }, "action": { "const": "pinch", "title": "Action", @@ -591,6 +709,25 @@ ], "default": null, "title": "Selection End" + }, + "ensure_state": { + "anyOf": [ + { + "$ref": "#/$defs/ControlCondition" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired observed state: skip an already satisfied action; otherwise write ONCE and verify by read-only polling. Never infer state from an icon or missing attributes." + }, + "verify_timeout_ms": { + "default": 1500, + "maximum": 3000, + "minimum": 1, + "title": "Verify Timeout Ms", + "type": "integer" } }, "required": [ @@ -672,6 +809,24 @@ "default": null, "title": "Require Visible" }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed gesture surface bounds in coordinate_space; every finger path must stay inside it.", + "title": "Region" + }, "action": { "const": "touch_gesture", "title": "Action", @@ -686,6 +841,56 @@ "title": "TouchGestureStep", "type": "object" }, + "ValueFeedback": { + "additionalProperties": false, + "properties": { + "selector": { + "$ref": "#/$defs/ControlSelector", + "description": "Unique, observed persistent value readout, not a transient drag tooltip. Copy its stable selector, excluding changing text." + }, + "value_format": { + "description": "range reads native range.current; number requires an entire numeric text; clock requires exactly one HH:MM:SS in readable text and returns seconds since midnight.", + "enum": [ + "range", + "number", + "clock" + ], + "title": "Value Format", + "type": "string" + }, + "target_value": { + "title": "Target Value", + "type": "number" + }, + "tolerance": { + "minimum": 0, + "title": "Tolerance", + "type": "number" + }, + "stable_ms": { + "default": 1000, + "maximum": 5000, + "minimum": 100, + "title": "Stable Ms", + "type": "integer" + }, + "timeout_ms": { + "default": 3000, + "maximum": 10000, + "minimum": 200, + "title": "Timeout Ms", + "type": "integer" + } + }, + "required": [ + "selector", + "value_format", + "target_value", + "tolerance" + ], + "title": "ValueFeedback", + "type": "object" + }, "WaitStep": { "additionalProperties": false, "properties": { @@ -725,6 +930,7 @@ "selected", "unselected", "text", + "content_desc", "value", "stable" ], @@ -830,6 +1036,17 @@ }, "title": "Steps", "type": "array" + }, + "feedback": { + "anyOf": [ + { + "$ref": "#/$defs/ValueFeedback" + }, + { + "type": "null" + } + ], + "default": null } }, "required": [ @@ -931,6 +1148,100 @@ "description": "Drag with exact coordinates or an observed two-anchor calibration.", "inputSchema": { "$defs": { + "ControlSelector": { + "additionalProperties": false, + "description": "Every supplied attribute is an exact AND match against a fresh native node.", + "properties": { + "resource_id": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Resource Id" + }, + "text": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Text" + }, + "content_desc": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Content Desc" + }, + "class_name": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Class Name" + }, + "package_name": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Package Name" + }, + "bounds": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Bounds" + } + }, + "title": "ControlSelector", + "type": "object" + }, "DragCalibration": { "additionalProperties": false, "properties": { @@ -976,6 +1287,34 @@ "default": null, "description": "Required for pan: the fixed indicator's observed axis position.", "title": "Indicator Position" + }, + "tolerance": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Allowed value error. Reject a plan whose pixel rounding exceeds it; this is geometric error only, not proof of application precision.", + "title": "Tolerance" + }, + "min_movement_px": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Previously measured minimum effective drag distance in pixels. If this calibrated move is smaller, refuse input and zoom first. Never guess a device-wide threshold.", + "title": "Min Movement Px" } }, "required": [ @@ -1007,6 +1346,56 @@ ], "title": "ScaleAnchor", "type": "object" + }, + "ValueFeedback": { + "additionalProperties": false, + "properties": { + "selector": { + "$ref": "#/$defs/ControlSelector", + "description": "Unique, observed persistent value readout, not a transient drag tooltip. Copy its stable selector, excluding changing text." + }, + "value_format": { + "description": "range reads native range.current; number requires an entire numeric text; clock requires exactly one HH:MM:SS in readable text and returns seconds since midnight.", + "enum": [ + "range", + "number", + "clock" + ], + "title": "Value Format", + "type": "string" + }, + "target_value": { + "title": "Target Value", + "type": "number" + }, + "tolerance": { + "minimum": 0, + "title": "Tolerance", + "type": "number" + }, + "stable_ms": { + "default": 1000, + "maximum": 5000, + "minimum": 100, + "title": "Stable Ms", + "type": "integer" + }, + "timeout_ms": { + "default": 3000, + "maximum": 10000, + "minimum": 200, + "title": "Timeout Ms", + "type": "integer" + } + }, + "required": [ + "selector", + "value_format", + "target_value", + "tolerance" + ], + "title": "ValueFeedback", + "type": "object" } }, "properties": { @@ -1050,6 +1439,34 @@ "title": "Duration Ms", "type": "integer" }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Region" + }, + "feedback": { + "anyOf": [ + { + "$ref": "#/$defs/ValueFeedback" + }, + { + "type": "null" + } + ], + "default": null + }, "calibration": { "anyOf": [ { @@ -2582,6 +2999,23 @@ ], "default": null, "title": "Require Visible" + }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Region" } }, "required": [ @@ -2756,6 +3190,23 @@ ], "default": null, "title": "Require Visible" + }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Region" } }, "required": [ @@ -3054,9 +3505,63 @@ } }, "control_action": { - "description": "[ACTION] Perform ONE supported native action on a freshly resolved unique control. Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. action=set_progress requires a finite in-range value; set_selection requires selection_start/end in UTF-16 code units (never split an emoji). set_text and replace_selection require text, including empty text; whitespace is preserved. Other actions take no text/value/selection. ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback or replay. Focus or selection changes may move the target; before subsequent edits, use wait_until state=stable when needed. STALE means no action started; observe again before rebuilding the request. Never resend an unknown write. An accepted action is not business verification; inspect the returned/current state.", + "description": "[ACTION] Perform ONE supported native action on a freshly resolved unique control. Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. For a desired state use ensure_state (checked/selected/text/content_desc/value), then verify_timeout_ms: skip an already satisfied state, otherwise write ONCE and poll read-only. Copy observed state semantics; do not assume selected means paused. Bind a stable selector excluding text/content_desc that the action changes. A failed postcondition does not undo the completed input and must not cause an automatic second toggle. action=set_progress requires a finite in-range value; set_selection requires selection_start/end in UTF-16 code units (never split an emoji). set_text and replace_selection require text, including empty text; whitespace is preserved. Other actions take no text/value/selection. ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback or replay. Focus or selection changes may move the target; before subsequent edits, use wait_until state=stable when needed. STALE means no action started; observe again before rebuilding the request. Never resend an unknown write. An accepted action is not business verification; inspect the returned/current state.", "inputSchema": { "$defs": { + "ControlCondition": { + "additionalProperties": false, + "properties": { + "state": { + "enum": [ + "checked", + "unchecked", + "selected", + "unselected", + "text", + "content_desc", + "value" + ], + "title": "State", + "type": "string" + }, + "expected_text": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Text" + }, + "expected_value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Value" + }, + "tolerance": { + "default": 0, + "minimum": 0, + "title": "Tolerance", + "type": "number" + } + }, + "required": [ + "state" + ], + "title": "ControlCondition", + "type": "object" + }, "ControlSelector": { "additionalProperties": false, "description": "Every supplied attribute is an exact AND match against a fresh native node.", @@ -3240,6 +3745,24 @@ ], "default": null, "title": "Selection End" + }, + "ensure_state": { + "anyOf": [ + { + "$ref": "#/$defs/ControlCondition" + }, + { + "type": "null" + } + ], + "default": null + }, + "verify_timeout_ms": { + "default": 1500, + "maximum": 3000, + "minimum": 1, + "title": "Verify Timeout Ms", + "type": "integer" } }, "required": [ @@ -3452,6 +3975,7 @@ "selected", "unselected", "text", + "content_desc", "value", "stable" ], diff --git a/tests/fixtures/action_surfaces/operator_shells.json b/tests/fixtures/action_surfaces/operator_shells.json index e545336..35b9db6 100644 --- a/tests/fixtures/action_surfaces/operator_shells.json +++ b/tests/fixtures/action_surfaces/operator_shells.json @@ -54,7 +54,7 @@ "type": "function", "function": { "name": "action_sequence", - "description": "[ACTION] General Android composer: execute ensure_visible, click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Prefer ensure_visible(target=observed panel control ID, opener=observed UI opener ID) followed by click/drag. This checks live UI, skips opening an already-visible panel, or taps its opener ONCE and waits up to timeout_ms (default 1500, max 3000) for the target. Each preparation counts as one member. Include it again in EVERY panel-operation call, even when the previous screenshot showed the panel open; do not split preparation and operation into different model turns. It never retries a business action. Copy all resource IDs VERBATIM, including ':id/' separators, never ':id:'. If preparation fails, re-observe/correct the ID; never guess fallback coordinates. Legacy click.unless_visible is ONLY for an opener, NEVER for an operation such as a +/- button: visible means SKIP, not require-visible. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.", + "description": "[ACTION] General Android composer: execute ensure_visible, click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Prefer ensure_visible(target=observed panel control ID, opener=observed UI opener ID) followed by click/drag. This checks live UI, skips opening an already-visible panel, or taps its opener ONCE and waits up to timeout_ms (default 1500, max 3000) for the target. Each preparation counts as one member. Include it again in EVERY panel-operation call, even when the previous screenshot showed the panel open; do not split preparation and operation into different model turns. It never retries a business action. Copy all resource IDs VERBATIM, including ':id/' separators, never ':id:'. If preparation fails, re-observe/correct the ID; never guess fallback coordinates. Legacy click.unless_visible is ONLY for an opener, NEVER for an operation such as a +/- button: visible means SKIP, not require-visible. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum. For a measurable target, bind feedback to its persistent native readout: final feedback is read AFTER ALL members, including any pause, finish. Never use a release-time tooltip. Use control_action.request.ensure_state for observed checked/selected/text/value state, so an already satisfied state is skipped rather than toggled. A playback icon alone does not prove a native boolean state: inspect it first. Feedback blocks the third consecutive ineffective strategy; zoom or increase the meaningful movement, then re-observe.", "parameters": { "properties": { "steps": { @@ -193,6 +193,24 @@ "title": "Duration Ms", "type": "integer" }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed gesture surface rectangle in coordinate_space, excluding date/navigation rails and unrelated overlays.", + "title": "Region" + }, "calibration": { "anyOf": [ { @@ -258,6 +276,34 @@ "default": null, "description": "Required for pan: the fixed indicator's observed axis position.", "title": "Indicator Position" + }, + "tolerance": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Allowed value error. Reject a plan whose pixel rounding exceeds it; this is geometric error only, not proof of application precision.", + "title": "Tolerance" + }, + "min_movement_px": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Previously measured minimum effective drag distance in pixels. If this calibrated move is smaller, refuse input and zoom first. Never guess a device-wide threshold.", + "title": "Min Movement Px" } }, "required": [ @@ -446,6 +492,24 @@ "default": null, "title": "Require Visible" }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed zoom surface bounds in coordinate_space; not the whole screen or date rail. Both fingers must remain inside.", + "title": "Region" + }, "action": { "const": "pinch", "title": "Action", @@ -531,6 +595,24 @@ "default": null, "title": "Require Visible" }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed gesture surface bounds in coordinate_space; every finger path must stay inside it.", + "title": "Region" + }, "action": { "const": "touch_gesture", "title": "Action", @@ -734,6 +816,76 @@ ], "default": null, "title": "Selection End" + }, + "ensure_state": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "state": { + "enum": [ + "checked", + "unchecked", + "selected", + "unselected", + "text", + "content_desc", + "value" + ], + "title": "State", + "type": "string" + }, + "expected_text": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Text" + }, + "expected_value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Value" + }, + "tolerance": { + "default": 0, + "minimum": 0, + "title": "Tolerance", + "type": "number" + } + }, + "required": [ + "state" + ], + "title": "ControlCondition", + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired observed state: skip an already satisfied action; otherwise write ONCE and verify by read-only polling. Never infer state from an icon or missing attributes." + }, + "verify_timeout_ms": { + "default": 1500, + "maximum": 3000, + "minimum": 1, + "title": "Verify Timeout Ms", + "type": "integer" } }, "required": [ @@ -859,6 +1011,7 @@ "selected", "unselected", "text", + "content_desc", "value", "stable" ], @@ -936,6 +1089,155 @@ "maxItems": 10, "minItems": 1, "type": "array" + }, + "feedback": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "selector": { + "additionalProperties": false, + "description": "Unique, observed persistent value readout, not a transient drag tooltip. Copy its stable selector, excluding changing text.", + "properties": { + "resource_id": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Resource Id" + }, + "text": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Text" + }, + "content_desc": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Content Desc" + }, + "class_name": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Class Name" + }, + "package_name": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Package Name" + }, + "bounds": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Bounds" + } + }, + "title": "ControlSelector", + "type": "object" + }, + "value_format": { + "description": "range reads native range.current; number requires an entire numeric text; clock requires exactly one HH:MM:SS in readable text and returns seconds since midnight.", + "enum": [ + "range", + "number", + "clock" + ], + "title": "Value Format", + "type": "string" + }, + "target_value": { + "title": "Target Value", + "type": "number" + }, + "tolerance": { + "minimum": 0, + "title": "Tolerance", + "type": "number" + }, + "stable_ms": { + "default": 1000, + "maximum": 5000, + "minimum": 100, + "title": "Stable Ms", + "type": "integer" + }, + "timeout_ms": { + "default": 3000, + "maximum": 10000, + "minimum": 200, + "title": "Timeout Ms", + "type": "integer" + } + }, + "required": [ + "selector", + "value_format", + "target_value", + "tolerance" + ], + "title": "ValueFeedback", + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional actual value/target comparison after the WHOLE sequence. clock uses seconds since midnight; exact bound and stable observation are checked locally, not inferred from dispatch." } }, "required": [ @@ -990,6 +1292,172 @@ "default": 300, "type": "integer" }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed gesture surface rectangle, in coordinate_space. Keep both endpoints inside." + }, + "feedback": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "selector": { + "additionalProperties": false, + "description": "Unique, observed persistent value readout, not a transient drag tooltip. Copy its stable selector, excluding changing text.", + "properties": { + "resource_id": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Resource Id" + }, + "text": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Text" + }, + "content_desc": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Content Desc" + }, + "class_name": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Class Name" + }, + "package_name": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Package Name" + }, + "bounds": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Bounds" + } + }, + "title": "ControlSelector", + "type": "object" + }, + "value_format": { + "description": "range reads native range.current; number requires an entire numeric text; clock requires exactly one HH:MM:SS in readable text and returns seconds since midnight.", + "enum": [ + "range", + "number", + "clock" + ], + "title": "Value Format", + "type": "string" + }, + "target_value": { + "title": "Target Value", + "type": "number" + }, + "tolerance": { + "minimum": 0, + "title": "Tolerance", + "type": "number" + }, + "stable_ms": { + "default": 1000, + "maximum": 5000, + "minimum": 100, + "title": "Stable Ms", + "type": "integer" + }, + "timeout_ms": { + "default": 3000, + "maximum": 10000, + "minimum": 200, + "title": "Timeout Ms", + "type": "integer" + } + }, + "required": [ + "selector", + "value_format", + "target_value", + "tolerance" + ], + "title": "ValueFeedback", + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Read the persistent actual value after the drag. For drag then pause, put feedback on action_sequence instead." + }, "calibration": { "anyOf": [ { @@ -1055,6 +1523,34 @@ "default": null, "description": "Required for pan: the fixed indicator's observed axis position.", "title": "Indicator Position" + }, + "tolerance": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Allowed value error. Reject a plan whose pixel rounding exceeds it; this is geometric error only, not proof of application precision.", + "title": "Tolerance" + }, + "min_movement_px": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Previously measured minimum effective drag distance in pixels. If this calibrated move is smaller, refuse input and zoom first. Never guess a device-wide threshold.", + "title": "Min Movement Px" } }, "required": [ @@ -1162,6 +1658,23 @@ } ], "default": null + }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed zoom surface bounds in coordinate_space; not the whole screen or date rail. Both fingers must remain inside." } }, "required": [ @@ -1239,6 +1752,23 @@ } ], "default": null + }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed gesture surface bounds in coordinate_space; every finger path must stay inside it." } }, "required": [ @@ -1355,7 +1885,7 @@ "type": "function", "function": { "name": "control_action", - "description": "[ACTION] Perform ONE supported native action on a freshly resolved unique control. Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. action=set_progress requires a finite in-range value; set_selection requires selection_start/end in UTF-16 code units (never split an emoji). set_text and replace_selection require text, including empty text; whitespace is preserved. Other actions take no text/value/selection. ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback or replay. Focus or selection changes may move the target; before subsequent edits, use wait_until state=stable when needed. STALE means no action started; observe again before rebuilding the request. Never resend an unknown write. An accepted action is not business verification; inspect the returned/current state.", + "description": "[ACTION] Perform ONE supported native action on a freshly resolved unique control. Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. For a desired state use ensure_state (checked/selected/text/content_desc/value), then verify_timeout_ms: skip an already satisfied state, otherwise write ONCE and poll read-only. Copy observed state semantics; do not assume selected means paused. Bind a stable selector excluding text/content_desc that the action changes. A failed postcondition does not undo the completed input and must not cause an automatic second toggle. action=set_progress requires a finite in-range value; set_selection requires selection_start/end in UTF-16 code units (never split an emoji). set_text and replace_selection require text, including empty text; whitespace is preserved. Other actions take no text/value/selection. ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback or replay. Focus or selection changes may move the target; before subsequent edits, use wait_until state=stable when needed. STALE means no action started; observe again before rebuilding the request. Never resend an unknown write. An accepted action is not business verification; inspect the returned/current state.", "parameters": { "properties": { "selector": { @@ -1523,6 +2053,75 @@ } ], "default": null + }, + "ensure_state": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "state": { + "enum": [ + "checked", + "unchecked", + "selected", + "unselected", + "text", + "content_desc", + "value" + ], + "title": "State", + "type": "string" + }, + "expected_text": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Text" + }, + "expected_value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Value" + }, + "tolerance": { + "default": 0, + "minimum": 0, + "title": "Tolerance", + "type": "number" + } + }, + "required": [ + "state" + ], + "title": "ControlCondition", + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired observed state: skip an already satisfied action; otherwise write ONCE and verify by read-only polling. Never infer state from an icon or missing attributes." + }, + "verify_timeout_ms": { + "default": 1500, + "maximum": 3000, + "minimum": 1, + "type": "integer" } }, "required": [ @@ -1638,6 +2237,7 @@ "selected", "unselected", "text", + "content_desc", "value", "stable" ], diff --git a/tests/fixtures/action_surfaces/tool_declarations.json b/tests/fixtures/action_surfaces/tool_declarations.json index 7051e91..1918f88 100644 --- a/tests/fixtures/action_surfaces/tool_declarations.json +++ b/tests/fixtures/action_surfaces/tool_declarations.json @@ -284,6 +284,172 @@ "default": 300, "type": "integer" }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed gesture surface rectangle, in coordinate_space. Keep both endpoints inside." + }, + "feedback": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "selector": { + "additionalProperties": false, + "description": "Unique, observed persistent value readout, not a transient drag tooltip. Copy its stable selector, excluding changing text.", + "properties": { + "resource_id": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Resource Id" + }, + "text": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Text" + }, + "content_desc": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Content Desc" + }, + "class_name": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Class Name" + }, + "package_name": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Package Name" + }, + "bounds": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Bounds" + } + }, + "title": "ControlSelector", + "type": "object" + }, + "value_format": { + "description": "range reads native range.current; number requires an entire numeric text; clock requires exactly one HH:MM:SS in readable text and returns seconds since midnight.", + "enum": [ + "range", + "number", + "clock" + ], + "title": "Value Format", + "type": "string" + }, + "target_value": { + "title": "Target Value", + "type": "number" + }, + "tolerance": { + "minimum": 0, + "title": "Tolerance", + "type": "number" + }, + "stable_ms": { + "default": 1000, + "maximum": 5000, + "minimum": 100, + "title": "Stable Ms", + "type": "integer" + }, + "timeout_ms": { + "default": 3000, + "maximum": 10000, + "minimum": 200, + "title": "Timeout Ms", + "type": "integer" + } + }, + "required": [ + "selector", + "value_format", + "target_value", + "tolerance" + ], + "title": "ValueFeedback", + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Read the persistent actual value after the drag. For drag then pause, put feedback on action_sequence instead." + }, "calibration": { "anyOf": [ { @@ -349,6 +515,34 @@ "default": null, "description": "Required for pan: the fixed indicator's observed axis position.", "title": "Indicator Position" + }, + "tolerance": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Allowed value error. Reject a plan whose pixel rounding exceeds it; this is geometric error only, not proof of application precision.", + "title": "Tolerance" + }, + "min_movement_px": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Previously measured minimum effective drag distance in pixels. If this calibrated move is smaller, refuse input and zoom first. Never guess a device-wide threshold.", + "title": "Min Movement Px" } }, "required": [ @@ -392,7 +586,7 @@ "type": "function", "function": { "name": "action_sequence", - "description": "[ACTION] General Android composer: execute ensure_visible, click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Prefer ensure_visible(target=observed panel control ID, opener=observed UI opener ID) followed by click/drag. This checks live UI, skips opening an already-visible panel, or taps its opener ONCE and waits up to timeout_ms (default 1500, max 3000) for the target. Each preparation counts as one member. Include it again in EVERY panel-operation call, even when the previous screenshot showed the panel open; do not split preparation and operation into different model turns. It never retries a business action. Copy all resource IDs VERBATIM, including ':id/' separators, never ':id:'. If preparation fails, re-observe/correct the ID; never guess fallback coordinates. Legacy click.unless_visible is ONLY for an opener, NEVER for an operation such as a +/- button: visible means SKIP, not require-visible. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.", + "description": "[ACTION] General Android composer: execute ensure_visible, click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Prefer ensure_visible(target=observed panel control ID, opener=observed UI opener ID) followed by click/drag. This checks live UI, skips opening an already-visible panel, or taps its opener ONCE and waits up to timeout_ms (default 1500, max 3000) for the target. Each preparation counts as one member. Include it again in EVERY panel-operation call, even when the previous screenshot showed the panel open; do not split preparation and operation into different model turns. It never retries a business action. Copy all resource IDs VERBATIM, including ':id/' separators, never ':id:'. If preparation fails, re-observe/correct the ID; never guess fallback coordinates. Legacy click.unless_visible is ONLY for an opener, NEVER for an operation such as a +/- button: visible means SKIP, not require-visible. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum. For a measurable target, bind feedback to its persistent native readout: final feedback is read AFTER ALL members, including any pause, finish. Never use a release-time tooltip. Use control_action.request.ensure_state for observed checked/selected/text/value state, so an already satisfied state is skipped rather than toggled. A playback icon alone does not prove a native boolean state: inspect it first. Feedback blocks the third consecutive ineffective strategy; zoom or increase the meaningful movement, then re-observe.", "parameters": { "properties": { "steps": { @@ -531,6 +725,24 @@ "title": "Duration Ms", "type": "integer" }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed gesture surface rectangle in coordinate_space, excluding date/navigation rails and unrelated overlays.", + "title": "Region" + }, "calibration": { "anyOf": [ { @@ -596,6 +808,34 @@ "default": null, "description": "Required for pan: the fixed indicator's observed axis position.", "title": "Indicator Position" + }, + "tolerance": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Allowed value error. Reject a plan whose pixel rounding exceeds it; this is geometric error only, not proof of application precision.", + "title": "Tolerance" + }, + "min_movement_px": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Previously measured minimum effective drag distance in pixels. If this calibrated move is smaller, refuse input and zoom first. Never guess a device-wide threshold.", + "title": "Min Movement Px" } }, "required": [ @@ -784,6 +1024,24 @@ "default": null, "title": "Require Visible" }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed zoom surface bounds in coordinate_space; not the whole screen or date rail. Both fingers must remain inside.", + "title": "Region" + }, "action": { "const": "pinch", "title": "Action", @@ -869,6 +1127,24 @@ "default": null, "title": "Require Visible" }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed gesture surface bounds in coordinate_space; every finger path must stay inside it.", + "title": "Region" + }, "action": { "const": "touch_gesture", "title": "Action", @@ -1072,6 +1348,76 @@ ], "default": null, "title": "Selection End" + }, + "ensure_state": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "state": { + "enum": [ + "checked", + "unchecked", + "selected", + "unselected", + "text", + "content_desc", + "value" + ], + "title": "State", + "type": "string" + }, + "expected_text": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Text" + }, + "expected_value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Value" + }, + "tolerance": { + "default": 0, + "minimum": 0, + "title": "Tolerance", + "type": "number" + } + }, + "required": [ + "state" + ], + "title": "ControlCondition", + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired observed state: skip an already satisfied action; otherwise write ONCE and verify by read-only polling. Never infer state from an icon or missing attributes." + }, + "verify_timeout_ms": { + "default": 1500, + "maximum": 3000, + "minimum": 1, + "title": "Verify Timeout Ms", + "type": "integer" } }, "required": [ @@ -1197,6 +1543,7 @@ "selected", "unselected", "text", + "content_desc", "value", "stable" ], @@ -1274,6 +1621,155 @@ "maxItems": 10, "minItems": 1, "type": "array" + }, + "feedback": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "selector": { + "additionalProperties": false, + "description": "Unique, observed persistent value readout, not a transient drag tooltip. Copy its stable selector, excluding changing text.", + "properties": { + "resource_id": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Resource Id" + }, + "text": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Text" + }, + "content_desc": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Content Desc" + }, + "class_name": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Class Name" + }, + "package_name": { + "anyOf": [ + { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Package Name" + }, + "bounds": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Bounds" + } + }, + "title": "ControlSelector", + "type": "object" + }, + "value_format": { + "description": "range reads native range.current; number requires an entire numeric text; clock requires exactly one HH:MM:SS in readable text and returns seconds since midnight.", + "enum": [ + "range", + "number", + "clock" + ], + "title": "Value Format", + "type": "string" + }, + "target_value": { + "title": "Target Value", + "type": "number" + }, + "tolerance": { + "minimum": 0, + "title": "Tolerance", + "type": "number" + }, + "stable_ms": { + "default": 1000, + "maximum": 5000, + "minimum": 100, + "title": "Stable Ms", + "type": "integer" + }, + "timeout_ms": { + "default": 3000, + "maximum": 10000, + "minimum": 200, + "title": "Timeout Ms", + "type": "integer" + } + }, + "required": [ + "selector", + "value_format", + "target_value", + "tolerance" + ], + "title": "ValueFeedback", + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional actual value/target comparison after the WHOLE sequence. clock uses seconds since midnight; exact bound and stable observation are checked locally, not inferred from dispatch." } }, "required": [ @@ -1423,6 +1919,23 @@ } ], "default": null + }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed zoom surface bounds in coordinate_space; not the whole screen or date rail. Both fingers must remain inside." } }, "required": [ @@ -1500,6 +2013,23 @@ } ], "default": null + }, + "region": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 4, + "minItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed gesture surface bounds in coordinate_space; every finger path must stay inside it." } }, "required": [ @@ -1616,7 +2146,7 @@ "type": "function", "function": { "name": "control_action", - "description": "[ACTION] Perform ONE supported native action on a freshly resolved unique control. Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. action=set_progress requires a finite in-range value; set_selection requires selection_start/end in UTF-16 code units (never split an emoji). set_text and replace_selection require text, including empty text; whitespace is preserved. Other actions take no text/value/selection. ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback or replay. Focus or selection changes may move the target; before subsequent edits, use wait_until state=stable when needed. STALE means no action started; observe again before rebuilding the request. Never resend an unknown write. An accepted action is not business verification; inspect the returned/current state.", + "description": "[ACTION] Perform ONE supported native action on a freshly resolved unique control. Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. For a desired state use ensure_state (checked/selected/text/content_desc/value), then verify_timeout_ms: skip an already satisfied state, otherwise write ONCE and poll read-only. Copy observed state semantics; do not assume selected means paused. Bind a stable selector excluding text/content_desc that the action changes. A failed postcondition does not undo the completed input and must not cause an automatic second toggle. action=set_progress requires a finite in-range value; set_selection requires selection_start/end in UTF-16 code units (never split an emoji). set_text and replace_selection require text, including empty text; whitespace is preserved. Other actions take no text/value/selection. ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback or replay. Focus or selection changes may move the target; before subsequent edits, use wait_until state=stable when needed. STALE means no action started; observe again before rebuilding the request. Never resend an unknown write. An accepted action is not business verification; inspect the returned/current state.", "parameters": { "properties": { "selector": { @@ -1784,6 +2314,75 @@ } ], "default": null + }, + "ensure_state": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "state": { + "enum": [ + "checked", + "unchecked", + "selected", + "unselected", + "text", + "content_desc", + "value" + ], + "title": "State", + "type": "string" + }, + "expected_text": { + "anyOf": [ + { + "maxLength": 4096, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Text" + }, + "expected_value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Value" + }, + "tolerance": { + "default": 0, + "minimum": 0, + "title": "Tolerance", + "type": "number" + } + }, + "required": [ + "state" + ], + "title": "ControlCondition", + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Desired observed state: skip an already satisfied action; otherwise write ONCE and verify by read-only polling. Never infer state from an icon or missing attributes." + }, + "verify_timeout_ms": { + "default": 1500, + "maximum": 3000, + "minimum": 1, + "type": "integer" } }, "required": [ @@ -1899,6 +2498,7 @@ "selected", "unselected", "text", + "content_desc", "value", "stable" ], diff --git a/tests/unit/mcp/test_native_executor.py b/tests/unit/mcp/test_native_executor.py index bace45d..c21b50b 100644 --- a/tests/unit/mcp/test_native_executor.py +++ b/tests/unit/mcp/test_native_executor.py @@ -71,6 +71,81 @@ async def run(name, args, native): return await execute_native(name, args, 400, 800, native) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "condition,fields", + [ + ({"state": "unchecked"}, {"checked": False}), + ({"state": "text", "expected_text": " exact "}, {"text": " exact "}), + ({"state": "content_desc", "expected_text": "播放"}, {"content_desc": "播放"}), + ({"state": "value", "expected_value": 3.0}, {}), + ], +) +async def test_ensure_state_already_satisfied_never_toggles(condition, fields): + native = AsyncMock(return_value=observed(**fields)) + result = await run( + "control_action", + {"selector": SELECTOR, "action": "click", "ensure_state": condition}, + native, + ) + assert result["success"] and result["skipped"] + assert result["execution"] == "observed" + assert [call.args[0] for call in native.await_args_list] == ["inspect_control"] + + +@pytest.mark.asyncio +async def test_ensure_state_missing_native_boolean_refuses_input(): + native = AsyncMock(return_value=observed(checked=None)) + result = await run( + "control_action", + {"selector": SELECTOR, "action": "click", "ensure_state": {"state": "unchecked"}}, + native, + ) + assert not result["success"] and result["execution"] == "not_started" + native.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_ensure_state_writes_once_then_reads_actual_state(): + native = AsyncMock(side_effect=[observed(checked=True), completed(), observed(checked=False)]) + result = await run( + "control_action", + {"selector": SELECTOR, "action": "click", "ensure_state": {"state": "unchecked"}}, + native, + ) + assert result["success"] and result["verification"] == "condition_satisfied" + assert [call.args[0] for call in native.await_args_list] == [ + "inspect_control", + "control_action", + "inspect_control", + ] + args = native.await_args_list[1].args[1] + assert "ensure_state" not in args and "verify_timeout_ms" not in args + assert args["expected_token"] == TOKEN + + +@pytest.mark.asyncio +async def test_failed_postcondition_preserves_completed_input_without_retoggling(clock): + async def native(command, args, **kwargs): + return completed() if command == "control_action" else observed(checked=True) + + native = AsyncMock(side_effect=native) + result = await run( + "control_action", + { + "selector": SELECTOR, + "action": "click", + "ensure_state": {"state": "unchecked"}, + "verify_timeout_ms": 400, + }, + native, + ) + assert not result["success"] and result["code"] == "TIMEOUT" + assert result["execution"] == "completed" + assert result["native_receipt"]["success"] + assert sum(c.args[0] == "control_action" for c in native.await_args_list) == 1 + + @pytest.mark.asyncio @pytest.mark.parametrize("name", ["pinch", "touch_gesture"]) async def test_all_gesture_geometry_is_validated_before_a_single_native_dispatch(name): diff --git a/tests/unit/mcp/test_value_feedback.py b/tests/unit/mcp/test_value_feedback.py new file mode 100644 index 0000000..bc2fc2e --- /dev/null +++ b/tests/unit/mcp/test_value_feedback.py @@ -0,0 +1,346 @@ +"""Regress real actuator dispatch, state verification and value convergence without devices.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from artemis.mcp import value_feedback +from artemis.mcp.action_manifest import DEVICE_ACTIONS +from artemis.mcp.actuators import MockActuator +from artemis.mcp.gestures import DragRequest, plan_drag +from artemis.mcp.native_actions import PinchRequest, plan_pinch +from artemis.mcp.value_feedback import read_value + + +@pytest.fixture(autouse=True) +def deterministic_clock(monkeypatch): + now = [0.0] + + async def sleep(seconds): + now[0] += seconds + + monkeypatch.setattr(value_feedback, "monotonic", lambda: now[0]) + monkeypatch.setattr(value_feedback, "sleep", sleep) + monkeypatch.setattr( + "artemis.config.agent.load_agent_config", + lambda: SimpleNamespace( + pro=SimpleNamespace(execution=SimpleNamespace(max_burst_actions=10)) + ), + ) + + +def setup_actuator(*, current="100", after="100", readable=True): + actuator = MockActuator(width=720, height=1600, capabilities=DEVICE_ACTIONS) + state = {"text": current, "playing": False} + events = [] + + async def native(command, args, **kwargs): + events.append(command) + if command == "control_action": + state["playing"] = not state["playing"] + return {"success": True, "execution": "completed", "code": "OK"} + is_player = args["selector"]["resource_id"] == "player" + return { + "success": True, + "execution": "observed", + "code": "OK", + "node": { + "token": "a" * 64, + "content_signature": "b" * 64, + "visible": True, + "enabled": True, + "password": False, + "actions": ["click"], + "text": state["text"] if readable else None, + "content_desc": ("暂停" if state["playing"] else "播放") + if is_player + else "readout", + }, + } + + async def swipe(*args): + events.append("swipe") + state.update(text=after, playing=True) + return None + + actuator.controller.native_request = AsyncMock(side_effect=native) + actuator.controller.swipe_coords = AsyncMock(side_effect=swipe) + actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None)) + return actuator, state, events + + +def feedback(**overrides): + return { + "selector": {"resource_id": "readout"}, + "value_format": "number", + "target_value": 90.0, + "tolerance": 1.0, + "stable_ms": 100, + "timeout_ms": 500, + **overrides, + } + + +def drag(**overrides): + return { + "start": [300, 1050], + "end": [325, 1050], + "coordinate_space": "pixel", + "target_description": "observed timeline", + **overrides, + } + + +@pytest.mark.asyncio +async def test_two_ineffective_attempts_block_third_but_allow_meaningful_strategy_change(): + actuator, _, _ = setup_actuator() + for _ in range(2): + result = await actuator.drag(**drag(feedback=feedback())) + assert not result.ok and result.metadata["execution"] == "completed" + assert result.metadata["value_feedback"]["after_value"] == 100 + result = await actuator.drag(**drag(feedback=feedback())) + assert not result.ok and "NO_PROGRESS" in result.message + assert result.metadata["execution"] == "not_started" + assert actuator.controller.swipe_coords.await_count == 2 + # Tiny endpoint jitter is the same strategy, not an escape from the guard. + await actuator.drag(**drag(end=[328, 1050], feedback=feedback())) + assert actuator.controller.swipe_coords.await_count == 2 + await actuator.drag(**drag(end=[424, 1050], feedback=feedback())) + assert actuator.controller.swipe_coords.await_count == 3 + # History belongs to this actuator/task, not a global cache. + other, _, _ = setup_actuator() + await other.drag(**drag(feedback=feedback())) + assert other.controller.swipe_coords.await_count == 1 + + +@pytest.mark.asyncio +async def test_changed_actual_value_resets_stalled_history(): + actuator, state, _ = setup_actuator() + for _ in range(2): + await actuator.drag(**drag(feedback=feedback())) + state["text"] = "95" + await actuator.drag(**drag(feedback=feedback())) + assert actuator.controller.swipe_coords.await_count == 3 + + +@pytest.mark.asyncio +async def test_one_pixel_change_across_power_of_two_is_still_the_same_strategy(): + actuator, _, _ = setup_actuator() + for endpoint in (331, 332, 333): + await actuator.drag(**drag(end=[endpoint, 1050], feedback=feedback())) + assert actuator.controller.swipe_coords.await_count == 2 + # A substantially slower gesture is a meaningful new approach. + await actuator.drag(**drag(end=[333, 1050], duration_ms=1500, feedback=feedback())) + assert actuator.controller.swipe_coords.await_count == 3 + + +@pytest.mark.asyncio +async def test_sequence_reads_final_value_only_after_pause_and_skips_redundant_toggle(): + actuator, state, events = setup_actuator( + current="11-02 周一 17:43:12", after="11-02 周一 17:42:17" + ) + pause = { + "action": "control_action", + "request": { + "action": "click", + "selector": {"resource_id": "player"}, + "ensure_state": {"state": "content_desc", "expected_text": "播放"}, + }, + } + result = await actuator.action_sequence( + [pause, {"action": "drag", **drag()}, pause], + feedback=feedback( + value_format="clock", target_value=17.0 * 3600 + 42 * 60 + 17, tolerance=5.0 + ), + ) + assert result.ok and not state["playing"] + assert result.metadata["skipped_steps"] == 1 + assert result.metadata["value_feedback"]["residual"] == 0 + observation = result.metadata["value_feedback"]["after_observation"] + assert observation["node"]["text"] == "11-02 周一 17:42:17" + assert observation["stable_for_ms"] >= 100 and observation["samples"] >= 2 + assert result.metadata["verification"] == "value_matched" + assert events.count("control_action") == 1 + assert events.index("swipe") < events.index("control_action") + assert events[-1] == "inspect_control" + + +@pytest.mark.asyncio +async def test_missing_value_refuses_input_and_postwrite_read_failure_preserves_receipt(): + actuator, _, _ = setup_actuator(readable=False) + result = await actuator.drag(**drag(feedback=feedback())) + assert not result.ok and result.metadata["execution"] == "not_started" + actuator.controller.swipe_coords.assert_not_awaited() + actuator, _, _ = setup_actuator(after="unreadable") + result = await actuator.drag(**drag(feedback=feedback())) + assert not result.ok and result.metadata["execution"] == "completed" + assert result.metadata["coordinates"] == [300, 1050, 325, 1050] + assert result.metadata["verification"] == "unavailable" + actuator.controller.swipe_coords.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_native_feedback_unavailable_has_no_coordinate_fallback(): + actuator = MockActuator(capabilities=DEVICE_ACTIONS - {"read_control", "wait_until"}) + actuator.controller.swipe_coords = AsyncMock() + result = await actuator.drag(**drag(feedback=feedback())) + assert not result.ok and result.code == "UNSUPPORTED" + actuator.controller.swipe_coords.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_whole_sequence_rejects_bad_region_before_any_click(): + actuator, _, _ = setup_actuator() + result = await actuator.action_sequence( + [ + {"action": "click", "target": [50, 50], "target_description": "opener"}, + {"action": "drag", **drag(region=[400, 970, 650, 1118])}, + ] + ) + assert not result.ok + actuator.controller.tap_at.assert_not_awaited() + actuator.controller.swipe_coords.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_failed_state_verification_keeps_completed_members_and_stops_next_input(): + actuator, _, _ = setup_actuator() + + async def unchanged(command, args, **kwargs): + if command == "control_action": + return {"success": True, "execution": "completed", "code": "OK"} + return { + "success": True, + "execution": "observed", + "code": "OK", + "node": { + "token": "a" * 64, + "visible": True, + "enabled": True, + "text": "100", + "content_desc": "暂停", + "actions": ["click"], + }, + } + + actuator.controller.native_request = AsyncMock(side_effect=unchanged) + result = await actuator.action_sequence( + [ + {"action": "drag", **drag()}, + { + "action": "control_action", + "request": { + "action": "click", + "selector": {"resource_id": "player"}, + "ensure_state": {"state": "content_desc", "expected_text": "播放"}, + "verify_timeout_ms": 1, + }, + }, + {"action": "click", "target": [50, 50], "target_description": "must not execute"}, + ], + feedback=feedback(), + ) + assert not result.ok and result.metadata["execution"] == "partial" + assert [r["execution"] for r in result.metadata["sequence_results"]] == [ + "completed", + "completed", + "not_started", + ] + assert result.metadata["sequence_results"][1]["verification"] == "condition_not_met" + actuator.controller.tap_at.assert_not_awaited() + actuator.controller.swipe_coords.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_transient_post_action_read_failure_can_settle_without_replaying_input(): + actuator, _, events = setup_actuator(after="90") + original = actuator.controller.native_request.side_effect + failed = False + + async def transient(command, args, **kwargs): + nonlocal failed + if "swipe" in events and not failed: + failed = True + return {"success": False, "execution": "not_started", "code": "READ_FAILED"} + return await original(command, args, **kwargs) + + actuator.controller.native_request = AsyncMock(side_effect=transient) + result = await actuator.drag(**drag(feedback=feedback())) + assert result.ok and result.metadata["value_feedback"]["after_value"] == 90 + actuator.controller.swipe_coords.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_feedback_timeout_never_repeats_a_completed_input(monkeypatch): + actuator, state, _ = setup_actuator() + original = value_feedback.sleep + + async def moving(seconds): + state["text"] = str(float(state["text"]) + 1) + await original(seconds) + + monkeypatch.setattr(value_feedback, "sleep", moving) + result = await actuator.drag(**drag(feedback=feedback())) + assert not result.ok and result.code == "TIMEOUT" + assert result.metadata["execution"] == "completed" + assert result.metadata["verification"] == "unavailable" + actuator.controller.swipe_coords.assert_awaited_once() + + +@pytest.mark.parametrize( + "text", ["17:42:17 / 17:43:12", "25:42:17", "117:42:17", "17:42:170", "17:42"] +) +def test_clock_readout_rejects_ambiguous_or_invalid_times(text): + with pytest.raises(ValueError): + read_value({"visible": True, "text": text}, "clock") + + +@pytest.mark.parametrize( + "value_format,node", + [ + ("number", {"text": "0"}), + ("clock", {"text": "00:00:00"}), + ("range", {"range": {"min": 0, "current": 0, "max": 10}}), + ], +) +def test_zero_is_an_actual_value(value_format, node): + assert read_value({"visible": True, **node}, value_format) == 0 + + +def test_precision_guard_uses_observed_minimum_and_reports_geometric_budget(): + calibration = { + "axis": "x", + "mode": "thumb", + "anchors": [{"position": 100.0, "value": 0.0}, {"position": 200.0, "value": 1000.0}], + "target_value": 125.0, + "tolerance": 1.0, + } + request = { + "start": [100, 1050], + "coordinate_space": "pixel", + "target_description": "observed scale", + "calibration": calibration, + } + with pytest.raises(ValueError, match="rounding"): + plan_drag(DragRequest.model_validate(request), 720, 1600) + calibration.update(tolerance=5.0, min_movement_px=20.0) + with pytest.raises(ValueError, match="minimum effective"): + plan_drag(DragRequest.model_validate(request), 720, 1600) + calibration.update(min_movement_px=10.0) + plan = plan_drag(DragRequest.model_validate(request), 720, 1600) + assert plan["calibration"]["pixel_error_budget"] == 0.5 + assert plan["calibration"]["rounding_error_value"] == 5 + + +def test_both_pinch_fingers_must_stay_in_observed_surface(): + request = PinchRequest( + center=[360, 656], + start_span=40, + end_span=80, + coordinate_space="pixel", + target_description="observed timeline", + region=[200, 970, 650, 1118], + ) + with pytest.raises(ValueError, match="region"): + plan_pinch(request, 720, 1600)