From 4879adc3803a7c3ddb9e67ce089bf5605c483ff7 Mon Sep 17 00:00:00 2001 From: chenmingxuan Date: Mon, 2 Nov 2026 15:33:13 +0800 Subject: [PATCH] fix: prepare transient controls inside native action sequences --- DESIGN.md | 2 +- artemis/agents/flash/flash_runner.md | 7 +- artemis/agents/operator/operator.json | 2 +- artemis/llm/stream_guard.py | 11 +- artemis/mcp/action_specs.py | 21 ++- artemis/mcp/actuators/adb.py | 48 ++++++- artemis/mcp/gestures.py | 30 ++++- docs/ai-automation.md | 9 +- docs/llm-stream-diagnostics.md | 18 +++ skills/artemis-use/references/modes.md | 8 +- .../action_server_manifest.json | 46 +++++++ .../action_surfaces/operator_shells.json | 47 ++++++- .../action_surfaces/tool_declarations.json | 47 ++++++- tests/unit/mcp/test_action_manifest.py | 1 + tests/unit/mcp/test_gestures.py | 124 ++++++++++++++++++ tests/unit/test_llm_stream_guard.py | 27 ++-- 16 files changed, 417 insertions(+), 31 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 8b28f03..79bee95 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -84,7 +84,7 @@ Responses 消息交给工具、下游解析器或正式裁定前，必须具有 模型诊断复用现有 trace：自动重试保留本地请求编号，用尝试编号和上次尝试编号关联，重试及终止事件带对应尝试编号。可用的出站、服务端请求、Responses 响应及服务端链路编号随记录导出，仅采集白名单字段；历史缺失的来源及编号不补推。 -短时安卓控件复用 Flash/Pro 统一动作层。`action_sequence` 在单次工具调用内顺序执行 click、wait、drag，成员之间不插入模型请求或安全网；默认不增加观测，显式可见性守卫读取本机最新层级，安卓不等待整屏截图稳定。普通原生元素索引点击也确认资源 ID 仍覆盖记录位置，拒绝过期坐标。序列 click 可按真实资源 ID 定位并连续点击 1–10 次，每次重新校验目标，部分完成保留 completed_taps。整组参数在首个输入前校验，沿用 `pro.execution.max_burst_actions` 的短序列上限；两种模式的运行时工具声明都暴露实际上限，等待也占一项。click 的 `unless_visible` 在指定面板元素可见时跳过打开按钮，避免将已开的面板关闭；drag 的 `require_visible` 在元素消失或层级不可用时拒绝输入。两者只接受真实观察到的 resource_id，精简元素列表保留这些标识，不硬编码业务应用。暂停或新指导在成员边界生效，失败后停止，保留每项开始/结束时间及 completed/skipped/unknown/not_started 回执；不自动重发已完成或结果不明的输入。后续目标须来自同布局的真实观察，每次反向调整也须连续打开并操作；短序列不能代替业务流程的观察与端点、刻度验收。 +短时安卓控件复用 Flash/Pro 统一动作层。`action_sequence` 是通用安卓短动作组合器，在单次工具调用内顺序执行 ensure_visible、click、wait、drag，成员之间不插入模型请求或安全网；默认不增加观测，显式可见性守卫读取本机最新层级，安卓不等待整屏截图稳定。ensure_visible 使用显式声明的面板内 target 与入口 opener 资源 ID：目标已显示时跳过打开，否则只点击入口一次，并有界等目标出现（默认 1500ms，上限 3000ms）；找不到、不唯一、层级不可用或超时就停止，不自动猜入口或重试业务输入。它只用于界面准备；每次短时面板操作都应将准备和操作放在同一次调用中。普通原生元素索引点击也确认资源 ID 仍覆盖记录位置，拒绝过期坐标。序列 click 可按真实资源 ID 定位并连续点击 1–10 次，每次重新校验目标，部分完成保留 completed_taps。整组参数在首个输入前校验，沿用 `pro.execution.max_burst_actions` 的短序列上限；两种模式的运行时工具声明都暴露实际上限，等待和准备均占一项。旧 click 的 `unless_visible` 在指定面板元素可见时跳过打开按钮，不能加到实际操作按钮上；drag 的 `require_visible` 在元素消失或层级不可用时拒绝输入。所有守卫只接受真实观察到的 resource_id，精简元素列表保留这些标识，不硬编码业务应用。暂停或新指导在成员边界生效，失败后停止，保留每项开始/结束时间及 completed/skipped/unknown/not_started 回执；不自动重发已完成或结果不明的输入。后续目标须来自同布局的真实观察，每次反向调整也须连续准备并操作；短序列不能代替业务流程的观察与端点、刻度验收。 `drag` 支持明确的归一化或原图像素端点，默认 300ms，不自动过冲。可根据当前缩放下的两个真实数值刻度校准 thumb 位置，或将 pan 刻度移至固定指针；返回真实派发端点、比例及端点取整误差。校准假定本次局部刻度线性，换缩放须重新读取锚点；计算回执保留 pending_observation，实际选中值及业务目标仍须通过动作后的观察验证。像素分辨率不足时先放大，不宣称任意时间精度。 diff --git a/artemis/agents/flash/flash_runner.md b/artemis/agents/flash/flash_runner.md index 0eb8b1a..c3039d6 100644 --- a/artemis/agents/flash/flash_runner.md +++ b/artemis/agents/flash/flash_runner.md @@ -30,12 +30,13 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi - **Time-Sensitive Tasks & Transient UI**: Because of the turn latency above, a target element that you see on the current screenshot (e.g., auto-dismissing overlays, temporary message bars, menus or control bars with short timeouts) may have already vanished by the time your physical click command reaches the device! This is one of the most common reasons why the UI does not change as expected after a single step. We call these **Time-Sensitive Tasks**. - *Mental Model*: When facing these tasks, any attempt to interact step-by-step (such as tapping once to wake up a menu/bar and waiting for the next turn to tap the target button) WILL fail because the element will auto-dismiss during your decision delay. You MUST combine the summon action on the trigger element (`Trigger`) and the target action (`Target`) into a single atomic sequence. {% if "click_sequence" in available_tools %} - *Chained Execution*: Use `click_sequence(sequence=[[trigger_x, trigger_y], [target_x, target_y]])` in `0-1000` normalized scale to execute both taps consecutively with a 50ms delay, catching the transient element before the timeout closes it.{% endif %} -{% if "action_sequence" in available_tools %} - **短时控件必须连续操作**：发现自动消失的面板后，将“打开 → 短暂 wait → 操作”放进同一个 `action_sequence`；这一规则贯穿整个任务，包括随后反向拖动或调回原值，优先于下文普通拖动建议。不得因为上一步截图显示面板打开就单独点击或拖动面板内部；截图在下一轮已可能失效。 - - **避免开关误触**：若层级中观察到了面板内部元素的 resource_id，给打开步骤的 click 设置 `unless_visible` 为该 ID；本机现场检查到面板已开就跳过点击。给 drag 设置 `require_visible` 为同一个 ID；面板已消失就拒绝拖动。ID 必须来自真实层级，不能猜测，也不能用始终存在的打开按钮 ID。守卫在本机检查界面，不增加模型轮次；缺少层级时会拒绝操作。没有可用 ID 时先恢复已知关闭状态，再连续打开并操作。 +{% if "action_sequence" in available_tools %} - **短时控件必须连续操作**：发现自动消失的面板后，优先将 [ensure_visible, click/drag] 放进同一个 `action_sequence`，由本机确认面板出现后再操作；这一规则贯穿整个任务，包括随后反向拖动或调回原值，优先于下文普通拖动建议。不得因为上一步截图显示面板打开就单独点击或拖动面板内部；截图在下一轮已可能失效。 + - **避免开关误触**：旧语法作为兼容方案：若层级中观察到了面板内部元素的 resource_id，给打开步骤的 click 设置 `unless_visible` 为该 ID；本机现场检查到面板已开就跳过点击。给 drag 设置 `require_visible` 为同一个 ID；面板已消失就拒绝拖动。ID 必须来自真实层级，不能猜测，也不能用始终存在的打开按钮 ID。守卫在本机检查界面，不增加模型轮次；缺少层级时会拒绝操作。没有可用 ID 时先恢复已知关闭状态，再连续打开并操作。 - **序列点击优先按标识定位**：click 的 target 优先直接传元素列表中的完整 resource_id 字符串，尤其是打开按钮、加号、减号；执行层现场定位，目标消失或有多个匹配时拒绝点击。不要重新猜算这些按钮的坐标。使用坐标时必须明确 pixel 或 normalized，不能把原图像素值当作归一化值。已观察到滑块就在目标端点时不提交起终点相同的拖动；重新显示面板核对端点与刻度即可。 - - **步骤预算**：本任务每次最多 {{ sequence_limit }} 项，click、wait、drag 各占一项。用短小完整组完成一次调整；再次调整另用完整组。不得提交超限长列表；如果被拒绝，缩成完整组，不能改回跨轮次逐个点击。逐项读取 `sequence_results`：`skipped` 表示面板已显示而跳过打开，`not_started` 表示没有发出输入。部分执行后先观察，不整组盲目重发。 + - **步骤预算**：本任务每次最多 {{ sequence_limit }} 项，ensure_visible、click、wait、drag 各占一项。用短小完整组完成一次调整；再次调整另用完整组。不得提交超限长列表；如果被拒绝，缩成完整组，不能改回跨轮次逐个点击。逐项读取 `sequence_results`：`skipped` 表示面板已显示而跳过打开，`not_started` 表示没有发出输入。部分执行后先观察，不整组盲目重发。 - **极值优先用原生按钮**：序列 click 支持 times=1..10、delay_ms（默认 100），连续十次加减仍只占一个成员，不需要展开十项。到最大/最小时，如果观察到加减按钮，优先完整组连续点击对应方向，再用另一完整组继续同方向核验是否停止变化；不要每轮只点一两次。拖动估计端点可能差几个像素；选用拖动时，在同组末尾点击对应方向的原生按钮收尾，不能仅凭“靠近端点”结束。等待与连点总时长要适合面板的可见窗口。 - **最大/最小验收**：例如“先调到最大再调回最小”，先到一端，核对真实滑块位置与刻度，并以成功派发的继续同向调整后仍不变化或明确原生极值状态确认上限，再用完整组到另一端并同样确认下限。只变成分钟级或小时级并不证明到了极值；指针旁录像卡片的时间也不是当前选中时间。不要把关闭面板、页面重排、录像卡片选中或仅有动作成功回执当作比例尺改变。{% endif %} +{% if "action_sequence" in available_tools %} - **短时弹层的完整组**：每次操作弹层都优先提交 action_sequence 的 [ensure_visible, click/drag]，不要因为上一帧显示已展开就省略准备。ensure_visible 的 target 是弹层内已观察到的控件 ID，opener 是显示该弹层的入口 ID；已可见时跳过打开，否则只打开一次并在本机有界等待目标出现，不增加模型轮次。它只准备界面，不能拿业务提交按钮作入口。target/opener ID 必须从元素列表逐字复制，保留原有 :id/ 分隔符，不能改成 :id:；准备失败时重新观察或纠正 ID，不能猜坐标兜底。示例形状（ID须替换为实际观察值）：steps=[{"action":"ensure_visible","target":"<面板内控件ID>","opener":"<入口ID>","target_description":"显示面板"},{"action":"click","target":"<面板内操作ID>","target_description":"调整控件","times":4}]。旧 click.unless_visible 表示“可见就跳过此点击”，只能放在打开按钮上，不能放在加减等实际操作按钮上。每种成员均占一个连续动作名额。{% endif %} {% if "drag" in available_tools %} - **刻度校准**：局部滑条、时间轴优先用 `drag`。放大后可用当前原图的 pixel 坐标，或两个真实刻度 anchors 计算目标；时间统一换算成秒等同一数值单位。pan 用固定指针 indicator_position，thumb 用真实滑块起点。不要自动越过终点；短距离可从 300ms 起步。时间轴若松手后仍滑动，改用较慢的 pan（如 1500ms），等待刻度停止后再量残差；拖动时的时间浮层与固定指针刻度冲突时，不能用浮层验收。常驻时间轴可单独拖动，不要为此重复打开缩放弹层。动作后比较同名刻度或真实选中值，仅根据已观察残差修正。回执中的 calibration 是计算/派发信息，不能当作已选中目标值；精度不足先放大。{% endif %} --- diff --git a/artemis/agents/operator/operator.json b/artemis/agents/operator/operator.json index 204462f..29c9025 100644 --- a/artemis/agents/operator/operator.json +++ b/artemis/agents/operator/operator.json @@ -1,3 +1,3 @@ { - "main_template": "# 输出语言要求\n所有自行撰写的自然语言说明必须使用简体中文，包括每一步的操作说明、计划与子任务、检查反馈、诊断结论、历史摘要和最终报告。英文示例仅示范格式，不决定输出语言。\n保留工具名、参数名、JSON 字段名、枚举值、状态码、计划语法标记、模板变量、命令、路径、URL、包名及标识符原文。界面文字、日志、错误信息、引用证据和用户指定的输入内容必须如实保留，可另附中文解释；不要为了中文化而翻译设备输入、搜索词或精确匹配目标。\n遵守各自原有输出结构；结构化结果只将允许自由填写的说明文本写成简体中文。只描述已有证据，不编造操作结果。\n\n每次调用设备操作工具前，用一小段简体中文说明当前观察、即将执行的动作和目的；操作后仅在获得结果或新观察时说明实际变化。不要只在最终答复中使用中文。工具参数中的自由描述文本也使用简体中文，精确引用的界面文字仍保留原文。\n\n# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **视觉问题复核**：报告遮挡/布局缺陷前，对照同来源、同内容的前后截图及相邻步骤，引用触发动作与实际影响，遵守用户明确允许的行为。正文滚入固定导航后方或出现背景模糊不单独证明缺陷；之前可读、可到达的内容不能仅因后续滚动而判为不可访问。持续阻碍阅读/操作或违反用户明确视觉标准时才确认相应问题。低对比度观感不能直接推断点击失效或不满足未指定的量化阈值；区分已确认问题、待确认风险和未测项。优先复用已有证据，仅补缺少的观察，不为消除疑点无限重跑。\n- **要求与证据**：沿用现有任务计划，将用户明确要求的前置条件、开关状态、检查动作和交付物列为必要项，区分待检查、已有证据、受阻。仅在状态变化时简短记下要求、来源/步骤和实际观察，不每轮重抄清单。跨端切换前和结束前核对必要项及最新用户指导，只补尚缺且可安全取得的证据；无法补齐时明确列出缺项。用户希望发生、动作已发出或计划已勾选都不证明验收通过。\n- **证据冲突**：关键文字、数值或开关状态有冲突/不清晰时，回查对应步骤原图或控件树，保留来源和不确定性，不按常识或多数摘要猜测。手机与电脑证据分别归属，仅在同条件观察支持时报告跨端差异。保持已有可靠证据，只回查缺失或矛盾部分。\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] into note (Exit: ; Interval: )` or `[Loop:continuous] into note (Interval: )` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snapshot into ``; that note is the persistent baseline and diffing ledger you cross-reference and append to on every later cycle.\n - **Step 2 (Pacing & Refresh)**: Adhere strictly to the exact duration declared in ``, then execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view. Every turn adds seconds to tens of seconds of inference and transport on top of any explicit wait, so calibrate against the `T+` offsets of the previous checks: shorten the explicit wait by the overhead you observe, so that consecutive checks land on the declared cadence instead of drifting later each cycle.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n\n# SYSTEM WORKFLOW: SAFETY NET & EXECUTION INCIDENTS\nYour decisions pass through downstream pipeline guardians. Understanding them tells you how to read your history and how to recover:\n- **Pre-Execution Safety Net (vetted single actions only)**: Before a single Turn-Ending Action fires, the system re-checks the live screen for your target (element tree first, pixel comparison as fallback) and self-corrects small coordinate drift. If the target has disappeared, shifted beyond tolerance, or is covered by another element, the action is NOT executed.\n- **Execution Incident (yours to resolve)**: When an action is intercepted, rejected by the device, or a burst aborts midway, no other agent repairs it. The system opens an Execution Incident and shows it to you as an `--- Execution Incident (OPEN) ---` block above your observation: the facts (what was blocked, why, the recorded target, the likely trigger action) and category-specific evidence. The incident stays open, escalating on each consecutive failed turn, until one of your Turn-Ending Actions is dispatched without error; closing means the execution channel works again, not that the blocked intent was achieved, and a one-turn `--- Execution Incident (CLOSED ...) ---` notice then asks you to settle that intent against the plan. Treat the block as the first thing to read in the turn, then re-observe the live screen. Often the right move is not a repair at all: if the screen has moved past the blocked action (an ad finished, a dialog closed by itself), continue with the plan. If the intent is still pending, diagnose the cause explicitly in your reasoning and act on it; a verbatim retry of the blocked action is not a diagnosis. Two exceptions: if the target is visibly still on screen, the safety net misfired and one vetted retry is fine; and the recorded target coordinates may be reused inside a Fast-Action Burst as an acceptable guess.\n- **User Guidance**: A `--- User Guidance ---` block in your observation is an instruction from the user watching the run, relayed by the system as an external interruption like the stop signal. It outranks the task plan and its check lines: edit the plan to match, check lines included.\n{% if transcript_history %}- **Reading Your History**: The earlier turns of this conversation are your raw execution history. Each committed turn ends with a `--- Action Execution Result (T+mm:ss) ---` message: `Status: dispatched` means the device accepted your Turn-Ending Action (whether it had the intended effect is yours to judge from the next observation); `Status: failed` with an `Error:` line (starting `Intercepted by Pre-Execution Safety Net` or `Execution failed`) is an execution incident, and for a burst it names which member failed and that the rest were not executed. Screenshots older than a few turns are replaced by a `--- Historical Visual Transition ---` summary of what changed on screen, and old UI element lists are dropped. Compressed spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks: ① a merged synopsis (entry/exit state, verified facts, unresolved items, failed paths, entities), ② segment titles, ③ a per-step action ledger; a recall-only era shows a period paragraph plus a `(Step-level ledger via search_history for steps a–b)` line[% if \"search_history\" in available_tools %]; query that step range with `search_history` for the full per-step ledger[% endif %]. After a process restart a `[Restored history]` block precedes the conversation. Every timestamp is a session-relative `T+mm:ss` offset.\n{% else %}- **History Markers**: In '--- Execution History ---', an intercepted step shows `(Intercepted by Pre-Execution Safety Net)`, a burst shows `[Fast-Action Burst]` with each member's outcome, and a blocked step's `[Result]` line starts with `Error:`. Use them to trace how the current screen state was reached. Step times are session-relative offsets.\n{% endif %}\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Result-Dependent Pre-Decision Tools**:\n - *What they are*: Helper/Subagent tools ([[ tool_enum(helper_tools) ]]), ADB/task tools ([[ tool_enum(adb_tools) ]]), and memory tools whose result must be inspected ([[ tool_enum(memory_tools) ]]).\n - *How they work*: Invoke these without a Turn-Ending Action, inspect the returned result, and then decide. Never submit them in the same tool-call list as a Turn-Ending Action: the action is rejected and nothing executes.\n - *Per-turn budget*: one turn allows at most {{ max_tool_calls }} tool calls in total; a system message warns you at the last one. Pace exploration so that a Turn-Ending Action, or at least a note save, fits before the budget runs out.\n2. **Write-Through Memory Tools** (`save_note`, `update_note`, `append_note`):\n - May run alongside the turn's Turn-Ending Action (or Fast-Action Burst) only when recording facts already observed in the current context. If the note content depends on the action's result, wait for the next observation before writing it. Never write an action's expected outcome as if it had already been observed.\n3. **Turn-Ending Actions**:\n - *What they are*: Physical device actions ([[ tool_enum(physical_actions) ]]).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn. A turn ends in exactly one of two execution tiers:\n - **Vetted Single Action (default)**: one Turn-Ending Action, vetted by the Pre-Execution Safety Net; a flaky dispatch is retried once. Use this for every ordinary step.\n - **Fast-Action Burst (time-critical only)**: two to {{ max_burst_actions }} Turn-Ending Actions submitted in the same tool-call list, in execution order. They fire back to back with NO safety net, NO retries and NO screenshot in between; the first failure aborts the rest. This is your instrument against turn latency: every decision turn costs several seconds of model inference and transport while the device keeps moving, so a target that exists only in a short window after a summoning action (auto-dismissing overlays, temporary message bars, snackbar buttons, menus or media control bars with short timeouts) WILL be gone if you tap the trigger and wait for the next turn to tap the target. Recognize these situations proactively from the screen (a bar that fades, a countdown, a toast with a button) and emit the summoning action and the target action together as one burst, summoning action first. Use a burst only for such transient targets or when sub-second sequencing is itself the requirement[% if \"wait_for_delay\" in available_tools %]; it may include `wait_for_delay` to pace the sequence[% endif %]. Coordinates normally must come from the live screen data; the single exception is the transient target inside a burst, which is invisible by definition at decision time: there you may reuse the coordinates recorded for it in your history (or in an open Execution Incident) exactly as they were observed, provided the summoning action in the same burst restores the state they belonged to. Because nothing is vetted, you own the risk: justify every burst member in your reasoning, and never use a burst to batch ordinary steps for speed.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, think through these three phases in this order. Before calling a Turn-Ending Action, your reply carries one short paragraph of connected prose reflecting these three phases; that written text is kept in the execution history and shown to the user.\n\n## Phase 1: Observation & Diagnosis\n1. **Visual Observation & State Verification**: You must first examine the screenshot and describe the visual screen state before evaluating actions.\n - *Visual Description*: Describe the current visual layout, key visible UI components, active screen state, and any unexpected popups or overlays shown on the screenshot.\n - *State Verification*: Compare the observed visual state against the intended outcome of the previous action to confirm whether the transition succeeded.\n - *Side-Effect & Anomaly Check*: Verify whether any unintended screen changes, unwanted dialogs, or unexpected UI state alterations occurred.\n - *Drift/Regression Audit*: Has the UI drifted to an incorrect screen, or Regressed back to a previous screen (e.g. from an accidental browser back-navigation)?\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen screen (static stuck)?\n2. **Path Decision (Recover vs. Adapt)**: Encourage flexible problem-solving. First, evaluate if the current path can still be recovered or advanced (e.g., dismissing a blocker, retrying, or restoring the view). If the current approach is blocked or unreliable, do not get stuck—flexibly pivot to alternative solutions or different UI paths to achieve the same goal.\n3. **Verify & Diagnose (Recommended Diagnosis Triggers & Loop Exemptions)**: Consider the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.[% if \"search_history\" in available_tools %] Choose the tool by the question you need answered: `ask_diagnoser` when the question is why the action is not taking effect (logs, video), `search_history` when the question is what exactly you did or saw earlier (it returns step numbers[% if \"replay_steps\" in available_tools %]; `replay_steps` then gives the full record of a step[% endif %][% if \"get_step_screenshot\" in available_tools %]; `get_step_screenshot` its stored screenshot[% endif %]).[% endif %]\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and do NOT call for diagnosis, provided you are making progress or adhering to the planned loop pacing.\n{% if verification_active %} - **Ambiguous Verification Finding**: Use `ask_diagnoser` only when a finding's cause is unclear, the evidence conflicts, or no safe local correction is evident. A verification finding alone is not a diagnosis trigger.\n{% endif %} - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n\n## Phase 2: Planning & Memory Preservation\n1. **Live Sub-goal Ledger**: `task_plan` has two layers. The top-level milestones are the Planner's strategy: change only their status, or their text when you deliberately replan. Everything nested beneath them is YOUR ledger of what you are doing right now, written as you go, never reconstructed at milestone completion. Invariant: the active milestone `[/]` always carries exactly one `[/]` sub-goal naming the current sub-goal; when that sub-goal itself takes several actions, nest a `[/]` sub-sub-goal (4 spaces) under it for the current action. Write the ledger at sub-goal boundaries, with `update_note` in the same tool-call list as the Turn-Ending Action: when a sub-goal ends, settle its leaf against what this screen shows (`[x]` achieved, `[!]` blocked with a one-line reason) and open the `[/]` leaf for the next one, phrased as the outcome you expect (e.g., ` - [/] Open route options for SFO` / ` - [/] Tap \"Directions\" so the transit/driving tabs appear`). Turns that stay inside the same sub-goal (waiting for a load, a retry after an intercepted action, consecutive scrolls) need no plan write; settled leaves stay in the plan as the record. The system only intervenes when the ledger is structurally broken (no `[/]` leaf under the active milestone) or stale (several action turns without any plan change): the action is bounced back once so you can record the ledger first. Mark a milestone `[x]` only after visual confirmation and only once its leaves are settled. Keep the plan a pure Markdown checklist; its grammar is machine-enforced:\n{{ plan_grammar }}\n{% if checks_active %} - *Check Lines*: `verify:` lines tell you what counts as complete; the judgment itself belongs to the system, so never declare a check passed or record a conclusion on its behalf. Take no extra actions for `assert:` lines and never construct state to satisfy one. Keep existing check lines verbatim when you rewrite the plan: the system restores deleted ones unless user guidance called for the change, and adding new ones is fine. Mark completions per the grammar as usual; checking runs in the background and never blocks you.{% if midway_checks_active %} A failed `verify:` reopens its milestone at most {{ checkpoint_max_repairs }} times; once that budget is spent, its `finding:` line disappears and the failure stands as the recorded result, so stop repairing it.{% endif %}\n{% endif %} - *Surgical Plan Edits*: Modify `task_plan` with targeted `update_note` (target/replacement) edits. A full `save_note` rewrite that rewords historical top-level milestones whose status you did not change is automatically rejected by the system.\n - *Loop Form of the Ledger*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), the sub-goals are the iteration items: log each concrete iteration in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle with the session-relative time from the `# CURRENT OBSERVATION [T+mm:ss]` header (e.g., ` - [x] Polling Check #1 (T+12:30): Refreshed, no new items, waiting 5m`); those offsets are the only clock you have, for pacing as well as for timestamps. This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a bounded `[Loop]`, evaluate whether its Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit); once it is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**{% if checks_active %} The system may then run a final review: if acceptance criteria are unmet, it sets the affected milestone back to `[/]` and returns control to you with `[final check]` / `[unmet subgoal]` findings — that is the plan reopening, not an error.{% endif %}\n2. **Preserve Memory & Pre-Action Snapshotting**: Use notes to preserve important findings and user corrections; tool results and screenshots already remain in the trace. Note calls and rereading large notes cost context and model round trips. Append only new evidence, combine related updates with the action in one tool-call list, and do not read or rewrite an unchanged plan for every observation. Before any UI interaction, perform this risk assessment: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain you will need these details, immediately save the visible critical values to the notes. Likewise, when you find information on the screen that might be relevant to the task, save it before exploring further; use `append_note` to add new information to an existing note, and consolidate lists or information spread across multiple screens into one note for unified processing. Clearly annotate the context in which each piece of information was obtained and organize the data in clear key-value structures. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated.\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data. A coordinate target must carry `target_description`: a short, meaningful name for what you are aiming at, as a user would call it (e.g. 'play button', 'search input', 'video body'), never a generic 'element' or 'button'. Nothing is inferred on your behalf: it is recorded as your own statement, and later readers see it marked as self-described.\n2. **Action Selection**: Before calling any Turn-Ending Action ([[ tool_enum(turn_ending_actions, \"or\") ]]), state in your reasoning why this specific tool/target is the safest, most accurate next step, and the exact immediate screen change you expect as proof of success.\n\n## Short-lived controls and measured drags\n[% if \"action_sequence\" in available_tools %]\nAuto-hiding controls MUST use one action_sequence containing opener, short wait, and operation, on EVERY adjustment, including reverse movement or restoring the minimum. This takes precedence over ordinary standalone drag advice. A previous screenshot showing an open popup is stale by execution time: never use it to justify a standalone click/drag inside that popup.\nFor sequence clicks prefer target=the complete observed resource_id string, especially opener/plus/minus: the actuator resolves fresh bounds and refuses missing/ambiguous targets. Otherwise specify pixel or normalized explicitly; never use screenshot pixels as normalized coordinates. When the observed handle is already at the requested endpoint, do not submit a zero-distance drag; re-show and verify the actual endpoint/ticks.\nUse an actually observed panel-control resource_id (not the always-visible opener) as click.unless_visible: a fresh local UI check skips the opener when the panel is already visible, avoiding toggling it closed. Set drag.require_visible to the same ID to refuse input when the panel has disappeared. Never invent IDs. If no panel ID is available, restore a known closed state before the complete opener/operation group. Guards use local observations without model turns; unavailable hierarchy refuses input.\nFor maximum/minimum, prefer observed native +/- buttons in complete guarded groups: click.times=1..10 with delay_ms (default 100), where ten taps occupy ONE member. Inspect, then continue in the same direction in another complete group to confirm the handle and ticks stop changing. Do not spend a model turn on every one or two increments. An estimated drag endpoint may miss by a few pixels; if dragging, finish the same group with the observed native button in the intended direction. Near an endpoint is not at the limit. Keep all waits/repeats within the panel visibility window.\nThe tool declares the ACTUAL per-call limit. Every click, wait and drag counts as one member. Use short complete opener/wait/operate groups; an over-limit rejection requires smaller complete groups, never separate model turns inside a transient popup. Read sequence_results: skipped means an unnecessary opener was not tapped, not_started means no input dispatched. After partial failure observe before a new plan; never blindly replay completed inputs.\nFor maximum-then-minimum, verify the real handle endpoint AND scale ticks after the first complete group, then use another complete guarded group and verify the opposite endpoint/ticks. Minute/hour ticks alone do not establish an extreme. Confirm the handle and ticks no longer change after successfully dispatched further adjustment in that direction, or use an explicit native extreme value. Event-card times are not the selected indicator time. A closed panel, reflowed layout, selected event card, or successful dispatch receipt is not evidence of changed scale.\n[% endif %]\n[% if \"drag\" in available_tools %]\nFor ordinary local timelines/sliders, use drag with explicit start/end (pixel preserves original screenshot precision) or calibration from TWO current observed numeric anchors. Use one consistent unit, such as seconds since midnight. For pan supply the fixed indicator_position; for thumb use the observed handle position. Start with a short 300ms drag; do not overshoot by default. If a panned timeline coasts after release, try a slower pan (e.g. 1500ms), wait for the ticks to stop, then measure the residual. A transient release-time bubble can disagree with the final scale at the fixed pointer; do not use that bubble as final position proof. A permanently visible timeline can be panned independently of its zoom popup. Compare actual ticks/selected value in the next observation and correct only the observed residual. Calibration receipts are pending observation, not proof of the selected value. Zoom further if pixels cannot resolve the requested accuracy. For auto-hiding controls use the complete guarded sequence above instead of standalone drag.\n[% endif %]\n" + "main_template": "# 输出语言要求\n所有自行撰写的自然语言说明必须使用简体中文，包括每一步的操作说明、计划与子任务、检查反馈、诊断结论、历史摘要和最终报告。英文示例仅示范格式，不决定输出语言。\n保留工具名、参数名、JSON 字段名、枚举值、状态码、计划语法标记、模板变量、命令、路径、URL、包名及标识符原文。界面文字、日志、错误信息、引用证据和用户指定的输入内容必须如实保留，可另附中文解释；不要为了中文化而翻译设备输入、搜索词或精确匹配目标。\n遵守各自原有输出结构；结构化结果只将允许自由填写的说明文本写成简体中文。只描述已有证据，不编造操作结果。\n\n每次调用设备操作工具前，用一小段简体中文说明当前观察、即将执行的动作和目的；操作后仅在获得结果或新观察时说明实际变化。不要只在最终答复中使用中文。工具参数中的自由描述文本也使用简体中文，精确引用的界面文字仍保留原文。\n\n# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **视觉问题复核**：报告遮挡/布局缺陷前，对照同来源、同内容的前后截图及相邻步骤，引用触发动作与实际影响，遵守用户明确允许的行为。正文滚入固定导航后方或出现背景模糊不单独证明缺陷；之前可读、可到达的内容不能仅因后续滚动而判为不可访问。持续阻碍阅读/操作或违反用户明确视觉标准时才确认相应问题。低对比度观感不能直接推断点击失效或不满足未指定的量化阈值；区分已确认问题、待确认风险和未测项。优先复用已有证据，仅补缺少的观察，不为消除疑点无限重跑。\n- **要求与证据**：沿用现有任务计划，将用户明确要求的前置条件、开关状态、检查动作和交付物列为必要项，区分待检查、已有证据、受阻。仅在状态变化时简短记下要求、来源/步骤和实际观察，不每轮重抄清单。跨端切换前和结束前核对必要项及最新用户指导，只补尚缺且可安全取得的证据；无法补齐时明确列出缺项。用户希望发生、动作已发出或计划已勾选都不证明验收通过。\n- **证据冲突**：关键文字、数值或开关状态有冲突/不清晰时，回查对应步骤原图或控件树，保留来源和不确定性，不按常识或多数摘要猜测。手机与电脑证据分别归属，仅在同条件观察支持时报告跨端差异。保持已有可靠证据，只回查缺失或矛盾部分。\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] into note (Exit: ; Interval: )` or `[Loop:continuous] into note (Interval: )` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snapshot into ``; that note is the persistent baseline and diffing ledger you cross-reference and append to on every later cycle.\n - **Step 2 (Pacing & Refresh)**: Adhere strictly to the exact duration declared in ``, then execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view. Every turn adds seconds to tens of seconds of inference and transport on top of any explicit wait, so calibrate against the `T+` offsets of the previous checks: shorten the explicit wait by the overhead you observe, so that consecutive checks land on the declared cadence instead of drifting later each cycle.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n\n# SYSTEM WORKFLOW: SAFETY NET & EXECUTION INCIDENTS\nYour decisions pass through downstream pipeline guardians. Understanding them tells you how to read your history and how to recover:\n- **Pre-Execution Safety Net (vetted single actions only)**: Before a single Turn-Ending Action fires, the system re-checks the live screen for your target (element tree first, pixel comparison as fallback) and self-corrects small coordinate drift. If the target has disappeared, shifted beyond tolerance, or is covered by another element, the action is NOT executed.\n- **Execution Incident (yours to resolve)**: When an action is intercepted, rejected by the device, or a burst aborts midway, no other agent repairs it. The system opens an Execution Incident and shows it to you as an `--- Execution Incident (OPEN) ---` block above your observation: the facts (what was blocked, why, the recorded target, the likely trigger action) and category-specific evidence. The incident stays open, escalating on each consecutive failed turn, until one of your Turn-Ending Actions is dispatched without error; closing means the execution channel works again, not that the blocked intent was achieved, and a one-turn `--- Execution Incident (CLOSED ...) ---` notice then asks you to settle that intent against the plan. Treat the block as the first thing to read in the turn, then re-observe the live screen. Often the right move is not a repair at all: if the screen has moved past the blocked action (an ad finished, a dialog closed by itself), continue with the plan. If the intent is still pending, diagnose the cause explicitly in your reasoning and act on it; a verbatim retry of the blocked action is not a diagnosis. Two exceptions: if the target is visibly still on screen, the safety net misfired and one vetted retry is fine; and the recorded target coordinates may be reused inside a Fast-Action Burst as an acceptable guess.\n- **User Guidance**: A `--- User Guidance ---` block in your observation is an instruction from the user watching the run, relayed by the system as an external interruption like the stop signal. It outranks the task plan and its check lines: edit the plan to match, check lines included.\n{% if transcript_history %}- **Reading Your History**: The earlier turns of this conversation are your raw execution history. Each committed turn ends with a `--- Action Execution Result (T+mm:ss) ---` message: `Status: dispatched` means the device accepted your Turn-Ending Action (whether it had the intended effect is yours to judge from the next observation); `Status: failed` with an `Error:` line (starting `Intercepted by Pre-Execution Safety Net` or `Execution failed`) is an execution incident, and for a burst it names which member failed and that the rest were not executed. Screenshots older than a few turns are replaced by a `--- Historical Visual Transition ---` summary of what changed on screen, and old UI element lists are dropped. Compressed spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks: ① a merged synopsis (entry/exit state, verified facts, unresolved items, failed paths, entities), ② segment titles, ③ a per-step action ledger; a recall-only era shows a period paragraph plus a `(Step-level ledger via search_history for steps a–b)` line[% if \"search_history\" in available_tools %]; query that step range with `search_history` for the full per-step ledger[% endif %]. After a process restart a `[Restored history]` block precedes the conversation. Every timestamp is a session-relative `T+mm:ss` offset.\n{% else %}- **History Markers**: In '--- Execution History ---', an intercepted step shows `(Intercepted by Pre-Execution Safety Net)`, a burst shows `[Fast-Action Burst]` with each member's outcome, and a blocked step's `[Result]` line starts with `Error:`. Use them to trace how the current screen state was reached. Step times are session-relative offsets.\n{% endif %}\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Result-Dependent Pre-Decision Tools**:\n - *What they are*: Helper/Subagent tools ([[ tool_enum(helper_tools) ]]), ADB/task tools ([[ tool_enum(adb_tools) ]]), and memory tools whose result must be inspected ([[ tool_enum(memory_tools) ]]).\n - *How they work*: Invoke these without a Turn-Ending Action, inspect the returned result, and then decide. Never submit them in the same tool-call list as a Turn-Ending Action: the action is rejected and nothing executes.\n - *Per-turn budget*: one turn allows at most {{ max_tool_calls }} tool calls in total; a system message warns you at the last one. Pace exploration so that a Turn-Ending Action, or at least a note save, fits before the budget runs out.\n2. **Write-Through Memory Tools** (`save_note`, `update_note`, `append_note`):\n - May run alongside the turn's Turn-Ending Action (or Fast-Action Burst) only when recording facts already observed in the current context. If the note content depends on the action's result, wait for the next observation before writing it. Never write an action's expected outcome as if it had already been observed.\n3. **Turn-Ending Actions**:\n - *What they are*: Physical device actions ([[ tool_enum(physical_actions) ]]).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn. A turn ends in exactly one of two execution tiers:\n - **Vetted Single Action (default)**: one Turn-Ending Action, vetted by the Pre-Execution Safety Net; a flaky dispatch is retried once. Use this for every ordinary step.\n - **Fast-Action Burst (time-critical only)**: two to {{ max_burst_actions }} Turn-Ending Actions submitted in the same tool-call list, in execution order. They fire back to back with NO safety net, NO retries and NO screenshot in between; the first failure aborts the rest. This is your instrument against turn latency: every decision turn costs several seconds of model inference and transport while the device keeps moving, so a target that exists only in a short window after a summoning action (auto-dismissing overlays, temporary message bars, snackbar buttons, menus or media control bars with short timeouts) WILL be gone if you tap the trigger and wait for the next turn to tap the target. Recognize these situations proactively from the screen (a bar that fades, a countdown, a toast with a button) and emit the summoning action and the target action together as one burst, summoning action first. Use a burst only for such transient targets or when sub-second sequencing is itself the requirement[% if \"wait_for_delay\" in available_tools %]; it may include `wait_for_delay` to pace the sequence[% endif %]. Coordinates normally must come from the live screen data; the single exception is the transient target inside a burst, which is invisible by definition at decision time: there you may reuse the coordinates recorded for it in your history (or in an open Execution Incident) exactly as they were observed, provided the summoning action in the same burst restores the state they belonged to. Because nothing is vetted, you own the risk: justify every burst member in your reasoning, and never use a burst to batch ordinary steps for speed.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, think through these three phases in this order. Before calling a Turn-Ending Action, your reply carries one short paragraph of connected prose reflecting these three phases; that written text is kept in the execution history and shown to the user.\n\n## Phase 1: Observation & Diagnosis\n1. **Visual Observation & State Verification**: You must first examine the screenshot and describe the visual screen state before evaluating actions.\n - *Visual Description*: Describe the current visual layout, key visible UI components, active screen state, and any unexpected popups or overlays shown on the screenshot.\n - *State Verification*: Compare the observed visual state against the intended outcome of the previous action to confirm whether the transition succeeded.\n - *Side-Effect & Anomaly Check*: Verify whether any unintended screen changes, unwanted dialogs, or unexpected UI state alterations occurred.\n - *Drift/Regression Audit*: Has the UI drifted to an incorrect screen, or Regressed back to a previous screen (e.g. from an accidental browser back-navigation)?\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen screen (static stuck)?\n2. **Path Decision (Recover vs. Adapt)**: Encourage flexible problem-solving. First, evaluate if the current path can still be recovered or advanced (e.g., dismissing a blocker, retrying, or restoring the view). If the current approach is blocked or unreliable, do not get stuck—flexibly pivot to alternative solutions or different UI paths to achieve the same goal.\n3. **Verify & Diagnose (Recommended Diagnosis Triggers & Loop Exemptions)**: Consider the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.[% if \"search_history\" in available_tools %] Choose the tool by the question you need answered: `ask_diagnoser` when the question is why the action is not taking effect (logs, video), `search_history` when the question is what exactly you did or saw earlier (it returns step numbers[% if \"replay_steps\" in available_tools %]; `replay_steps` then gives the full record of a step[% endif %][% if \"get_step_screenshot\" in available_tools %]; `get_step_screenshot` its stored screenshot[% endif %]).[% endif %]\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and do NOT call for diagnosis, provided you are making progress or adhering to the planned loop pacing.\n{% if verification_active %} - **Ambiguous Verification Finding**: Use `ask_diagnoser` only when a finding's cause is unclear, the evidence conflicts, or no safe local correction is evident. A verification finding alone is not a diagnosis trigger.\n{% endif %} - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n\n## Phase 2: Planning & Memory Preservation\n1. **Live Sub-goal Ledger**: `task_plan` has two layers. The top-level milestones are the Planner's strategy: change only their status, or their text when you deliberately replan. Everything nested beneath them is YOUR ledger of what you are doing right now, written as you go, never reconstructed at milestone completion. Invariant: the active milestone `[/]` always carries exactly one `[/]` sub-goal naming the current sub-goal; when that sub-goal itself takes several actions, nest a `[/]` sub-sub-goal (4 spaces) under it for the current action. Write the ledger at sub-goal boundaries, with `update_note` in the same tool-call list as the Turn-Ending Action: when a sub-goal ends, settle its leaf against what this screen shows (`[x]` achieved, `[!]` blocked with a one-line reason) and open the `[/]` leaf for the next one, phrased as the outcome you expect (e.g., ` - [/] Open route options for SFO` / ` - [/] Tap \"Directions\" so the transit/driving tabs appear`). Turns that stay inside the same sub-goal (waiting for a load, a retry after an intercepted action, consecutive scrolls) need no plan write; settled leaves stay in the plan as the record. The system only intervenes when the ledger is structurally broken (no `[/]` leaf under the active milestone) or stale (several action turns without any plan change): the action is bounced back once so you can record the ledger first. Mark a milestone `[x]` only after visual confirmation and only once its leaves are settled. Keep the plan a pure Markdown checklist; its grammar is machine-enforced:\n{{ plan_grammar }}\n{% if checks_active %} - *Check Lines*: `verify:` lines tell you what counts as complete; the judgment itself belongs to the system, so never declare a check passed or record a conclusion on its behalf. Take no extra actions for `assert:` lines and never construct state to satisfy one. Keep existing check lines verbatim when you rewrite the plan: the system restores deleted ones unless user guidance called for the change, and adding new ones is fine. Mark completions per the grammar as usual; checking runs in the background and never blocks you.{% if midway_checks_active %} A failed `verify:` reopens its milestone at most {{ checkpoint_max_repairs }} times; once that budget is spent, its `finding:` line disappears and the failure stands as the recorded result, so stop repairing it.{% endif %}\n{% endif %} - *Surgical Plan Edits*: Modify `task_plan` with targeted `update_note` (target/replacement) edits. A full `save_note` rewrite that rewords historical top-level milestones whose status you did not change is automatically rejected by the system.\n - *Loop Form of the Ledger*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), the sub-goals are the iteration items: log each concrete iteration in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle with the session-relative time from the `# CURRENT OBSERVATION [T+mm:ss]` header (e.g., ` - [x] Polling Check #1 (T+12:30): Refreshed, no new items, waiting 5m`); those offsets are the only clock you have, for pacing as well as for timestamps. This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a bounded `[Loop]`, evaluate whether its Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit); once it is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**{% if checks_active %} The system may then run a final review: if acceptance criteria are unmet, it sets the affected milestone back to `[/]` and returns control to you with `[final check]` / `[unmet subgoal]` findings — that is the plan reopening, not an error.{% endif %}\n2. **Preserve Memory & Pre-Action Snapshotting**: Use notes to preserve important findings and user corrections; tool results and screenshots already remain in the trace. Note calls and rereading large notes cost context and model round trips. Append only new evidence, combine related updates with the action in one tool-call list, and do not read or rewrite an unchanged plan for every observation. Before any UI interaction, perform this risk assessment: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain you will need these details, immediately save the visible critical values to the notes. Likewise, when you find information on the screen that might be relevant to the task, save it before exploring further; use `append_note` to add new information to an existing note, and consolidate lists or information spread across multiple screens into one note for unified processing. Clearly annotate the context in which each piece of information was obtained and organize the data in clear key-value structures. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated.\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data. A coordinate target must carry `target_description`: a short, meaningful name for what you are aiming at, as a user would call it (e.g. 'play button', 'search input', 'video body'), never a generic 'element' or 'button'. Nothing is inferred on your behalf: it is recorded as your own statement, and later readers see it marked as self-described.\n2. **Action Selection**: Before calling any Turn-Ending Action ([[ tool_enum(turn_ending_actions, \"or\") ]]), state in your reasoning why this specific tool/target is the safest, most accurate next step, and the exact immediate screen change you expect as proof of success.\n\n## Short-lived controls and measured drags\n[% if \"action_sequence\" in available_tools %]\nAuto-hiding controls MUST use one action_sequence containing opener, short wait, and operation, on EVERY adjustment, including reverse movement or restoring the minimum. This takes precedence over ordinary standalone drag advice. A previous screenshot showing an open popup is stale by execution time: never use it to justify a standalone click/drag inside that popup.\nFor sequence clicks prefer target=the complete observed resource_id string, especially opener/plus/minus: the actuator resolves fresh bounds and refuses missing/ambiguous targets. Otherwise specify pixel or normalized explicitly; never use screenshot pixels as normalized coordinates. When the observed handle is already at the requested endpoint, do not submit a zero-distance drag; re-show and verify the actual endpoint/ticks.\nUse an actually observed panel-control resource_id (not the always-visible opener) as click.unless_visible: a fresh local UI check skips the opener when the panel is already visible, avoiding toggling it closed. Set drag.require_visible to the same ID to refuse input when the panel has disappeared. Never invent IDs. If no panel ID is available, restore a known closed state before the complete opener/operation group. Guards use local observations without model turns; unavailable hierarchy refuses input.\nFor maximum/minimum, prefer observed native +/- buttons in complete guarded groups: click.times=1..10 with delay_ms (default 100), where ten taps occupy ONE member. Inspect, then continue in the same direction in another complete group to confirm the handle and ticks stop changing. Do not spend a model turn on every one or two increments. An estimated drag endpoint may miss by a few pixels; if dragging, finish the same group with the observed native button in the intended direction. Near an endpoint is not at the limit. Keep all waits/repeats within the panel visibility window.\nThe tool declares the ACTUAL per-call limit. Every click, wait and drag counts as one member. Use short complete opener/wait/operate groups; an over-limit rejection requires smaller complete groups, never separate model turns inside a transient popup. Read sequence_results: skipped means an unnecessary opener was not tapped, not_started means no input dispatched. After partial failure observe before a new plan; never blindly replay completed inputs.\nFor maximum-then-minimum, verify the real handle endpoint AND scale ticks after the first complete group, then use another complete guarded group and verify the opposite endpoint/ticks. Minute/hour ticks alone do not establish an extreme. Confirm the handle and ticks no longer change after successfully dispatched further adjustment in that direction, or use an explicit native extreme value. Event-card times are not the selected indicator time. A closed panel, reflowed layout, selected event card, or successful dispatch receipt is not evidence of changed scale.\n[% endif %]\n[% if \"drag\" in available_tools %]\nFor ordinary local timelines/sliders, use drag with explicit start/end (pixel preserves original screenshot precision) or calibration from TWO current observed numeric anchors. Use one consistent unit, such as seconds since midnight. For pan supply the fixed indicator_position; for thumb use the observed handle position. Start with a short 300ms drag; do not overshoot by default. If a panned timeline coasts after release, try a slower pan (e.g. 1500ms), wait for the ticks to stop, then measure the residual. A transient release-time bubble can disagree with the final scale at the fixed pointer; do not use that bubble as final position proof. A permanently visible timeline can be panned independently of its zoom popup. Compare actual ticks/selected value in the next observation and correct only the observed residual. Calibration receipts are pending observation, not proof of the selected value. Zoom further if pixels cannot resolve the requested accuracy. For auto-hiding controls use the complete guarded sequence above instead of standalone drag.\n[% endif %]\n\n[% if \"action_sequence\" in available_tools %]\nFor EVERY transient-panel operation, prefer one action_sequence with [ensure_visible, click/drag]. ensure_visible.target is an observed native control ID INSIDE the panel; opener is the observed UI entry ID. It skips opening when already visible, otherwise taps the opener ONCE and waits locally (timeout_ms, default1500, max3000); it is UI preparation, never a business submission or retry. Include preparation even if the previous screenshot showed the panel open. Example shape (replace all IDs with observed ones): steps=[{\"action\":\"ensure_visible\",\"target\":\"\",\"opener\":\"\",\"target_description\":\"show panel\"},{\"action\":\"click\",\"target\":\"\",\"target_description\":\"adjust\",\"times\":4}]. Copy resource IDs VERBATIM, including :id/ separators; never change them to :id:. After a preparation failure re-observe/correct the ID, never guess fallback coordinates. Each member consumes one slot. Legacy click.unless_visible means SKIP this click if visible; use it only on the opener, never on the actual operation button.\n[% endif %]\n" } diff --git a/artemis/llm/stream_guard.py b/artemis/llm/stream_guard.py index 3aeb7f4..804cb0d 100644 --- a/artemis/llm/stream_guard.py +++ b/artemis/llm/stream_guard.py @@ -10,7 +10,16 @@ from dataclasses import dataclass SMALL_ACTION_ARGUMENT_LIMIT = 16 * 1024 _SMALL_ACTIONS = frozenset( - {"click", "long_press", "swipe", "press_key", "manage_app", "wait_for_delay"} + { + "click", + "long_press", + "swipe", + "press_key", + "manage_app", + "wait_for_delay", + "action_sequence", + "drag", + } ) diff --git a/artemis/mcp/action_specs.py b/artemis/mcp/action_specs.py index 453f15e..7e1020b 100644 --- a/artemis/mcp/action_specs.py +++ b/artemis/mcp/action_specs.py @@ -261,11 +261,22 @@ _SPECS: tuple[ActionSpec, ...] = ( name="action_sequence", operator=OperatorDialect( description=( - "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, " + "[ACTION] General Android composer: execute ensure_visible, click, wait and drag steps in ONE call, " "in order, without model turns between members; explicit visibility guards " "may read the live UI locally. Use for " "temporary overlays: open the control, wait briefly, then operate it before " - "it disappears. For clicks prefer target=the exact observed resource_id string: " + "it disappears. Prefer ensure_visible(target=observed panel control ID, " + "opener=observed UI opener ID) followed by click/drag. This checks live UI, " + "skips opening an already-visible panel, or taps its opener ONCE and waits " + "up to timeout_ms (default 1500, max 3000) for the target. Each preparation " + "counts as one member. Include it again in EVERY panel-operation call, even " + "when the previous screenshot showed the panel open; do not split preparation " + "and operation into different model turns. It never retries a business action. " + "Copy all resource IDs VERBATIM, including ':id/' separators, never ':id:'. " + "If preparation fails, re-observe/correct the ID; never guess fallback coordinates. " + "Legacy click.unless_visible is ONLY for an opener, NEVER for an operation " + "such as a +/- button: visible means SKIP, not require-visible. " + "For clicks prefer target=the exact observed resource_id string: " "live bounds are resolved locally, refusing absent or ambiguous targets. " "Otherwise use observed [x,y] in the declared coordinate_space; do not confuse " "original screenshot pixels with 0..1000 normalized coordinates. Element indices " @@ -288,7 +299,11 @@ _SPECS: tuple[ActionSpec, ...] = ( "Verify actual ticks/handle endpoints before claiming maximum or minimum." ), params=( - ParamSpec("steps", list[GestureStep], "Ordered click, wait and drag members."), + ParamSpec( + "steps", + list[GestureStep], + "Ordered UI preparation, click, wait and drag members.", + ), ), ), wire=WireDialect( diff --git a/artemis/mcp/actuators/adb.py b/artemis/mcp/actuators/adb.py index e9af171..20fabe8 100644 --- a/artemis/mcp/actuators/adb.py +++ b/artemis/mcp/actuators/adb.py @@ -256,6 +256,37 @@ class AdbActuator: receipt["completed_taps"] += 1 return None + async def _ensure_visible_target(self, step: dict, receipt: dict) -> None: + """Prepare explicit UI once; poll only reads, never retry the opener.""" + receipt.update(execution="not_started", completed_taps=0, visibility_attempts=0) + elements = await self._live_ui_elements() + guard = await self._visibility_guard(step["target"], elements) + receipt["visibility_guard"] = guard + receipt["visibility_attempts"] += 1 + if guard["visible"]: + self._live_target_coordinates(step["target"], elements) + receipt["execution"] = "skipped" + return + error = await self._tap_native_target(step["opener"], receipt, elements=elements) + if error: + raise RuntimeError(error) + try: + async with asyncio.timeout(step["timeout_ms"] / 1000): + while True: + elements = await self._live_ui_elements() + guard = await self._visibility_guard(step["target"], elements) + receipt["visibility_guard"] = guard + receipt["visibility_attempts"] += 1 + if guard["visible"]: + self._live_target_coordinates(step["target"], elements) + receipt["execution"] = "completed" + return + await asyncio.sleep(0.1) + except TimeoutError as exc: + raise TimeoutError( + f"Panel target {step['target']} did not become visible within {step['timeout_ms']}ms after one opener tap. No opener retry; re-observe before continuing." + ) from exc + async def action_sequence(self, steps: list[Any]) -> ActionResult: """Execute once; optional visibility guards read live UI without model turns.""" import time @@ -296,7 +327,20 @@ class AdbActuator: receipt["execution"] = "unknown" member_execution = "completed" try: - if step["action"] == "click": + if step["action"] == "ensure_visible": + try: + await self._ensure_visible_target(step, receipt) + member_execution = receipt["execution"] + error = None + except (LookupError, TimeoutError) as exc: + error = str(exc) + member_execution = receipt["execution"] + failed_code = ( + ActionCode.TIMEOUT + if isinstance(exc, TimeoutError) + else ActionCode.TARGET_NOT_FOUND + ) + elif step["action"] == "click": guard = None elements = None error = None @@ -396,7 +440,7 @@ class AdbActuator: ) return ActionResult.success( "action_sequence", - f"Executed {completed} gesture steps consecutively; skipped {skipped} unnecessary opener clicks. Verify actual ticks/handle endpoints in the final screen.", + f"Executed {completed} gesture steps consecutively; skipped {skipped} unnecessary UI-opening steps. Verify the intended result in the final screen.", metadata=metadata, ) diff --git a/artemis/mcp/gestures.py b/artemis/mcp/gestures.py index da32736..796ef4b 100644 --- a/artemis/mcp/gestures.py +++ b/artemis/mcp/gestures.py @@ -97,7 +97,35 @@ class WaitStep(GestureModel): time_in_ms: int = Field(strict=True, ge=0, le=10000) -GestureStep = ClickStep | DragStep | WaitStep +class EnsureVisibleStep(GestureModel): + action: Literal["ensure_visible"] + target: str = Field( + min_length=1, + description="Panel control resource_id copied VERBATIM from observed elements, including the ':id/' separator. Never rewrite it as ':id:' or guess an ID.", + ) + opener: str = Field( + min_length=1, + description="UI opener resource_id copied VERBATIM from observed elements, including ':id/' (not ':id:'). Tap ONCE only if target is absent; never use a business submission as an opener.", + ) + target_description: str = Field(min_length=1) + timeout_ms: int = Field( + default=1500, + strict=True, + ge=100, + le=3000, + description="Bounded live-UI wait AFTER opening, without extra model turns or repeated opener taps.", + ) + + @model_validator(mode="after") + def distinct_opener(self): + if self.target == self.opener: + raise ValueError( + "ensure_visible target must be inside the panel, distinct from opener." + ) + return self + + +GestureStep = ClickStep | DragStep | WaitStep | EnsureVisibleStep class SequenceRequest(GestureModel): diff --git a/docs/ai-automation.md b/docs/ai-automation.md index 69a5eee..11ca795 100644 --- a/docs/ai-automation.md +++ b/docs/ai-automation.md @@ -27,6 +27,13 @@ Flash 和 Pro 都支持任务内的安卓文件工具 `android_file`：保存原 ## 短时安卓控件与精确拖动 Flash 和普通 Pro 都支持任务内原生 `action_sequence` 与 `drag`，外层 MCP 工具不变。 +这是通用的安卓短动作组合器，不限于时间轴。短时面板每次操作优先使用 `[ensure_visible, click/drag]`： +AI 指定已观察到的面板内 target、入口 opener 资源 ID 和 target_description，工具现场判断是否已经显示； +需要时只点击入口一次，并在本机等目标出现（timeout_ms 默认 1500，上限 3000），再执行后续成员。 +已显示时跳过打开，缺失、不唯一、层级不可用或超时则停止并保留回执，不重试已输入的动作。 +它只准备界面，不用于业务提交；无原生 ID 时仍可组合已观察坐标的 click/wait/drag。 +序列与校准拖动沿用小型动作的 16,384 字符参数流保护：异常膨胀时在执行前断流， +丢弃不完整参数，按已有的无效输出策略有限重试，不把截断内容派发到设备。 任务可以要求“先观察缩放弹层的真实滑块位置，恢复同布局后用一个连续动作打开、等待、拖动， 再检查刻度是否改变”。动作之间不再等待下一轮模型，适用于数秒后消失的控件。 序列使用 `pro.execution.max_burst_actions` 的数量限制（默认 4，等待也算一项）， @@ -40,7 +47,7 @@ click 的 times 支持 1–10 次连点，delay_ms 默认 100；连点仍占一 避免多轮稳定截图将短时面板消失后的画面覆盖为唯一证据；即时截图的稳定状态为未知，不能据此宣称画面已稳定。 普通 Pro 同样在完整连续组结束后保存截图，供立即进行的检查点验收使用；即使图片哈希与操作前相同， 也保留这次真实采集对应的动作后证据，不将图片去重当作没有采集。 -元素列表中的真实 `resource_id` 可用于 click 的 `unless_visible`（面板已显示时跳过打开按钮） +元素列表中的真实 `resource_id` 可用于 click 的 `unless_visible`（面板已显示时跳过打开按钮，不能加在实际操作按钮上） 及 drag 的 `require_visible`（面板消失时不拖动），现场检查不增加模型轮次；层级不可用时拒绝操作。 成员回执区分完成、跳过、结果不明和未执行；部分失败后不能将已成功或结果不明的输入整组重发。 “最大/最小”优先通过原生加减按钮调整，并核对继续同向调整后滑块、刻度是否停止变化，或读取明确的原生极值。 diff --git a/docs/llm-stream-diagnostics.md b/docs/llm-stream-diagnostics.md index e4f5540..d6c388b 100644 --- a/docs/llm-stream-diagnostics.md +++ b/docs/llm-stream-diagnostics.md @@ -34,6 +34,24 @@ 历史日志没有保存的原始帧无法由新增诊断恢复。HTTP 200 只说明响应头已到达；客户端保护后出现的 499 或 Broken pipe 也不能证明最初原因。网关写入本机下游 writer 成功不等于客户端已消费或执行。 +## 已核对的上游异常样本（2026-11-02） + +一次真实 `action_sequence` 调用在任务超时前持续接收参数，没有完成该工具调用。该样本发生在 `action_sequence` 纳入参数保护之前。按同一 `response_id + call_id` 核对生产诊断，得到以下字符数；请求编号和设备信息不在文档中保留： + +| 观测边界 | 参数字符数 | +| --- | ---: | +| CodexManager 收到的官方上游 WebSocket | 40907 | +| CodexManager 本地下游 writer | 40906 | +| NewAPI 收到的上游 SSE | 40904 | +| NewAPI 本地下游 writer | 40904 | +| ARTEMIS 的 SDK 参数片段 | 40619 | + +五个边界在 4096 到 36864 字节的九个累计 SHA256 上一致。NewAPI 使用透传模式，收到和写出的完整参数哈希也一致。上游没有终止事件，也没有完成参数快照。该样本证明参数在官方上游返回时就已持续增长，不能归因于 NewAPI 转换、下游重复累计完成快照或 ARTEMIS 最终消息拼接。取消、预读和缓冲会使各层停止计数的位置不同；尾部总长度不同本身不能证明重复发送或字符丢失。 + +诊断没有参数正文，不能进一步断言是空白循环、重复描述或官方服务内部的重复事件。这个定位也不能追溯证明没有原始诊断的历史事故。另一次 `click` 异常同样在 NewAPI 接收时已超过四万字符，说明这种现象并非新动作序列独有。 + +本地保护覆盖 `action_sequence`、`drag` 及其他小型内置动作：超限后关闭流、丢弃参数，并按既有策略有限重试；它限制异常对当前任务的影响，不修复上游生成行为，也不保证业务任务一定成功。缩放面板自动隐藏导致的定位失败由动作序列的 `ensure_visible` 准备步骤处理，与参数流异常分别诊断。 + ## 离线验证 `tests/unit/test_llm_stream_guard.py` 验证保护、有限重试、相同/不同备用端点，以及不会创建或清除其他请求的全局暂停。`tests/unit/test_request_diagnostics.py` 验证请求编号、超时重试、响应头及单次参数计数。`tests/unit/test_argument_fingerprint.py` 验证不同分片、中文和 emoji、固定字节前缀、有界统计与异常编码。这些测试不调用真实模型或设备。 diff --git a/skills/artemis-use/references/modes.md b/skills/artemis-use/references/modes.md index cba49e6..017d592 100644 --- a/skills/artemis-use/references/modes.md +++ b/skills/artemis-use/references/modes.md @@ -60,10 +60,14 @@ Flash 与普通 Pro 共用原生 `action_sequence` 和 `drag` 动作。外层 MC `mobile_run_task` 提交任务；在任务目标中说明短时控件、已观察布局、目标刻度和验收标准， 不需要切换探索或通过外部 ADB 脚本绕开队列。 -- `action_sequence(steps=[...])` 在一次调用内按序执行 click、wait、drag，不增加模型轮次； +- `action_sequence(steps=[...])` 是通用的安卓短动作组合器，在一次调用内按序执行 ensure_visible、click、wait、drag，不限于时间轴； + 每次短时面板操作优先使用 `[ensure_visible, click/drag]`，包括反向调整和上一帧看似仍展开时。 + ensure_visible 指定面板内 target、入口 opener 的真实 resource_id 及 target_description：已显示则跳过打开；否则只点击入口一次， + 在本机等目标出现（timeout_ms 默认 1500，上限 3000）。找不到、不唯一、层级不可用或超时会停止；不重试入口或业务输入。 + 此成员只准备界面，不能拿业务提交按钮作入口；没有原生 ID 的界面仍需使用已观察坐标和原有 click/wait/drag。 显式可见性检查会获取本机最新界面。工具声明显示实际 `pro.execution.max_burst_actions` 上限 （默认 4，等待也占一项）。序列内 click.target 优先用真实 resource_id，由执行层现场定位； - 找不到或不唯一时不点击。click.unless_visible 可在面板已开时跳过打开按钮； + 找不到或不唯一时不点击。旧 click.unless_visible 仅在面板已开时跳过打开按钮，不能加到加减等实际操作按钮上； drag.require_visible 可拒绝对已消失面板的拖动。ID 来自观察列表，不能猜测。 先验证整组参数，失败后停止剩余成员；检查 completed/skipped/unknown/not_started 回执， 不重发已完成输入。click 支持 times=1..10、delay_ms（默认 100），连点只占一个成员；按资源 ID 连点时 diff --git a/tests/fixtures/action_surfaces/action_server_manifest.json b/tests/fixtures/action_surfaces/action_server_manifest.json index d4a1121..c657396 100644 --- a/tests/fixtures/action_surfaces/action_server_manifest.json +++ b/tests/fixtures/action_surfaces/action_server_manifest.json @@ -228,6 +228,49 @@ "title": "DragStep", "type": "object" }, + "EnsureVisibleStep": { + "additionalProperties": false, + "properties": { + "action": { + "const": "ensure_visible", + "title": "Action", + "type": "string" + }, + "target": { + "description": "Panel control resource_id copied VERBATIM from observed elements, including the ':id/' separator. Never rewrite it as ':id:' or guess an ID.", + "minLength": 1, + "title": "Target", + "type": "string" + }, + "opener": { + "description": "UI opener resource_id copied VERBATIM from observed elements, including ':id/' (not ':id:'). Tap ONCE only if target is absent; never use a business submission as an opener.", + "minLength": 1, + "title": "Opener", + "type": "string" + }, + "target_description": { + "minLength": 1, + "title": "Target Description", + "type": "string" + }, + "timeout_ms": { + "default": 1500, + "description": "Bounded live-UI wait AFTER opening, without extra model turns or repeated opener taps.", + "maximum": 3000, + "minimum": 100, + "title": "Timeout Ms", + "type": "integer" + } + }, + "required": [ + "action", + "target", + "opener", + "target_description" + ], + "title": "EnsureVisibleStep", + "type": "object" + }, "ScaleAnchor": { "additionalProperties": false, "properties": { @@ -284,6 +327,9 @@ }, { "$ref": "#/$defs/WaitStep" + }, + { + "$ref": "#/$defs/EnsureVisibleStep" } ] }, diff --git a/tests/fixtures/action_surfaces/operator_shells.json b/tests/fixtures/action_surfaces/operator_shells.json index 42a7502..550e800 100644 --- a/tests/fixtures/action_surfaces/operator_shells.json +++ b/tests/fixtures/action_surfaces/operator_shells.json @@ -54,11 +54,11 @@ "type": "function", "function": { "name": "action_sequence", - "description": "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.", + "description": "[ACTION] General Android composer: execute ensure_visible, click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Prefer ensure_visible(target=observed panel control ID, opener=observed UI opener ID) followed by click/drag. This checks live UI, skips opening an already-visible panel, or taps its opener ONCE and waits up to timeout_ms (default 1500, max 3000) for the target. Each preparation counts as one member. Include it again in EVERY panel-operation call, even when the previous screenshot showed the panel open; do not split preparation and operation into different model turns. It never retries a business action. Copy all resource IDs VERBATIM, including ':id/' separators, never ':id:'. If preparation fails, re-observe/correct the ID; never guess fallback coordinates. Legacy click.unless_visible is ONLY for an opener, NEVER for an operation such as a +/- button: visible means SKIP, not require-visible. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.", "parameters": { "properties": { "steps": { - "description": "Ordered click, wait and drag members.", + "description": "Ordered UI preparation, click, wait and drag members.", "items": { "anyOf": [ { @@ -322,6 +322,49 @@ ], "title": "WaitStep", "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "action": { + "const": "ensure_visible", + "title": "Action", + "type": "string" + }, + "target": { + "description": "Panel control resource_id copied VERBATIM from observed elements, including the ':id/' separator. Never rewrite it as ':id:' or guess an ID.", + "minLength": 1, + "title": "Target", + "type": "string" + }, + "opener": { + "description": "UI opener resource_id copied VERBATIM from observed elements, including ':id/' (not ':id:'). Tap ONCE only if target is absent; never use a business submission as an opener.", + "minLength": 1, + "title": "Opener", + "type": "string" + }, + "target_description": { + "minLength": 1, + "title": "Target Description", + "type": "string" + }, + "timeout_ms": { + "default": 1500, + "description": "Bounded live-UI wait AFTER opening, without extra model turns or repeated opener taps.", + "maximum": 3000, + "minimum": 100, + "title": "Timeout Ms", + "type": "integer" + } + }, + "required": [ + "action", + "target", + "opener", + "target_description" + ], + "title": "EnsureVisibleStep", + "type": "object" } ] }, diff --git a/tests/fixtures/action_surfaces/tool_declarations.json b/tests/fixtures/action_surfaces/tool_declarations.json index 4001837..45b1054 100644 --- a/tests/fixtures/action_surfaces/tool_declarations.json +++ b/tests/fixtures/action_surfaces/tool_declarations.json @@ -464,11 +464,11 @@ "type": "function", "function": { "name": "action_sequence", - "description": "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.", + "description": "[ACTION] General Android composer: execute ensure_visible, click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Prefer ensure_visible(target=observed panel control ID, opener=observed UI opener ID) followed by click/drag. This checks live UI, skips opening an already-visible panel, or taps its opener ONCE and waits up to timeout_ms (default 1500, max 3000) for the target. Each preparation counts as one member. Include it again in EVERY panel-operation call, even when the previous screenshot showed the panel open; do not split preparation and operation into different model turns. It never retries a business action. Copy all resource IDs VERBATIM, including ':id/' separators, never ':id:'. If preparation fails, re-observe/correct the ID; never guess fallback coordinates. Legacy click.unless_visible is ONLY for an opener, NEVER for an operation such as a +/- button: visible means SKIP, not require-visible. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.", "parameters": { "properties": { "steps": { - "description": "Ordered click, wait and drag members.", + "description": "Ordered UI preparation, click, wait and drag members.", "items": { "anyOf": [ { @@ -732,6 +732,49 @@ ], "title": "WaitStep", "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "action": { + "const": "ensure_visible", + "title": "Action", + "type": "string" + }, + "target": { + "description": "Panel control resource_id copied VERBATIM from observed elements, including the ':id/' separator. Never rewrite it as ':id:' or guess an ID.", + "minLength": 1, + "title": "Target", + "type": "string" + }, + "opener": { + "description": "UI opener resource_id copied VERBATIM from observed elements, including ':id/' (not ':id:'). Tap ONCE only if target is absent; never use a business submission as an opener.", + "minLength": 1, + "title": "Opener", + "type": "string" + }, + "target_description": { + "minLength": 1, + "title": "Target Description", + "type": "string" + }, + "timeout_ms": { + "default": 1500, + "description": "Bounded live-UI wait AFTER opening, without extra model turns or repeated opener taps.", + "maximum": 3000, + "minimum": 100, + "title": "Timeout Ms", + "type": "integer" + } + }, + "required": [ + "action", + "target", + "opener", + "target_description" + ], + "title": "EnsureVisibleStep", + "type": "object" } ] }, diff --git a/tests/unit/mcp/test_action_manifest.py b/tests/unit/mcp/test_action_manifest.py index fba4af4..9cccfe4 100644 --- a/tests/unit/mcp/test_action_manifest.py +++ b/tests/unit/mcp/test_action_manifest.py @@ -64,6 +64,7 @@ NON_TOOL_IDENTIFIERS = frozenset( "target_description", "report_note_key", "unless_visible", + "ensure_visible", "require_visible", "sequence_results", "skipped", diff --git a/tests/unit/mcp/test_gestures.py b/tests/unit/mcp/test_gestures.py index 081e5f9..ba9e1f5 100644 --- a/tests/unit/mcp/test_gestures.py +++ b/tests/unit/mcp/test_gestures.py @@ -12,6 +12,130 @@ from artemis.mcp.actuators import MockActuator from artemis.mcp.gestures import DragRequest, plan_drag +def prepared_sequence(): + return [ + { + "action": "ensure_visible", + "target": "test.app:id/panel", + "opener": "test.app:id/opener", + "target_description": "show panel", + "timeout_ms": 100, + }, + { + "action": "click", + "target": "test.app:id/panel", + "target_description": "operate panel", + }, + ] + + +PANEL = {"resource_id": "test.app:id/panel", "bounds": [200, 300, 300, 400]} +OPENER = {"resource_id": "test.app:id/opener", "bounds": [100, 1400, 200, 1500]} + + +@pytest.mark.asyncio +async def test_preparation_keeps_open_panel_open_and_operates_its_live_target(): + actuator = MockActuator(width=720, height=1600) + actuator.controller.get_ui_elements = AsyncMock(return_value=[OPENER, PANEL]) + actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None)) + result = await actuator.action_sequence(prepared_sequence()) + assert result.ok + actuator.controller.tap_at.assert_awaited_once_with(250, 350) + assert result.metadata["sequence_results"][0]["execution"] == "skipped" + assert result.metadata["sequence_results"][0]["completed_taps"] == 0 + + +@pytest.mark.asyncio +async def test_preparation_waits_for_delayed_panel_then_uses_current_bounds(): + actuator = MockActuator(width=720, height=1600) + moved = {**PANEL, "bounds": [300, 400, 400, 500]} + actuator.controller.get_ui_elements = AsyncMock( + side_effect=[[OPENER], [OPENER], [OPENER, PANEL], [OPENER, moved]] + ) + actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None)) + sequence = prepared_sequence() + sequence[0]["timeout_ms"] = 500 + result = await actuator.action_sequence(sequence) + assert result.ok + assert [c.args for c in actuator.controller.tap_at.await_args_list] == [(150, 1450), (350, 450)] + receipt = result.metadata["sequence_results"][0] + assert receipt["completed_taps"] == 1 + assert receipt["visibility_attempts"] == 3 + assert receipt["visibility_guard"]["visible"] + + +@pytest.mark.asyncio +async def test_preparation_timeout_never_retries_opener_or_executes_body(): + actuator = MockActuator(width=720, height=1600) + actuator.controller.get_ui_elements = AsyncMock(return_value=[OPENER]) + actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None)) + result = await actuator.action_sequence(prepared_sequence()) + assert not result.ok and result.code.value == "TIMEOUT" + actuator.controller.tap_at.assert_awaited_once_with(150, 1450) + assert result.metadata["side_effect"] == "possible" + assert [r["execution"] for r in result.metadata["sequence_results"]] == [ + "unknown", + "not_started", + ] + assert result.metadata["sequence_results"][0]["completed_taps"] == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("elements", [[PANEL, PANEL], [{"resource_id": "test.app:id/root"}], []]) +async def test_preparation_refuses_ambiguous_target_missing_opener_or_unavailable_ui(elements): + actuator = MockActuator(width=720, height=1600) + actuator.controller.get_ui_elements = AsyncMock(return_value=elements) + actuator.controller.tap_at = AsyncMock() + result = await actuator.action_sequence(prepared_sequence()) + assert not result.ok and result.metadata["execution"] == "not_started" + assert result.metadata["side_effect"] == "none" + actuator.controller.tap_at.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_panel_disappearing_after_preparation_stops_without_reopening(): + actuator = MockActuator(width=720, height=1600) + actuator.controller.get_ui_elements = AsyncMock( + side_effect=[[OPENER], [OPENER, PANEL], [OPENER]] + ) + actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None)) + result = await actuator.action_sequence(prepared_sequence()) + assert not result.ok and result.code.value == "TARGET_NOT_FOUND" + actuator.controller.tap_at.assert_awaited_once_with(150, 1450) + assert result.metadata["sequence_results"][1]["execution"] == "not_started" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("invalid", ["same_id", "bad_late_member", "bad_timeout"]) +async def test_preparation_validates_whole_sequence_before_opening(invalid): + actuator = MockActuator(width=720, height=1600) + actuator.controller.tap_at = AsyncMock() + sequence = prepared_sequence() + if invalid == "same_id": + sequence[0]["opener"] = sequence[0]["target"] + elif invalid == "bad_timeout": + sequence[0]["timeout_ms"] = 3001 + else: + sequence[1]["times"] = 0 + result = await actuator.action_sequence(sequence) + assert not result.ok and result.code.value == "INVALID_ARGS" + actuator.controller.tap_at.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_preparation_round_trips_through_shared_mcp_action_server(): + actuator = MockActuator(width=720, height=1600) + actuator.controller.get_ui_elements = AsyncMock(return_value=[OPENER, PANEL]) + actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None)) + session = await ActionSession(build_action_server(actuator)).start() + try: + result = await session.call("action_sequence", {"steps": prepared_sequence()}) + assert result.ok + actuator.controller.tap_at.assert_awaited_once_with(250, 350) + finally: + await session.aclose() + + def test_pan_uses_decreasing_observed_time_scale_without_overshoot(): request = DragRequest( start=[190, 1059], diff --git a/tests/unit/test_llm_stream_guard.py b/tests/unit/test_llm_stream_guard.py index 1cef531..feba123 100644 --- a/tests/unit/test_llm_stream_guard.py +++ b/tests/unit/test_llm_stream_guard.py @@ -1,4 +1,4 @@ -"""Runaway click arguments must never reach the executor or endless retries.""" +"""Runaway native-action arguments must never reach the executor or endless retries.""" from copy import deepcopy import json @@ -45,25 +45,27 @@ def test_free_text_tools_and_regular_text_are_not_limited(name): guard.observe(delta(json.dumps({"text": text}, ensure_ascii=False), name=name)) -def test_split_and_interleaved_arguments_are_counted_per_call_without_storing_body(): +@pytest.mark.parametrize("name", ["click", "action_sequence", "drag"]) +def test_split_and_interleaved_arguments_are_counted_per_call_without_storing_body(name): guard = ToolArgumentGuard() for index in (0, 1): - guard.observe(delta('{"target_description":"', name="click", index=index)) + guard.observe(delta('{"target_description":"', name=name, index=index)) for index in (0, 1): guard.observe(delta("private" * 1200, index=index)) with pytest.raises(ToolArgumentOverflow) as caught: guard.observe(delta("private" * 1200, index=1)) - assert caught.value.details["tool"] == "click" + assert caught.value.details["tool"] == name assert caught.value.details["argument_chars"] > SMALL_ACTION_ARGUMENT_LIMIT assert "private" not in str(caught.value) assert "private" not in repr(guard.calls) -def test_name_arriving_after_arguments_still_triggers_guard(): +@pytest.mark.parametrize("name", ["click", "action_sequence", "drag"]) +def test_name_arriving_after_arguments_still_triggers_guard(name): guard = ToolArgumentGuard() guard.observe(delta("x" * (SMALL_ACTION_ARGUMENT_LIMIT + 1))) with pytest.raises(ToolArgumentOverflow): - guard.observe(delta("", name="click")) + guard.observe(delta("", name=name)) def test_invalid_output_retry_policy_is_bounded_and_honors_zero_retries(monkeypatch): @@ -76,8 +78,9 @@ def test_invalid_output_retry_policy_is_bounded_and_honors_zero_retries(monkeypa @pytest.mark.asyncio @pytest.mark.parametrize("recover", [False, True]) @pytest.mark.parametrize("first_chunk_overflows", [False, True]) +@pytest.mark.parametrize("name", ["click", "action_sequence", "drag"]) async def test_gateway_closes_bad_stream_discards_partial_and_bounds_retry( - monkeypatch, recover, first_chunk_overflows + monkeypatch, recover, first_chunk_overflows, name ): records, resets, closed = [], [], [] @@ -98,13 +101,13 @@ async def test_gateway_closes_bad_stream_discards_partial_and_bounds_retry( self.calls += 1 try: if recover and self.calls == 2: - yield delta('{"target":3}', name="click") + yield delta('{"target":3}', name=name) return if not first_chunk_overflows: - yield delta('{"target_description":"', name="click") + yield delta('{"target_description":"', name=name) yield delta( "private" * (SMALL_ACTION_ARGUMENT_LIMIT // 7 + 1), - name="click" if first_chunk_overflows else None, + name=name if first_chunk_overflows else None, ) pytest.fail("Runaway stream was consumed after the safety bound") finally: @@ -117,7 +120,7 @@ async def test_gateway_closes_bad_stream_discards_partial_and_bounds_retry( if recover: response = await wrapper.complete([]) assert response.tool_calls == [ - {"name": "click", "args": {"target": 3}, "id": None, "type": "tool_call"} + {"name": name, "args": {"target": 3}, "id": None, "type": "tool_call"} ] llm._wait_for_resume.assert_not_awaited() else: @@ -134,7 +137,7 @@ async def test_gateway_closes_bad_stream_discards_partial_and_bounds_retry( for r in records if r["type"] == "llm_diagnostic" and r["payload"]["phase"] == "failed" ) - assert failure["stream_guard"]["tool"] == "click" + assert failure["stream_guard"]["tool"] == name assert failure["stream_guard"]["argument_chars"] > SMALL_ACTION_ARGUMENT_LIMIT assert failure["failure_phase"] == "generating_tool_arguments" assert failure["category"] == "invalid_output"