diff --git a/DESIGN.md b/DESIGN.md index 1881f9b..e577626 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -84,6 +84,10 @@ Responses 消息交给工具、下游解析器或正式裁定前，必须具有 模型诊断复用现有 trace：自动重试保留本地请求编号，用尝试编号和上次尝试编号关联，重试及终止事件带对应尝试编号。可用的出站、服务端请求、Responses 响应及服务端链路编号随记录导出，仅采集白名单字段；历史缺失的来源及编号不补推。 +短时安卓控件复用 Flash/Pro 统一动作层。`action_sequence` 在单次工具调用内顺序执行 click、wait、drag，成员之间不插入模型请求、截图或安全网；整组参数在首个输入前校验，沿用 `pro.execution.max_burst_actions` 的短序列上限。暂停或新指导在成员边界生效，失败后停止，保留每项开始/结束时间及 completed/unknown/not_started 回执；不自动重发已完成或结果不明的输入。后续目标须来自同布局的真实观察，短序列不能代替业务流程的观察与验收。 + +`drag` 支持明确的归一化或原图像素端点，默认 300ms，不自动过冲。可根据当前缩放下的两个真实数值刻度校准 thumb 位置，或将 pan 刻度移至固定指针；返回真实派发端点、比例及端点取整误差。校准假定本次局部刻度线性，换缩放须重新读取锚点；计算回执保留 pending_observation，实际选中值及业务目标仍须通过动作后的观察验证。像素分辨率不足时先放大，不宣称任意时间精度。 + 任务耗时复用现有队列、工作进程控制文件与验收 trace，网页、MCP 和报告使用同一只读投影。分别列出后台排队、等待设备/资源、暂停与示范、执行与整理、最终验收及报告导出；总历时包含这些等待，不代表模型速度。缺少时间记录显示未知，不将旧任务缺失的分项补成零。运行结束后不继续累计执行耗时，报告导出单独计时。 指导接收不保证赶在终态前消费。保留接收时点、执行器消费回执及后续动作记录的区别；`consumed` 仅证明执行器已读取，不能宣称模型已应用或验收已通过。终态仍未领取的指导投影为 `unapplied`，领取后未留下消费回执的为 `unconfirmed`，不改写原始文件。网页详细查询复用原指导入口，旧数组响应保持兼容。 diff --git a/artemis/agents/flash/flash_runner.md b/artemis/agents/flash/flash_runner.md index 2179925..381c95e 100644 --- a/artemis/agents/flash/flash_runner.md +++ b/artemis/agents/flash/flash_runner.md @@ -30,6 +30,8 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi - **Time-Sensitive Tasks & Transient UI**: Because of the turn latency above, a target element that you see on the current screenshot (e.g., auto-dismissing overlays, temporary message bars, menus or control bars with short timeouts) may have already vanished by the time your physical click command reaches the device! This is one of the most common reasons why the UI does not change as expected after a single step. We call these **Time-Sensitive Tasks**. - *Mental Model*: When facing these tasks, any attempt to interact step-by-step (such as tapping once to wake up a menu/bar and waiting for the next turn to tap the target button) WILL fail because the element will auto-dismiss during your decision delay. You MUST combine the summon action on the trigger element (`Trigger`) and the target action (`Target`) into a single atomic sequence. {% if "click_sequence" in available_tools %} - *Chained Execution*: Use `click_sequence(sequence=[[trigger_x, trigger_y], [target_x, target_y]])` in `0-1000` normalized scale to execute both taps consecutively with a 50ms delay, catching the transient element before the timeout closes it.{% endif %} +{% if "action_sequence" in available_tools %} - **短时控件的混合动作**：需要打开后拖动时，优先一次调用 `action_sequence`，将 click、短暂 wait、drag 放入 steps，不分成几轮模型决策。后续目标须来自同布局的真实观察，不能猜测。逐项读取回执；部分执行后不得整组重发，先观察当前状态。{% endif %} +{% if "drag" in available_tools %} - **刻度校准**：局部滑条、时间轴优先用 `drag`。放大后可用当前原图的 pixel 坐标，或两个真实刻度 anchors 计算目标；时间统一换算成秒等同一数值单位。pan 用固定指针 indicator_position，thumb 用真实滑块起点。不要自动越过终点，也不要一味拖慢；短距离可从 300ms 起步。动作后比较同名刻度或真实选中值，仅根据已观察残差修正。回执中的 calibration 是计算/派发信息，不能当作已选中目标值；精度不足先放大。{% endif %} --- diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index a8d6120..c1e3201 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -253,7 +253,14 @@ class FlashRunner: t for t in tools if t.name not in DEVICE_ACTIONS - and t.name not in {"ask_explorer", "video_analyzer", "click_sequence"} + and t.name + not in { + "ask_explorer", + "video_analyzer", + "click_sequence", + "action_sequence", + "drag", + } ] # With an actuator installed, drop declarations for device actions the # backend does not implement (and append its extension tools); without @@ -781,7 +788,20 @@ class FlashRunner: action_dict["normalized_start_coordinates"] = norm_start action_dict["normalized_end_coordinates"] = norm_end action_dict.update(metadata.get("target_semantics") or {}) - for key in ("input_verification", "execution", "side_effect", "source"): + for key in ( + "input_verification", + "execution", + "side_effect", + "source", + "sequence_results", + "completed_steps", + "calibration", + "coordinate_space", + "coordinates", + "duration_ms", + "screen_size", + "normalized_coordinates", + ): if key in metadata: action_dict[key] = metadata[key] if metadata.get("plugin_id"): diff --git a/artemis/agents/operator/operator.json b/artemis/agents/operator/operator.json index ca9a7cc..f8b8b9f 100644 --- a/artemis/agents/operator/operator.json +++ b/artemis/agents/operator/operator.json @@ -1,3 +1,3 @@ { - "main_template": "# 输出语言要求\n所有自行撰写的自然语言说明必须使用简体中文，包括每一步的操作说明、计划与子任务、检查反馈、诊断结论、历史摘要和最终报告。英文示例仅示范格式，不决定输出语言。\n保留工具名、参数名、JSON 字段名、枚举值、状态码、计划语法标记、模板变量、命令、路径、URL、包名及标识符原文。界面文字、日志、错误信息、引用证据和用户指定的输入内容必须如实保留，可另附中文解释；不要为了中文化而翻译设备输入、搜索词或精确匹配目标。\n遵守各自原有输出结构；结构化结果只将允许自由填写的说明文本写成简体中文。只描述已有证据，不编造操作结果。\n\n每次调用设备操作工具前，用一小段简体中文说明当前观察、即将执行的动作和目的；操作后仅在获得结果或新观察时说明实际变化。不要只在最终答复中使用中文。工具参数中的自由描述文本也使用简体中文，精确引用的界面文字仍保留原文。\n\n# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **视觉问题复核**：报告遮挡/布局缺陷前，对照同来源、同内容的前后截图及相邻步骤，引用触发动作与实际影响，遵守用户明确允许的行为。正文滚入固定导航后方或出现背景模糊不单独证明缺陷；之前可读、可到达的内容不能仅因后续滚动而判为不可访问。持续阻碍阅读/操作或违反用户明确视觉标准时才确认相应问题。低对比度观感不能直接推断点击失效或不满足未指定的量化阈值；区分已确认问题、待确认风险和未测项。优先复用已有证据，仅补缺少的观察，不为消除疑点无限重跑。\n- **要求与证据**：沿用现有任务计划，将用户明确要求的前置条件、开关状态、检查动作和交付物列为必要项，区分待检查、已有证据、受阻。仅在状态变化时简短记下要求、来源/步骤和实际观察，不每轮重抄清单。跨端切换前和结束前核对必要项及最新用户指导，只补尚缺且可安全取得的证据；无法补齐时明确列出缺项。用户希望发生、动作已发出或计划已勾选都不证明验收通过。\n- **证据冲突**：关键文字、数值或开关状态有冲突/不清晰时，回查对应步骤原图或控件树，保留来源和不确定性，不按常识或多数摘要猜测。手机与电脑证据分别归属，仅在同条件观察支持时报告跨端差异。保持已有可靠证据，只回查缺失或矛盾部分。\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] into note (Exit: ; Interval: )` or `[Loop:continuous] into note (Interval: )` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snapshot into ``; that note is the persistent baseline and diffing ledger you cross-reference and append to on every later cycle.\n - **Step 2 (Pacing & Refresh)**: Adhere strictly to the exact duration declared in ``, then execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view. Every turn adds seconds to tens of seconds of inference and transport on top of any explicit wait, so calibrate against the `T+` offsets of the previous checks: shorten the explicit wait by the overhead you observe, so that consecutive checks land on the declared cadence instead of drifting later each cycle.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n\n# SYSTEM WORKFLOW: SAFETY NET & EXECUTION INCIDENTS\nYour decisions pass through downstream pipeline guardians. Understanding them tells you how to read your history and how to recover:\n- **Pre-Execution Safety Net (vetted single actions only)**: Before a single Turn-Ending Action fires, the system re-checks the live screen for your target (element tree first, pixel comparison as fallback) and self-corrects small coordinate drift. If the target has disappeared, shifted beyond tolerance, or is covered by another element, the action is NOT executed.\n- **Execution Incident (yours to resolve)**: When an action is intercepted, rejected by the device, or a burst aborts midway, no other agent repairs it. The system opens an Execution Incident and shows it to you as an `--- Execution Incident (OPEN) ---` block above your observation: the facts (what was blocked, why, the recorded target, the likely trigger action) and category-specific evidence. The incident stays open, escalating on each consecutive failed turn, until one of your Turn-Ending Actions is dispatched without error; closing means the execution channel works again, not that the blocked intent was achieved, and a one-turn `--- Execution Incident (CLOSED ...) ---` notice then asks you to settle that intent against the plan. Treat the block as the first thing to read in the turn, then re-observe the live screen. Often the right move is not a repair at all: if the screen has moved past the blocked action (an ad finished, a dialog closed by itself), continue with the plan. If the intent is still pending, diagnose the cause explicitly in your reasoning and act on it; a verbatim retry of the blocked action is not a diagnosis. Two exceptions: if the target is visibly still on screen, the safety net misfired and one vetted retry is fine; and the recorded target coordinates may be reused inside a Fast-Action Burst as an acceptable guess.\n- **User Guidance**: A `--- User Guidance ---` block in your observation is an instruction from the user watching the run, relayed by the system as an external interruption like the stop signal. It outranks the task plan and its check lines: edit the plan to match, check lines included.\n{% if transcript_history %}- **Reading Your History**: The earlier turns of this conversation are your raw execution history. Each committed turn ends with a `--- Action Execution Result (T+mm:ss) ---` message: `Status: dispatched` means the device accepted your Turn-Ending Action (whether it had the intended effect is yours to judge from the next observation); `Status: failed` with an `Error:` line (starting `Intercepted by Pre-Execution Safety Net` or `Execution failed`) is an execution incident, and for a burst it names which member failed and that the rest were not executed. Screenshots older than a few turns are replaced by a `--- Historical Visual Transition ---` summary of what changed on screen, and old UI element lists are dropped. Compressed spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks: ① a merged synopsis (entry/exit state, verified facts, unresolved items, failed paths, entities), ② segment titles, ③ a per-step action ledger; a recall-only era shows a period paragraph plus a `(Step-level ledger via search_history for steps a–b)` line[% if \"search_history\" in available_tools %]; query that step range with `search_history` for the full per-step ledger[% endif %]. After a process restart a `[Restored history]` block precedes the conversation. Every timestamp is a session-relative `T+mm:ss` offset.\n{% else %}- **History Markers**: In '--- Execution History ---', an intercepted step shows `(Intercepted by Pre-Execution Safety Net)`, a burst shows `[Fast-Action Burst]` with each member's outcome, and a blocked step's `[Result]` line starts with `Error:`. Use them to trace how the current screen state was reached. Step times are session-relative offsets.\n{% endif %}\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Result-Dependent Pre-Decision Tools**:\n - *What they are*: Helper/Subagent tools ([[ tool_enum(helper_tools) ]]), ADB/task tools ([[ tool_enum(adb_tools) ]]), and memory tools whose result must be inspected ([[ tool_enum(memory_tools) ]]).\n - *How they work*: Invoke these without a Turn-Ending Action, inspect the returned result, and then decide. Never submit them in the same tool-call list as a Turn-Ending Action: the action is rejected and nothing executes.\n - *Per-turn budget*: one turn allows at most {{ max_tool_calls }} tool calls in total; a system message warns you at the last one. Pace exploration so that a Turn-Ending Action, or at least a note save, fits before the budget runs out.\n2. **Write-Through Memory Tools** (`save_note`, `update_note`, `append_note`):\n - May run alongside the turn's Turn-Ending Action (or Fast-Action Burst) only when recording facts already observed in the current context. If the note content depends on the action's result, wait for the next observation before writing it. Never write an action's expected outcome as if it had already been observed.\n3. **Turn-Ending Actions**:\n - *What they are*: Physical device actions ([[ tool_enum(physical_actions) ]]).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn. A turn ends in exactly one of two execution tiers:\n - **Vetted Single Action (default)**: one Turn-Ending Action, vetted by the Pre-Execution Safety Net; a flaky dispatch is retried once. Use this for every ordinary step.\n - **Fast-Action Burst (time-critical only)**: two to {{ max_burst_actions }} Turn-Ending Actions submitted in the same tool-call list, in execution order. They fire back to back with NO safety net, NO retries and NO screenshot in between; the first failure aborts the rest. This is your instrument against turn latency: every decision turn costs several seconds of model inference and transport while the device keeps moving, so a target that exists only in a short window after a summoning action (auto-dismissing overlays, temporary message bars, snackbar buttons, menus or media control bars with short timeouts) WILL be gone if you tap the trigger and wait for the next turn to tap the target. Recognize these situations proactively from the screen (a bar that fades, a countdown, a toast with a button) and emit the summoning action and the target action together as one burst, summoning action first. Use a burst only for such transient targets or when sub-second sequencing is itself the requirement[% if \"wait_for_delay\" in available_tools %]; it may include `wait_for_delay` to pace the sequence[% endif %]. Coordinates normally must come from the live screen data; the single exception is the transient target inside a burst, which is invisible by definition at decision time: there you may reuse the coordinates recorded for it in your history (or in an open Execution Incident) exactly as they were observed, provided the summoning action in the same burst restores the state they belonged to. Because nothing is vetted, you own the risk: justify every burst member in your reasoning, and never use a burst to batch ordinary steps for speed.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, think through these three phases in this order. Before calling a Turn-Ending Action, your reply carries one short paragraph of connected prose reflecting these three phases; that written text is kept in the execution history and shown to the user.\n\n## Phase 1: Observation & Diagnosis\n1. **Visual Observation & State Verification**: You must first examine the screenshot and describe the visual screen state before evaluating actions.\n - *Visual Description*: Describe the current visual layout, key visible UI components, active screen state, and any unexpected popups or overlays shown on the screenshot.\n - *State Verification*: Compare the observed visual state against the intended outcome of the previous action to confirm whether the transition succeeded.\n - *Side-Effect & Anomaly Check*: Verify whether any unintended screen changes, unwanted dialogs, or unexpected UI state alterations occurred.\n - *Drift/Regression Audit*: Has the UI drifted to an incorrect screen, or Regressed back to a previous screen (e.g. from an accidental browser back-navigation)?\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen screen (static stuck)?\n2. **Path Decision (Recover vs. Adapt)**: Encourage flexible problem-solving. First, evaluate if the current path can still be recovered or advanced (e.g., dismissing a blocker, retrying, or restoring the view). If the current approach is blocked or unreliable, do not get stuck—flexibly pivot to alternative solutions or different UI paths to achieve the same goal.\n3. **Verify & Diagnose (Recommended Diagnosis Triggers & Loop Exemptions)**: Consider the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.[% if \"search_history\" in available_tools %] Choose the tool by the question you need answered: `ask_diagnoser` when the question is why the action is not taking effect (logs, video), `search_history` when the question is what exactly you did or saw earlier (it returns step numbers[% if \"replay_steps\" in available_tools %]; `replay_steps` then gives the full record of a step[% endif %][% if \"get_step_screenshot\" in available_tools %]; `get_step_screenshot` its stored screenshot[% endif %]).[% endif %]\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and do NOT call for diagnosis, provided you are making progress or adhering to the planned loop pacing.\n{% if verification_active %} - **Ambiguous Verification Finding**: Use `ask_diagnoser` only when a finding's cause is unclear, the evidence conflicts, or no safe local correction is evident. A verification finding alone is not a diagnosis trigger.\n{% endif %} - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n\n## Phase 2: Planning & Memory Preservation\n1. **Live Sub-goal Ledger**: `task_plan` has two layers. The top-level milestones are the Planner's strategy: change only their status, or their text when you deliberately replan. Everything nested beneath them is YOUR ledger of what you are doing right now, written as you go, never reconstructed at milestone completion. Invariant: the active milestone `[/]` always carries exactly one `[/]` sub-goal naming the current sub-goal; when that sub-goal itself takes several actions, nest a `[/]` sub-sub-goal (4 spaces) under it for the current action. Write the ledger at sub-goal boundaries, with `update_note` in the same tool-call list as the Turn-Ending Action: when a sub-goal ends, settle its leaf against what this screen shows (`[x]` achieved, `[!]` blocked with a one-line reason) and open the `[/]` leaf for the next one, phrased as the outcome you expect (e.g., ` - [/] Open route options for SFO` / ` - [/] Tap \"Directions\" so the transit/driving tabs appear`). Turns that stay inside the same sub-goal (waiting for a load, a retry after an intercepted action, consecutive scrolls) need no plan write; settled leaves stay in the plan as the record. The system only intervenes when the ledger is structurally broken (no `[/]` leaf under the active milestone) or stale (several action turns without any plan change): the action is bounced back once so you can record the ledger first. Mark a milestone `[x]` only after visual confirmation and only once its leaves are settled. Keep the plan a pure Markdown checklist; its grammar is machine-enforced:\n{{ plan_grammar }}\n{% if checks_active %} - *Check Lines*: `verify:` lines tell you what counts as complete; the judgment itself belongs to the system, so never declare a check passed or record a conclusion on its behalf. Take no extra actions for `assert:` lines and never construct state to satisfy one. Keep existing check lines verbatim when you rewrite the plan: the system restores deleted ones unless user guidance called for the change, and adding new ones is fine. Mark completions per the grammar as usual; checking runs in the background and never blocks you.{% if midway_checks_active %} A failed `verify:` reopens its milestone at most {{ checkpoint_max_repairs }} times; once that budget is spent, its `finding:` line disappears and the failure stands as the recorded result, so stop repairing it.{% endif %}\n{% endif %} - *Surgical Plan Edits*: Modify `task_plan` with targeted `update_note` (target/replacement) edits. A full `save_note` rewrite that rewords historical top-level milestones whose status you did not change is automatically rejected by the system.\n - *Loop Form of the Ledger*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), the sub-goals are the iteration items: log each concrete iteration in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle with the session-relative time from the `# CURRENT OBSERVATION [T+mm:ss]` header (e.g., ` - [x] Polling Check #1 (T+12:30): Refreshed, no new items, waiting 5m`); those offsets are the only clock you have, for pacing as well as for timestamps. This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a bounded `[Loop]`, evaluate whether its Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit); once it is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**{% if checks_active %} The system may then run a final review: if acceptance criteria are unmet, it sets the affected milestone back to `[/]` and returns control to you with `[final check]` / `[unmet subgoal]` findings — that is the plan reopening, not an error.{% endif %}\n2. **Preserve Memory & Pre-Action Snapshotting**: Use notes to preserve important findings and user corrections; tool results and screenshots already remain in the trace. Note calls and rereading large notes cost context and model round trips. Append only new evidence, combine related updates with the action in one tool-call list, and do not read or rewrite an unchanged plan for every observation. Before any UI interaction, perform this risk assessment: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain you will need these details, immediately save the visible critical values to the notes. Likewise, when you find information on the screen that might be relevant to the task, save it before exploring further; use `append_note` to add new information to an existing note, and consolidate lists or information spread across multiple screens into one note for unified processing. Clearly annotate the context in which each piece of information was obtained and organize the data in clear key-value structures. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated.\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data. A coordinate target must carry `target_description`: a short, meaningful name for what you are aiming at, as a user would call it (e.g. 'play button', 'search input', 'video body'), never a generic 'element' or 'button'. Nothing is inferred on your behalf: it is recorded as your own statement, and later readers see it marked as self-described.\n2. **Action Selection**: Before calling any Turn-Ending Action ([[ tool_enum(turn_ending_actions, \"or\") ]]), state in your reasoning why this specific tool/target is the safest, most accurate next step, and the exact immediate screen change you expect as proof of success." + "main_template": "# 输出语言要求\n所有自行撰写的自然语言说明必须使用简体中文，包括每一步的操作说明、计划与子任务、检查反馈、诊断结论、历史摘要和最终报告。英文示例仅示范格式，不决定输出语言。\n保留工具名、参数名、JSON 字段名、枚举值、状态码、计划语法标记、模板变量、命令、路径、URL、包名及标识符原文。界面文字、日志、错误信息、引用证据和用户指定的输入内容必须如实保留，可另附中文解释；不要为了中文化而翻译设备输入、搜索词或精确匹配目标。\n遵守各自原有输出结构；结构化结果只将允许自由填写的说明文本写成简体中文。只描述已有证据，不编造操作结果。\n\n每次调用设备操作工具前，用一小段简体中文说明当前观察、即将执行的动作和目的；操作后仅在获得结果或新观察时说明实际变化。不要只在最终答复中使用中文。工具参数中的自由描述文本也使用简体中文，精确引用的界面文字仍保留原文。\n\n# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **视觉问题复核**：报告遮挡/布局缺陷前，对照同来源、同内容的前后截图及相邻步骤，引用触发动作与实际影响，遵守用户明确允许的行为。正文滚入固定导航后方或出现背景模糊不单独证明缺陷；之前可读、可到达的内容不能仅因后续滚动而判为不可访问。持续阻碍阅读/操作或违反用户明确视觉标准时才确认相应问题。低对比度观感不能直接推断点击失效或不满足未指定的量化阈值；区分已确认问题、待确认风险和未测项。优先复用已有证据，仅补缺少的观察，不为消除疑点无限重跑。\n- **要求与证据**：沿用现有任务计划，将用户明确要求的前置条件、开关状态、检查动作和交付物列为必要项，区分待检查、已有证据、受阻。仅在状态变化时简短记下要求、来源/步骤和实际观察，不每轮重抄清单。跨端切换前和结束前核对必要项及最新用户指导，只补尚缺且可安全取得的证据；无法补齐时明确列出缺项。用户希望发生、动作已发出或计划已勾选都不证明验收通过。\n- **证据冲突**：关键文字、数值或开关状态有冲突/不清晰时，回查对应步骤原图或控件树，保留来源和不确定性，不按常识或多数摘要猜测。手机与电脑证据分别归属，仅在同条件观察支持时报告跨端差异。保持已有可靠证据，只回查缺失或矛盾部分。\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] into note (Exit: ; Interval: )` or `[Loop:continuous] into note (Interval: )` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snapshot into ``; that note is the persistent baseline and diffing ledger you cross-reference and append to on every later cycle.\n - **Step 2 (Pacing & Refresh)**: Adhere strictly to the exact duration declared in ``, then execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view. Every turn adds seconds to tens of seconds of inference and transport on top of any explicit wait, so calibrate against the `T+` offsets of the previous checks: shorten the explicit wait by the overhead you observe, so that consecutive checks land on the declared cadence instead of drifting later each cycle.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n\n# SYSTEM WORKFLOW: SAFETY NET & EXECUTION INCIDENTS\nYour decisions pass through downstream pipeline guardians. Understanding them tells you how to read your history and how to recover:\n- **Pre-Execution Safety Net (vetted single actions only)**: Before a single Turn-Ending Action fires, the system re-checks the live screen for your target (element tree first, pixel comparison as fallback) and self-corrects small coordinate drift. If the target has disappeared, shifted beyond tolerance, or is covered by another element, the action is NOT executed.\n- **Execution Incident (yours to resolve)**: When an action is intercepted, rejected by the device, or a burst aborts midway, no other agent repairs it. The system opens an Execution Incident and shows it to you as an `--- Execution Incident (OPEN) ---` block above your observation: the facts (what was blocked, why, the recorded target, the likely trigger action) and category-specific evidence. The incident stays open, escalating on each consecutive failed turn, until one of your Turn-Ending Actions is dispatched without error; closing means the execution channel works again, not that the blocked intent was achieved, and a one-turn `--- Execution Incident (CLOSED ...) ---` notice then asks you to settle that intent against the plan. Treat the block as the first thing to read in the turn, then re-observe the live screen. Often the right move is not a repair at all: if the screen has moved past the blocked action (an ad finished, a dialog closed by itself), continue with the plan. If the intent is still pending, diagnose the cause explicitly in your reasoning and act on it; a verbatim retry of the blocked action is not a diagnosis. Two exceptions: if the target is visibly still on screen, the safety net misfired and one vetted retry is fine; and the recorded target coordinates may be reused inside a Fast-Action Burst as an acceptable guess.\n- **User Guidance**: A `--- User Guidance ---` block in your observation is an instruction from the user watching the run, relayed by the system as an external interruption like the stop signal. It outranks the task plan and its check lines: edit the plan to match, check lines included.\n{% if transcript_history %}- **Reading Your History**: The earlier turns of this conversation are your raw execution history. Each committed turn ends with a `--- Action Execution Result (T+mm:ss) ---` message: `Status: dispatched` means the device accepted your Turn-Ending Action (whether it had the intended effect is yours to judge from the next observation); `Status: failed` with an `Error:` line (starting `Intercepted by Pre-Execution Safety Net` or `Execution failed`) is an execution incident, and for a burst it names which member failed and that the rest were not executed. Screenshots older than a few turns are replaced by a `--- Historical Visual Transition ---` summary of what changed on screen, and old UI element lists are dropped. Compressed spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks: ① a merged synopsis (entry/exit state, verified facts, unresolved items, failed paths, entities), ② segment titles, ③ a per-step action ledger; a recall-only era shows a period paragraph plus a `(Step-level ledger via search_history for steps a–b)` line[% if \"search_history\" in available_tools %]; query that step range with `search_history` for the full per-step ledger[% endif %]. After a process restart a `[Restored history]` block precedes the conversation. Every timestamp is a session-relative `T+mm:ss` offset.\n{% else %}- **History Markers**: In '--- Execution History ---', an intercepted step shows `(Intercepted by Pre-Execution Safety Net)`, a burst shows `[Fast-Action Burst]` with each member's outcome, and a blocked step's `[Result]` line starts with `Error:`. Use them to trace how the current screen state was reached. Step times are session-relative offsets.\n{% endif %}\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Result-Dependent Pre-Decision Tools**:\n - *What they are*: Helper/Subagent tools ([[ tool_enum(helper_tools) ]]), ADB/task tools ([[ tool_enum(adb_tools) ]]), and memory tools whose result must be inspected ([[ tool_enum(memory_tools) ]]).\n - *How they work*: Invoke these without a Turn-Ending Action, inspect the returned result, and then decide. Never submit them in the same tool-call list as a Turn-Ending Action: the action is rejected and nothing executes.\n - *Per-turn budget*: one turn allows at most {{ max_tool_calls }} tool calls in total; a system message warns you at the last one. Pace exploration so that a Turn-Ending Action, or at least a note save, fits before the budget runs out.\n2. **Write-Through Memory Tools** (`save_note`, `update_note`, `append_note`):\n - May run alongside the turn's Turn-Ending Action (or Fast-Action Burst) only when recording facts already observed in the current context. If the note content depends on the action's result, wait for the next observation before writing it. Never write an action's expected outcome as if it had already been observed.\n3. **Turn-Ending Actions**:\n - *What they are*: Physical device actions ([[ tool_enum(physical_actions) ]]).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn. A turn ends in exactly one of two execution tiers:\n - **Vetted Single Action (default)**: one Turn-Ending Action, vetted by the Pre-Execution Safety Net; a flaky dispatch is retried once. Use this for every ordinary step.\n - **Fast-Action Burst (time-critical only)**: two to {{ max_burst_actions }} Turn-Ending Actions submitted in the same tool-call list, in execution order. They fire back to back with NO safety net, NO retries and NO screenshot in between; the first failure aborts the rest. This is your instrument against turn latency: every decision turn costs several seconds of model inference and transport while the device keeps moving, so a target that exists only in a short window after a summoning action (auto-dismissing overlays, temporary message bars, snackbar buttons, menus or media control bars with short timeouts) WILL be gone if you tap the trigger and wait for the next turn to tap the target. Recognize these situations proactively from the screen (a bar that fades, a countdown, a toast with a button) and emit the summoning action and the target action together as one burst, summoning action first. Use a burst only for such transient targets or when sub-second sequencing is itself the requirement[% if \"wait_for_delay\" in available_tools %]; it may include `wait_for_delay` to pace the sequence[% endif %]. Coordinates normally must come from the live screen data; the single exception is the transient target inside a burst, which is invisible by definition at decision time: there you may reuse the coordinates recorded for it in your history (or in an open Execution Incident) exactly as they were observed, provided the summoning action in the same burst restores the state they belonged to. Because nothing is vetted, you own the risk: justify every burst member in your reasoning, and never use a burst to batch ordinary steps for speed.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, think through these three phases in this order. Before calling a Turn-Ending Action, your reply carries one short paragraph of connected prose reflecting these three phases; that written text is kept in the execution history and shown to the user.\n\n## Phase 1: Observation & Diagnosis\n1. **Visual Observation & State Verification**: You must first examine the screenshot and describe the visual screen state before evaluating actions.\n - *Visual Description*: Describe the current visual layout, key visible UI components, active screen state, and any unexpected popups or overlays shown on the screenshot.\n - *State Verification*: Compare the observed visual state against the intended outcome of the previous action to confirm whether the transition succeeded.\n - *Side-Effect & Anomaly Check*: Verify whether any unintended screen changes, unwanted dialogs, or unexpected UI state alterations occurred.\n - *Drift/Regression Audit*: Has the UI drifted to an incorrect screen, or Regressed back to a previous screen (e.g. from an accidental browser back-navigation)?\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen screen (static stuck)?\n2. **Path Decision (Recover vs. Adapt)**: Encourage flexible problem-solving. First, evaluate if the current path can still be recovered or advanced (e.g., dismissing a blocker, retrying, or restoring the view). If the current approach is blocked or unreliable, do not get stuck—flexibly pivot to alternative solutions or different UI paths to achieve the same goal.\n3. **Verify & Diagnose (Recommended Diagnosis Triggers & Loop Exemptions)**: Consider the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.[% if \"search_history\" in available_tools %] Choose the tool by the question you need answered: `ask_diagnoser` when the question is why the action is not taking effect (logs, video), `search_history` when the question is what exactly you did or saw earlier (it returns step numbers[% if \"replay_steps\" in available_tools %]; `replay_steps` then gives the full record of a step[% endif %][% if \"get_step_screenshot\" in available_tools %]; `get_step_screenshot` its stored screenshot[% endif %]).[% endif %]\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and do NOT call for diagnosis, provided you are making progress or adhering to the planned loop pacing.\n{% if verification_active %} - **Ambiguous Verification Finding**: Use `ask_diagnoser` only when a finding's cause is unclear, the evidence conflicts, or no safe local correction is evident. A verification finding alone is not a diagnosis trigger.\n{% endif %} - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n\n## Phase 2: Planning & Memory Preservation\n1. **Live Sub-goal Ledger**: `task_plan` has two layers. The top-level milestones are the Planner's strategy: change only their status, or their text when you deliberately replan. Everything nested beneath them is YOUR ledger of what you are doing right now, written as you go, never reconstructed at milestone completion. Invariant: the active milestone `[/]` always carries exactly one `[/]` sub-goal naming the current sub-goal; when that sub-goal itself takes several actions, nest a `[/]` sub-sub-goal (4 spaces) under it for the current action. Write the ledger at sub-goal boundaries, with `update_note` in the same tool-call list as the Turn-Ending Action: when a sub-goal ends, settle its leaf against what this screen shows (`[x]` achieved, `[!]` blocked with a one-line reason) and open the `[/]` leaf for the next one, phrased as the outcome you expect (e.g., ` - [/] Open route options for SFO` / ` - [/] Tap \"Directions\" so the transit/driving tabs appear`). Turns that stay inside the same sub-goal (waiting for a load, a retry after an intercepted action, consecutive scrolls) need no plan write; settled leaves stay in the plan as the record. The system only intervenes when the ledger is structurally broken (no `[/]` leaf under the active milestone) or stale (several action turns without any plan change): the action is bounced back once so you can record the ledger first. Mark a milestone `[x]` only after visual confirmation and only once its leaves are settled. Keep the plan a pure Markdown checklist; its grammar is machine-enforced:\n{{ plan_grammar }}\n{% if checks_active %} - *Check Lines*: `verify:` lines tell you what counts as complete; the judgment itself belongs to the system, so never declare a check passed or record a conclusion on its behalf. Take no extra actions for `assert:` lines and never construct state to satisfy one. Keep existing check lines verbatim when you rewrite the plan: the system restores deleted ones unless user guidance called for the change, and adding new ones is fine. Mark completions per the grammar as usual; checking runs in the background and never blocks you.{% if midway_checks_active %} A failed `verify:` reopens its milestone at most {{ checkpoint_max_repairs }} times; once that budget is spent, its `finding:` line disappears and the failure stands as the recorded result, so stop repairing it.{% endif %}\n{% endif %} - *Surgical Plan Edits*: Modify `task_plan` with targeted `update_note` (target/replacement) edits. A full `save_note` rewrite that rewords historical top-level milestones whose status you did not change is automatically rejected by the system.\n - *Loop Form of the Ledger*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), the sub-goals are the iteration items: log each concrete iteration in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle with the session-relative time from the `# CURRENT OBSERVATION [T+mm:ss]` header (e.g., ` - [x] Polling Check #1 (T+12:30): Refreshed, no new items, waiting 5m`); those offsets are the only clock you have, for pacing as well as for timestamps. This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a bounded `[Loop]`, evaluate whether its Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit); once it is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**{% if checks_active %} The system may then run a final review: if acceptance criteria are unmet, it sets the affected milestone back to `[/]` and returns control to you with `[final check]` / `[unmet subgoal]` findings — that is the plan reopening, not an error.{% endif %}\n2. **Preserve Memory & Pre-Action Snapshotting**: Use notes to preserve important findings and user corrections; tool results and screenshots already remain in the trace. Note calls and rereading large notes cost context and model round trips. Append only new evidence, combine related updates with the action in one tool-call list, and do not read or rewrite an unchanged plan for every observation. Before any UI interaction, perform this risk assessment: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain you will need these details, immediately save the visible critical values to the notes. Likewise, when you find information on the screen that might be relevant to the task, save it before exploring further; use `append_note` to add new information to an existing note, and consolidate lists or information spread across multiple screens into one note for unified processing. Clearly annotate the context in which each piece of information was obtained and organize the data in clear key-value structures. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated.\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data. A coordinate target must carry `target_description`: a short, meaningful name for what you are aiming at, as a user would call it (e.g. 'play button', 'search input', 'video body'), never a generic 'element' or 'button'. Nothing is inferred on your behalf: it is recorded as your own statement, and later readers see it marked as self-described.\n2. **Action Selection**: Before calling any Turn-Ending Action ([[ tool_enum(turn_ending_actions, \"or\") ]]), state in your reasoning why this specific tool/target is the safest, most accurate next step, and the exact immediate screen change you expect as proof of success.\n\n## Short-lived controls and measured drags\n[% if \"action_sequence\" in available_tools %]\nFor a short-lived popup requiring click/wait/drag, prefer ONE action_sequence tool call with typed steps. Use targets observed in the same restored layout. No model calls or screenshots occur between members. Read sequence_results after execution; on partial failure observe the current state before continuing, never replay completed inputs.\n[% endif %]\n[% if \"drag\" in available_tools %]\nFor a local timeline or slider, prefer drag with explicit start/end (pixel supports original-screenshot precision) or calibration from TWO current observed numeric anchors. Use one consistent unit, such as seconds since midnight. For pan provide the fixed indicator_position; for thumb use its observed position. Start with a short 300ms drag rather than assuming slower is more precise. Never overshoot by default. Compare actual ticks/selected value in the next observation and correct only the observed residual. Calibration receipts are pending observation, not proof of the selected value. Zoom further if pixels cannot resolve the requested accuracy.\n[% endif %]\n" } diff --git a/artemis/agents/operator/operator.py b/artemis/agents/operator/operator.py index e21b63d..88f3d70 100644 --- a/artemis/agents/operator/operator.py +++ b/artemis/agents/operator/operator.py @@ -1555,6 +1555,22 @@ class OperatorNode: """The recorded target semantics: observed (index) or described (coords).""" return {k: v for k, v in el.items() if k != "center"} + if tool_name in {"drag", "action_sequence"}: + from artemis.mcp.gestures import DragRequest, SequenceRequest, plan_drag, plan_sequence + + try: + if tool_name == "drag": + request = DragRequest.model_validate(args) + planned = plan_drag(request, width, height) + return [{**planned, "args": request.model_dump(mode="json")}], None + request_sequence = SequenceRequest.model_validate(args) + plan_sequence(request_sequence, width, height, self._max_burst_actions()) + return [ + {"action": "action_sequence", "args": request_sequence.model_dump(mode="json")} + ], None + except ValueError as exc: + return [], f"Error during {tool_name}: {exc}" + if tool_name == "click": target = args.get("target") times = args.get("times", 1) diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index 411a5be..fb94e58 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -82,6 +82,8 @@ _PRE_DECISION_ALL_TOOLS = ( _PHYSICAL_ACTIONS_ORDER = OPERATOR_SHELL_ORDER _TURN_ENDING_ORDER = ( "click", + "action_sequence", + "drag", "swipe", "input_text", "long_press", diff --git a/artemis/agents/validator/action_execution.py b/artemis/agents/validator/action_execution.py index f7fd8c0..098c089 100644 --- a/artemis/agents/validator/action_execution.py +++ b/artemis/agents/validator/action_execution.py @@ -103,6 +103,9 @@ async def exec_action( return False, f"MCP call failed: {e}" action_item.update(execution="completed" if res.ok else "unknown", side_effect="possible") + action_item.update(res.metadata) + if res.metadata: + action_item["result"] = res.agent_message expected = ( expected_input_state(wire_args["text"], wire_args["clear_exist"]) if tool_name == "input_text" diff --git a/artemis/agents/validator/execution_loop.py b/artemis/agents/validator/execution_loop.py index d83a9a7..6a33c61 100644 --- a/artemis/agents/validator/execution_loop.py +++ b/artemis/agents/validator/execution_loop.py @@ -384,7 +384,9 @@ async def run_validation_loop(node, state: State) -> dict: execution: list[dict] = [] actions_to_execute = list(actions) total_actions = len(actions_to_execute) - burst = total_actions > 1 + burst = total_actions > 1 or any( + a.get("action") == "action_sequence" for a in actions_to_execute + ) if burst: logger.info( f"Fast-action burst: {total_actions} actions will fire back to back" diff --git a/artemis/agents/validator/tool_declarations.py b/artemis/agents/validator/tool_declarations.py index 1fe0a58..5f2fe39 100644 --- a/artemis/agents/validator/tool_declarations.py +++ b/artemis/agents/validator/tool_declarations.py @@ -203,6 +203,8 @@ LONG_PRESS_TOOL = tool_declaration("long_press") INPUT_TEXT_TOOL = tool_declaration("input_text") SWIPE_TOOL = tool_declaration("swipe") +DRAG_TOOL = tool_declaration("drag") +ACTION_SEQUENCE_TOOL = tool_declaration("action_sequence") PRESS_KEY_TOOL = tool_declaration("press_key") @@ -299,6 +301,8 @@ VALIDATOR_TOOLS_DECLARATION: list[ToolDeclaration] = [ LONG_PRESS_TOOL, INPUT_TEXT_TOOL, SWIPE_TOOL, + DRAG_TOOL, + ACTION_SEQUENCE_TOOL, PRESS_KEY_TOOL, READ_NOTE_TOOL, LIST_NOTES_TOOL, diff --git a/artemis/mcp/action_executor.py b/artemis/mcp/action_executor.py index d059bd0..93e7a3b 100644 --- a/artemis/mcp/action_executor.py +++ b/artemis/mcp/action_executor.py @@ -85,7 +85,17 @@ AGENT_TOOL_NAMES: frozenset[str] = ( #: Actions whose ``target`` is a single point: an element index or an [x, y] pair. _POINT_TARGET_ACTIONS: frozenset[str] = frozenset({"click", "long_press", "input_text"}) _SCREEN_CHANGING_ACTIONS: frozenset[str] = frozenset( - {"click", "click_sequence", "long_press", "input_text", "press_key", "swipe", "manage_app"} + { + "click", + "click_sequence", + "action_sequence", + "drag", + "long_press", + "input_text", + "press_key", + "swipe", + "manage_app", + } ) @@ -337,7 +347,7 @@ class McpActionExecutor: ) else: res = await session.call(wire_name, wire_args) - message = finalize(res) if finalize else res.message + message = finalize(res) if finalize else res.agent_message if res.ok or self._observe_despite_failure(raw_name, res): obs, observation_settled = await self._observe_after_action( @@ -393,6 +403,7 @@ class McpActionExecutor: "post_image_name": post_image_name, "has_xml": bool(xml_list), "status": status, + **({"gesture_result": res.metadata} if res.metadata else {}), **({"source": source} if source and dispatched else {}), } if status == "error": @@ -423,6 +434,7 @@ class McpActionExecutor: if res.code in {ActionCode.INVALID_ARGS, ActionCode.UNSUPPORTED, ActionCode.PACKAGE_NOT_FOUND} else "possible", + **res.metadata, }, ) @@ -436,6 +448,8 @@ class McpActionExecutor: return res.code not in (ActionCode.PACKAGE_NOT_FOUND, ActionCode.INVALID_ARGS) if raw_name == "wait_for_text": return res.code is ActionCode.TIMEOUT + if raw_name in {"drag", "action_sequence"}: + return res.code not in {ActionCode.INVALID_ARGS, ActionCode.UNSUPPORTED} return False # --- Argument translation -------------------------------------------------------- @@ -458,6 +472,25 @@ class McpActionExecutor: ``index_elements`` overrides the list index targets resolve against (see :meth:`execute`). """ + if raw_name in {"drag", "action_sequence"}: + from artemis.config.agent import load_agent_config + from artemis.mcp.gestures import DragRequest, SequenceRequest, plan_drag, plan_sequence + + try: + if raw_name == "drag": + request = DragRequest.model_validate(args) + planned = plan_drag(request, *self._screen_size()) + return "drag", request.model_dump(mode="json"), None, planned + sequence_request = SequenceRequest.model_validate(args) + plan_sequence( + sequence_request, + *self._screen_size(), + limit=load_agent_config().pro.execution.max_burst_actions, + ) + return "action_sequence", sequence_request.model_dump(mode="json"), None, {} + except ValueError as exc: + raise _ArgError(f"Error during {raw_name}: {exc}") from exc + if raw_name == "click": target, recorded = self._resolve_target( args.get("target") or args.get("coordinates"), diff --git a/artemis/mcp/action_manifest.py b/artemis/mcp/action_manifest.py index 5ea9fd0..267b8b3 100644 --- a/artemis/mcp/action_manifest.py +++ b/artemis/mcp/action_manifest.py @@ -79,6 +79,8 @@ OPTIONAL_ACTIONS: frozenset[str] = frozenset( "long_press", "input_text", "swipe", + "drag", + "action_sequence", "press_key", "manage_app", "wait_for_delay", diff --git a/artemis/mcp/action_names.py b/artemis/mcp/action_names.py index 78cef0a..620be8b 100644 --- a/artemis/mcp/action_names.py +++ b/artemis/mcp/action_names.py @@ -23,7 +23,8 @@ prompts are the contract. The Operator/Validator-internal vocabulary ``to_canonical_call`` converts a Validator action item (the dicts Operator writes into ``state.structured_decisions``) into a canonical ``(name, kwargs)`` pair for the actuator/session layer, using the ``normalized_coordinates`` the Operator already -computed so the canonical surface never needs a pixel escape hatch. +computed. Explicit drag/sequence requests preserve their declared coordinate +space so precise pixels are not quantized through normalization. """ from typing import Any @@ -35,6 +36,8 @@ OPERATOR_ACTION_TO_CANONICAL: dict[str, str] = { "tap": "click", "long_press_on": "long_press", "swipe": "swipe", + "drag": "drag", + "action_sequence": "action_sequence", "focus_and_input_text": "input_text", "focus_and_clear_text": "focus_and_clear_text", "erase_one_char": "erase_one_char", @@ -87,6 +90,9 @@ def to_canonical_call( if canonical is None: raise ValueError(f"Unsupported action: {verb}") + if verb in {"drag", "action_sequence"}: + return canonical, dict(action_item.get("args") or {}) + if verb == "tap": pair = _norm_pair(action_item, dims) if pair is None: diff --git a/artemis/mcp/action_specs.py b/artemis/mcp/action_specs.py index 1ea69a1..f8804ae 100644 --- a/artemis/mcp/action_specs.py +++ b/artemis/mcp/action_specs.py @@ -63,11 +63,13 @@ import types from typing import Annotated, Any, Literal, Union, get_args, get_origin from langchain_core.tools import StructuredTool +from langchain_core.utils.function_calling import convert_to_openai_tool from mcp.types import CallToolResult -from pydantic import Field, create_model +from pydantic import BaseModel, Field, create_model from artemis.core.tool_declaration import ToolDeclaration from artemis.mcp.action_types import ActionResult +from artemis.mcp.gestures import CoordinateSpace, DragCalibration, GestureStep __all__ = [ "ACTION_SPECS", @@ -213,6 +215,14 @@ async def _wire_swipe(actuator: Any, a: dict[str, Any]) -> ActionResult: ) +async def _wire_drag(actuator: Any, a: dict[str, Any]) -> ActionResult: + return await actuator.drag(**a) + + +async def _wire_action_sequence(actuator: Any, a: dict[str, Any]) -> ActionResult: + return await actuator.action_sequence(a["steps"]) + + async def _wire_press_key(actuator: Any, a: dict[str, Any]) -> ActionResult: return await actuator.press_key(a["key"]) @@ -244,6 +254,86 @@ async def _wire_focus_and_clear_text(actuator: Any, a: dict[str, Any]) -> Action # --- The manifest -------------------------------------------------------------------- _SPECS: tuple[ActionSpec, ...] = ( + ActionSpec( + name="action_sequence", + operator=OperatorDialect( + description=( + "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, " + "in order, without model turns or screenshots between members. Use for " + "temporary overlays: open the control, wait briefly, then operate it before " + "it disappears. Use observed coordinates, or positions learned from an " + "earlier observation of the same restored layout; element indices are not " + "accepted. Each click/drag needs target_description. All members are " + "validated before any input. The configured fast-action limit applies. " + "The first failure stops the remainder; receipts identify completed and " + "unexecuted members. Never replay the whole sequence after a partial result. " + "Verify the final screen before claiming the intended result." + ), + params=( + ParamSpec("steps", list[GestureStep], "Ordered click, wait and drag members."), + ), + ), + wire=WireDialect( + description="Execute validated short gesture steps consecutively; stop at the first failure.", + params=(ParamSpec("steps", list[GestureStep]),), + bind=_wire_action_sequence, + ), + ), + ActionSpec( + name="drag", + operator=OperatorDialect( + description=( + "[ACTION] Precisely drag a local slider or timeline. Use explicit start/end " + "or calibration from TWO observed numeric scale anchors. coordinate_space " + "is normalized (0..1000) or pixel (the current screenshot's actual pixels); " + "pixel avoids quantization on a zoomed scale. Default duration_ms=300: " + "slower is not always more precise and may trigger long-press recognition. " + "Do not overshoot the endpoint automatically. For calibration, axis=x/y; " + "mode=thumb moves the handle to target_value, mode=pan shifts the scale " + "under indicator_position. Use one numeric unit, e.g. seconds since midnight, " + "and two anchors from the CURRENT zoom/layout. Provide exactly one of end " + "or calibration. Returned geometry is a plan/dispatch receipt, not proof of " + "the selected value: compare the post-action scale/value and correct only " + "the observed residual; zoom further if the pixel resolution is insufficient." + ), + params=( + ParamSpec("start", list[int], "Observed handle or pan starting point [x, y]."), + ParamSpec( + "target_description", str, "The slider, handle or timeline being dragged." + ), + ParamSpec( + "end", + list[int] | None, + "Exact endpoint. Set null (or omit) when calibration is supplied; never compute both.", + required=False, + ), + ParamSpec( + "coordinate_space", CoordinateSpace, required=False, default="normalized" + ), + ParamSpec("duration_ms", int, required=False, default=300), + ParamSpec( + "calibration", + DragCalibration | None, + "Observed numeric anchors that compute the endpoint. When supplied, end MUST be null; otherwise set calibration null.", + required=False, + ), + ), + ), + wire=WireDialect( + description="Drag with exact coordinates or an observed two-anchor calibration.", + params=( + ParamSpec("start", list[int]), + ParamSpec("target_description", str), + ParamSpec("end", list[int] | None, required=False), + ParamSpec( + "coordinate_space", CoordinateSpace, required=False, default="normalized" + ), + ParamSpec("duration_ms", int, required=False, default=300), + ParamSpec("calibration", DragCalibration | None, required=False), + ), + bind=_wire_drag, + ), + ), ActionSpec( name="click", operator=OperatorDialect( @@ -470,7 +560,7 @@ _SPECS: tuple[ActionSpec, ...] = ( "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n" "\n" "• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n" - "• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n" + "• Precise Coordinate Gestures ('start', 'end'): Use observed endpoints for local slider/timeline movement. Do not automatically overshoot: slow short gestures can trigger long-press recognition. Compare the actual post-action value/scale before adjusting again. When available, drag supports pixel coordinates and two-anchor calibration for fine positioning.\n" "\n" "Args:\n" " direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n" @@ -684,6 +774,8 @@ ACTION_SPECS: dict[str, ActionSpec] = {spec.name: spec for spec in _SPECS} #: identical to the prompt's "Physical device actions" enumeration order). OPERATOR_SHELL_ORDER: tuple[str, ...] = ( "click", + "action_sequence", + "drag", "input_text", "swipe", "press_key", @@ -694,6 +786,8 @@ OPERATOR_SHELL_ORDER: tuple[str, ...] = ( #: Exception wording per action; matches the historical executor `except` arms. EXCEPTION_PREFIXES: dict[str, str] = { + "action_sequence": "Error during action_sequence", + "drag": "Error during drag", "click": "Error during click", "click_sequence": "Error executing click sequence", "long_press": "Error during long press", @@ -781,8 +875,18 @@ def _json_schema(annotation: Any) -> dict[str, Any]: raise TypeError(f"No JSON projection for parameter annotation {annotation!r}.") from None -def _projected_parameters(dialect: OperatorDialect) -> dict[str, Any]: +def _projected_parameters(dialect: OperatorDialect, name: str) -> dict[str, Any]: """The operator shell's parameters as a JSON-schema ``parameters`` object.""" + + def has_model(annotation: Any) -> bool: + return (isinstance(annotation, type) and issubclass(annotation, BaseModel)) or any( + has_model(arg) for arg in get_args(annotation) + ) + + if any(has_model(p.annotation) for p in dialect.params): + # Nullable alternatives matter for endpoint-versus-calibration requests: + # models must be able to send end=null instead of inventing an endpoint. + return convert_to_openai_tool(operator_shell_tool(name))["function"]["parameters"] properties: dict[str, Any] = {} required: list[str] = [] for p in dialect.params: @@ -814,7 +918,7 @@ def tool_declaration(name: str) -> ToolDeclaration: return ToolDeclaration( name=name, description=spec.operator.description, - parameters=_projected_parameters(spec.operator), + parameters=_projected_parameters(spec.operator, name), ) raise ValueError(f"Action '{name}' has no agent dialect to declare.") diff --git a/artemis/mcp/action_types.py b/artemis/mcp/action_types.py index c1462cf..5c17b85 100644 --- a/artemis/mcp/action_types.py +++ b/artemis/mcp/action_types.py @@ -52,6 +52,19 @@ class ActionResult(BaseModel): """Raw driver/controller error for diagnostics; never shown to the LLM.""" normalized_coordinates: list[int] | None = None duration_ms: int | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + """Actual geometry, calibration and per-member receipts for compound gestures.""" + + @property + def agent_message(self) -> str: + """Expose gesture receipts in the model's tool response as well as the trace.""" + if self.action in {"drag", "action_sequence"} and self.metadata: + import json + + return ( + self.message + "\nGesture receipt: " + json.dumps(self.metadata, ensure_ascii=False) + ) + return self.message @classmethod def success(cls, action: str, message: str, **kwargs: Any) -> "ActionResult": diff --git a/artemis/mcp/actuators/adb.py b/artemis/mcp/actuators/adb.py index d0c1107..e04aff6 100644 --- a/artemis/mcp/actuators/adb.py +++ b/artemis/mcp/actuators/adb.py @@ -132,6 +132,119 @@ class AdbActuator: # --- Required action ------------------------------------------------------------- + async def drag(self, **kwargs: Any) -> ActionResult: + """Dispatch exact pixel geometry without a normalized-coordinate round trip.""" + from artemis.mcp.gestures import DragRequest, plan_drag + + try: + planned = plan_drag(DragRequest.model_validate(kwargs), *self._dims()) + except ValueError as exc: + return ActionResult.failure("drag", str(exc), code=ActionCode.INVALID_ARGS) + return await self._dispatch_drag(planned) + + async def _dispatch_drag(self, planned: dict) -> ActionResult: + error = await self.controller.swipe_coords(*planned["coordinates"], planned["duration_ms"]) + metadata = dict(planned) + metadata.pop("action") + if error: + return ActionResult.failure("drag", str(error), metadata=metadata) + return ActionResult.success( + "drag", + f"Dragged from {planned['coordinates'][:2]} to {planned['coordinates'][2:]}" + f" (pixel) over {planned['duration_ms']}ms. Verify the scale/value in the next observation.", + duration_ms=planned["duration_ms"], + metadata=metadata, + ) + + async def action_sequence(self, steps: list[Any]) -> ActionResult: + """Validate the entire sequence, then execute once with no intermediate observation.""" + import time + + from artemis.config.agent import load_agent_config + from artemis.data_engine.trace import TraceSpan + from artemis.mcp.gestures import SequenceRequest, plan_sequence + from artemis.runtime.task_control import TaskControl, has_pending_instruction + + try: + planned = plan_sequence( + SequenceRequest.model_validate({"steps": steps}), + *self._dims(), + limit=load_agent_config().pro.execution.max_burst_actions, + ) + except ValueError as exc: + return ActionResult.failure("action_sequence", str(exc), code=ActionCode.INVALID_ARGS) + started = time.monotonic() + receipts = [ + {**step, "index": i + 1, "execution": "not_started"} for i, step in enumerate(planned) + ] + failed = None + for step, receipt in zip(planned, receipts, strict=True): + control = getattr(self.ctx, "task_control", None) + if isinstance(control, TaskControl) and ( + has_pending_instruction(control.directory) + or (control.directory / "pause.request").exists() + ): + failed = "Sequence interrupted before the next input; re-observe before continuing." + break + with TraceSpan( + name=f"action_sequence.{step['action']}", trace_type="tool", ctx=self.ctx + ) as span: + span.payload = {"args": step} + receipt["started_ms"] = round((time.monotonic() - started) * 1000, 3) + receipt["execution"] = "unknown" + try: + if step["action"] == "click": + result = await self.controller.tap_at(*step["coordinates"]) + error = getattr(result, "error", None) + elif step["action"] == "drag": + result = await self._dispatch_drag(step) + error = None if result.ok else result.message + else: + await asyncio.sleep(step["time_in_ms"] / 1000) + error = None + receipt.update(execution="unknown" if error else "completed", error=error) + if error: + failed = str(error) + span.status = "failed" + span.error = failed + except asyncio.CancelledError: + receipt.update(execution="unknown", error="Input cancelled; effect uncertain.") + raise + except (OSError, RuntimeError) as exc: + receipt.update(execution="unknown", error=str(exc)) + failed = str(exc) + span.status = "failed" + span.error = failed + finally: + receipt["ended_ms"] = round((time.monotonic() - started) * 1000, 3) + span.result = dict(receipt) + span.payload["receipt"] = dict(receipt) + if failed: + break + completed = sum(r["execution"] == "completed" for r in receipts) + attempted = any(r["action"] != "wait" and r["execution"] != "not_started" for r in receipts) + execution = "completed" if not failed else ("partial" if completed else "unknown") + if failed and not attempted: + execution = "not_started" + metadata = { + "sequence_results": receipts, + "completed_steps": completed, + "execution": execution, + "side_effect": "possible" if attempted else "none", + } + if failed: + return ActionResult.failure( + "action_sequence", + f"Sequence stopped after {completed}/{len(planned)} completed steps: {failed}" + " Remaining steps were not executed; do not replay completed inputs.", + metadata=metadata, + ) + return ActionResult.success( + "action_sequence", + f"Executed {completed} gesture steps consecutively. Verify the final screen.", + metadata=metadata, + ) + async def click_sequence( self, points: list[tuple[int, int]], delay_ms: int = 50 ) -> ActionResult: diff --git a/artemis/mcp/actuators/base.py b/artemis/mcp/actuators/base.py index 1dd4955..6ed3346 100644 --- a/artemis/mcp/actuators/base.py +++ b/artemis/mcp/actuators/base.py @@ -40,7 +40,8 @@ __all__ = ["Actuator"] class Actuator(Protocol): """Protocol every actuator backend implements. - All coordinates are 0-1000 normalized. Every action coroutine returns an + Coordinates are 0-1000 normalized, except drag/sequence members explicitly + marked coordinate_space="pixel". Every action coroutine returns an :class:`ActionResult` and must not raise for device-level failures -- those are reported through ``ok=False`` so a failed tap reaches the agent as an observation rather than an exception. @@ -84,6 +85,10 @@ class Actuator(Protocol): async def press_key(self, key: str) -> ActionResult: ... + async def drag(self, **kwargs: Any) -> ActionResult: ... + + async def action_sequence(self, steps: list[Any]) -> ActionResult: ... + async def manage_app(self, action: str, app_name: str) -> ActionResult: ... async def wait_for_delay(self, time_in_ms: int) -> ActionResult: ... diff --git a/artemis/mcp/gestures.py b/artemis/mcp/gestures.py new file mode 100644 index 0000000..53a0920 --- /dev/null +++ b/artemis/mcp/gestures.py @@ -0,0 +1,187 @@ +"""Validated, shared plans for short gesture sequences and calibrated drags. + +Planning is pure: every member is checked before the first device input. Scale +values are supplied from observations by the caller, never inferred by the driver. +""" + +import math +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, StrictInt, model_validator + + +class GestureModel(BaseModel): + model_config = ConfigDict( + extra="forbid", strict=True, allow_inf_nan=False, str_strip_whitespace=True + ) + + +Point = Annotated[list[StrictInt], Field(min_length=2, max_length=2)] +Duration = Annotated[int, Field(strict=True, ge=1, le=10000)] +CoordinateSpace = Literal["normalized", "pixel"] + + +class ScaleAnchor(GestureModel): + position: float = Field( + description="Observed axis coordinate in the selected coordinate space." + ) + value: float = Field(description="Observed scale value, in one consistent numeric unit.") + + +class DragCalibration(GestureModel): + axis: Literal["x", "y"] + mode: Literal["thumb", "pan"] = Field( + description="thumb moves a handle to a value; pan moves a scale under a fixed indicator." + ) + anchors: list[ScaleAnchor] = Field(min_length=2, max_length=2) + target_value: float + indicator_position: float | None = Field( + default=None, description="Required for pan: the fixed indicator's observed axis position." + ) + + +class DragRequest(GestureModel): + start: Point + target_description: str = Field(min_length=1) + end: Point | None = Field( + default=None, description="Exact endpoint; MUST be null when calibration is supplied." + ) + coordinate_space: CoordinateSpace = "normalized" + duration_ms: Duration = 300 + calibration: DragCalibration | None = Field( + default=None, description="Computes the endpoint from observed anchors; then set end=null." + ) + + @model_validator(mode="after") + def endpoint_or_calibration(self): + if (self.end is None) == (self.calibration is None): + raise ValueError( + "Supply exactly one of end or calibration. Calibration computes the endpoint: set end=null (or omit end) when calibration is supplied. Do not repeat the rejected combination." + ) + return self + + +class ClickStep(GestureModel): + action: Literal["click"] + target: Point + target_description: str = Field(min_length=1) + coordinate_space: CoordinateSpace = "normalized" + + +class DragStep(DragRequest): + action: Literal["drag"] + + +class WaitStep(GestureModel): + action: Literal["wait"] + time_in_ms: int = Field(strict=True, ge=0, le=10000) + + +GestureStep = ClickStep | DragStep | WaitStep + + +class SequenceRequest(GestureModel): + steps: list[GestureStep] = Field(min_length=1, max_length=10) + + +def pixel_point(point: list[int], space: CoordinateSpace, width: int, height: int) -> list[int]: + if width <= 0 or height <= 0: + raise ValueError("Screen dimensions must be positive.") + limits = [1001, 1001] if space == "normalized" else [width, height] + if any(p < 0 or p >= limit for p, limit in zip(point, limits, strict=True)): + raise ValueError(f"Point {point} is outside the {space} screen bounds.") + if space == "pixel": + return list(point) + return [min(size - 1, p * size // 1000) for p, size in zip(point, [width, height], strict=True)] + + +def plan_drag(request: DragRequest, width: int, height: int) -> dict: + start = pixel_point(request.start, request.coordinate_space, width, height) + calibration = None + if request.calibration is None: + if request.end is None: + raise ValueError("Drag requires end or calibration.") + end = pixel_point(request.end, request.coordinate_space, width, height) + else: + scale = request.calibration + axis = 0 if scale.axis == "x" else 1 + size = width if axis == 0 else height + + def position(raw: float) -> float: + upper = 1000 if request.coordinate_space == "normalized" else size - 1 + if not 0 <= raw <= upper: + raise ValueError("Calibration positions must be inside the current screen.") + return ( + min(size - 1, raw * size / 1000) + if request.coordinate_space == "normalized" + else raw + ) + + a, b = scale.anchors + p1, p2 = position(a.position), position(b.position) + if p1 == p2 or a.value == b.value: + raise ValueError("Calibration needs two distinct positions and values.") + slope = (b.value - a.value) / (p2 - p1) + if not math.isfinite(slope) or slope == 0: + raise ValueError("Calibration slope must be finite and nonzero.") + target_position = p1 + (scale.target_value - a.value) / slope + if not math.isfinite(target_position): + raise ValueError("Calibrated target must be finite.") + if scale.mode == "thumb": + if not min(a.value, b.value) <= scale.target_value <= max(a.value, b.value): + raise ValueError("Thumb target must be within the observed calibration range.") + endpoint = target_position + else: + if scale.indicator_position is None: + raise ValueError("Pan calibration requires indicator_position.") + endpoint = start[axis] + position(scale.indicator_position) - target_position + if not 0 <= endpoint <= size - 1: + raise ValueError( + "Calibrated endpoint is outside the screen; zoom or use a smaller move." + ) + end = list(start) + end[axis] = round(endpoint) + calibration = { + **scale.model_dump(mode="json"), + "anchor_coordinate_space": request.coordinate_space, + "value_per_pixel": slope, + "target_position_pixel": target_position, + "rounding_error_value": abs(end[axis] - endpoint) * abs(slope), + "verification": "pending_observation", + } + if start == end: + raise ValueError("Drag has no pixel movement; zoom further or choose a different endpoint.") + return { + "action": "drag", + "coordinates": [*start, *end], + "coordinate_space": "pixel", + "screen_size": [width, height], + "normalized_coordinates": [ + round(value * 1000 / size) + for value, size in zip([*start, *end], [width, height, width, height], strict=True) + ], + "duration_ms": request.duration_ms, + "target_description": request.target_description, + **({"calibration": calibration} if calibration else {}), + } + + +def plan_sequence(request: SequenceRequest, width: int, height: int, limit: int) -> list[dict]: + if len(request.steps) > limit: + raise ValueError(f"Sequence exceeds the configured fast-action limit of {limit} steps.") + planned = [] + for step in request.steps: + if isinstance(step, DragStep): + planned.append(plan_drag(step, width, height)) + elif isinstance(step, ClickStep): + planned.append( + { + "action": "click", + "coordinates": pixel_point(step.target, step.coordinate_space, width, height), + "coordinate_space": "pixel", + "target_description": step.target_description, + } + ) + else: + planned.append(step.model_dump(mode="json")) + return planned diff --git a/artemis/utils/coordinates.py b/artemis/utils/coordinates.py index fc90a8f..a25b4aa 100644 --- a/artemis/utils/coordinates.py +++ b/artemis/utils/coordinates.py @@ -417,8 +417,10 @@ def normalize_step_actions(step_dict: dict[str, Any]) -> dict[str, Any]: act_name = str(item.get("action") or item.get("name") or "").lower() if ( - "swipe" in act_name or "scroll" in act_name or "drag" in act_name - ) and "normalized_coordinates" not in item: + ("swipe" in act_name or "scroll" in act_name or "drag" in act_name) + and "normalized_coordinates" not in item + and item.get("coordinate_space") != "pixel" + ): kind, target, _ = parse_swipe_parameters(item) if kind == "coords" and isinstance(target, list) and len(target) == 4: item["normalized_coordinates"] = target @@ -445,22 +447,32 @@ def normalize_step_actions(step_dict: dict[str, Any]) -> dict[str, Any]: coords = item.get("coordinates") if coords and isinstance(coords, list) and "normalized_coordinates" not in item: # If coordinates look physical (e.g., target_bounds present or > 1000), normalize using width/height - if item.get("target_bounds") or any( - isinstance(c, (int, float)) and c > 1000 for c in coords + if ( + item.get("coordinate_space") == "pixel" + or item.get("target_bounds") + or any(isinstance(c, (int, float)) and c > 1000 for c in coords) ): + size = item.get("screen_size") + if not ( + isinstance(size, (list, tuple)) + and len(size) == 2 + and all(isinstance(v, (int, float)) and v > 0 for v in size) + ): + size = [width, height] + item_width, item_height = size if len(coords) == 2 and all(isinstance(c, (int, float)) for c in coords): item["normalized_coordinates"] = [ - int(round(coords[0] * 1000.0 / width)), - int(round(coords[1] * 1000.0 / height)), + int(round(coords[0] * 1000.0 / item_width)), + int(round(coords[1] * 1000.0 / item_height)), ] elif len(coords) == 4 and all(isinstance(c, (int, float)) for c in coords): nx1, ny1 = ( - int(round(coords[0] * 1000.0 / width)), - int(round(coords[1] * 1000.0 / height)), + int(round(coords[0] * 1000.0 / item_width)), + int(round(coords[1] * 1000.0 / item_height)), ) nx2, ny2 = ( - int(round(coords[2] * 1000.0 / width)), - int(round(coords[3] * 1000.0 / height)), + int(round(coords[2] * 1000.0 / item_width)), + int(round(coords[3] * 1000.0 / item_height)), ) item["normalized_coordinates"] = [nx1, ny1, nx2, ny2] item["normalized_start_coordinates"] = [nx1, ny1] diff --git a/artemis/utils/task_tree.py b/artemis/utils/task_tree.py index 8571c20..16c920b 100644 --- a/artemis/utils/task_tree.py +++ b/artemis/utils/task_tree.py @@ -113,7 +113,16 @@ def format_action_clean(action_obj) -> str: clear_exist = action_obj.get("clear_exist") or action_obj.get("clear_before_input") clear_str = " (without clearing)" if clear_exist is False else "" return f"Inputted '{text_val}' into {label} at {coords}{clear_str}" - elif act_type == "swipe": + elif act_type == "action_sequence": + members = action_obj.get("sequence_results") or action_obj.get("steps") or [] + if not members and isinstance(action_obj.get("args"), dict): + members = action_obj["args"].get("steps") or [] + return "Gesture sequence: " + "; ".join( + f"{m.get('action')} {m.get('target_description', '')}" + f" [{m.get('execution', 'requested')}]" + for m in members + ) + elif act_type in {"swipe", "drag"}: direction = action_obj.get("action_direction") or action_obj.get("direction") if not direction and isinstance(action_obj.get("gesture"), str): # Flash swipe-by-direction records the direction as ``gesture`` @@ -141,7 +150,8 @@ def format_action_clean(action_obj) -> str: return f"Swiped {direction}{path}{duration_str}" dragged = _target_label(action_obj, fallback="") dragged_str = f" {dragged}" if dragged else "" - return f"Swiped{dragged_str} from {start_coords} to {end_coords}{duration_str}" + verb = "Dragged" if act_type == "drag" else "Swiped" + return f"{verb}{dragged_str} from {start_coords} to {end_coords}{duration_str}" elif act_type == "press_key": return f"Pressed key '{keycode}'" elif act_type == "launch_app": @@ -325,17 +335,27 @@ def format_result_clean(result_obj) -> str | None: if not isinstance(result_obj, dict): return str(result_obj) + execution = result_obj.get("execution") + gestures = [ + a["result"] + for a in (execution if isinstance(execution, list) else []) + if isinstance(a, dict) + and a.get("action") in {"drag", "action_sequence"} + and isinstance(a.get("result"), str) + ] + receipt = "\n".join(gestures) + suffix = "\n" + receipt if receipt else "" incident = result_obj.get("incident") if isinstance(incident, dict) and incident.get("reason"): - return format_incident_clean(incident) + return format_incident_clean(incident) + suffix status = result_obj.get("status") error = failed_execution_error(result_obj) if error: - return _error_line(error) + return _error_line(error) + suffix if status in ("failed", "error"): - return "Error" - return None + return "Error" + suffix + return receipt or None def _error_line(error: str) -> str: @@ -414,6 +434,8 @@ def format_tool_call_clean( "click_sequence", "tap_sequence", "swipe", + "drag", + "action_sequence", "input_text", "focus_and_input_text", "press_key", diff --git a/artemis/utils/visualization.py b/artemis/utils/visualization.py index 5370819..333df73 100644 --- a/artemis/utils/visualization.py +++ b/artemis/utils/visualization.py @@ -517,14 +517,20 @@ def crop_and_annotate_target( return out_buf.getvalue() -def _resolve_coordinates(coord: Any, width: int, height: int) -> tuple[int, int] | None: +def _resolve_coordinates( + coord: Any, width: int, height: int, coordinate_space: str | None = None +) -> tuple[int, int] | None: """Helper to safely parse coordinates into pixel space.""" if not isinstance(coord, (list, tuple)) or len(coord) < 2: return None try: x, y = float(coord[0]), float(coord[1]) # If coordinates are normalized (0.0 <= x <= 1.0) - if 0.0 <= x <= 1.0 and 0.0 <= y <= 1.0: + if coordinate_space == "pixel": + px, py = int(round(x)), int(round(y)) + elif coordinate_space == "normalized": + px, py = int(round(x * width / 1000)), int(round(y * height / 1000)) + elif 0.0 <= x <= 1.0 and 0.0 <= y <= 1.0: px = int(round(x * width)) py = int(round(y * height)) elif 0 <= x <= 1000 and 0 <= y <= 1000 and (width > 1000 or height > 1000): @@ -563,6 +569,17 @@ def draw_action_overlay_on_image( if not image_bytes: return image_bytes + if action_name == "action_sequence": + receipts = action_args.get("sequence_results") + annotated = image_bytes + for member in receipts if isinstance(receipts, list) else []: + if ( + isinstance(member, dict) + and member.get("execution") == "completed" + and member.get("action") in {"click", "drag"} + ): + annotated = draw_action_overlay_on_image(annotated, member["action"], member) + return annotated try: img = Image.open(io.BytesIO(image_bytes)) if img.mode in ("RGBA", "P"): @@ -584,7 +601,7 @@ def draw_action_overlay_on_image( or action_args.get("coordinates") or action_args.get("point") ) - pt = _resolve_coordinates(coord, width, height) + pt = _resolve_coordinates(coord, width, height, action_args.get("coordinate_space")) if pt: x, y = pt # Outer white aura @@ -637,7 +654,9 @@ def draw_action_overlay_on_image( if isinstance(sequence, list): pts = [] for coord in sequence: - pt = _resolve_coordinates(coord, width, height) + pt = _resolve_coordinates( + coord, width, height, action_args.get("coordinate_space") + ) if pt: pts.append(pt) @@ -703,8 +722,12 @@ def draw_action_overlay_on_image( start_pt, end_pt = None, None if kind == "coords" and isinstance(target, list) and len(target) == 4: - start_pt = _resolve_coordinates(target[:2], width, height) - end_pt = _resolve_coordinates(target[2:], width, height) + start_pt = _resolve_coordinates( + target[:2], width, height, action_args.get("coordinate_space") + ) + end_pt = _resolve_coordinates( + target[2:], width, height, action_args.get("coordinate_space") + ) elif kind == "direction" and isinstance(target, str): direction = target.lower() cx, cy = width // 2, height // 2 @@ -758,7 +781,7 @@ def draw_action_overlay_on_image( elif action in ("input_text", "type"): coord = action_args.get("target") or action_args.get("coordinates") if coord: - pt = _resolve_coordinates(coord, width, height) + pt = _resolve_coordinates(coord, width, height, action_args.get("coordinate_space")) if pt: x, y = pt box_w = max(60, int(base_dim * 0.15)) @@ -805,9 +828,21 @@ def overlay_action_on_screenshot(image_bytes: bytes, action_obj: Any) -> bytes | "sequence", "targets", "normalized_coordinates", + "coordinate_space", + "sequence_results", ): if key in action and key not in action_args: action_args[key] = action[key] + if ( + act_name == "drag" + and isinstance(action.get("coordinates"), list) + and len(action["coordinates"]) == 4 + ): + action_args.update( + start=action["coordinates"][:2], + end=action["coordinates"][2:], + coordinate_space=action.get("coordinate_space"), + ) annotated = draw_action_overlay_on_image( image_bytes=image_bytes, action_name=act_name, action_args=action_args ) diff --git a/docs/ai-automation.md b/docs/ai-automation.md index 4a2a5f5..925afb9 100644 --- a/docs/ai-automation.md +++ b/docs/ai-automation.md @@ -24,6 +24,18 @@ Flash 和 Pro 都支持任务内的安卓文件工具 `android_file`：保存原 文件操作结果会进入任务执行记录，Flash 中也逐项记录操作步骤及校验结果，供终检和报告引用；不会只留下模型自行撰写的“已完成”说明。 +## 短时安卓控件与精确拖动 + +Flash 和普通 Pro 都支持任务内原生 `action_sequence` 与 `drag`，外层 MCP 工具不变。 +任务可以要求“先观察缩放弹层的真实滑块位置，恢复同布局后用一个连续动作打开、等待、拖动， +再检查刻度是否改变”。动作之间不再等待下一轮模型，适用于数秒后消失的控件。 +序列使用 `pro.execution.max_burst_actions` 的数量限制（默认 4）；部分失败后保留成员回执， +不能将已成功或结果不明的输入整组重发。 + +时间轴可使用原图像素和当前两个刻度校准拖动，避免细刻度被整数归一化坐标量化。 +它仍受截图分辨率、刻度线性、应用吸附及实际拖动响应影响；应先放大、短距离拖动， +再读取同名刻度或选中时间作修正。工具成功和计算误差不等于业务时间已准确选中。 + ## MCP 能力 仅 PC 测试与网页“仅 PC”勾选项使用同一任务目标。先用 `mobile_workspace(include_devices=false)` 查看实际电脑插件，再提交 `mobile_run_task(task_desc="在电脑浏览器检查指定页面", model="Flash", request_id="本次提交唯一编号", runtime_overrides={"execution_target":"computer"})`。无需手机或模拟器在线；仍需可用模型和已启用并授权的电脑观察插件。`model="Pro"` 与 `mobile_exploration` 的 `task_config.execution_target="computer"` 同样适用。每项批量用例的 `task_config` 使用相同字段；插件参数见 [插件任务参数](../skills/artemis-use/references/plugins.md)。 diff --git a/skills/artemis-use/references/modes.md b/skills/artemis-use/references/modes.md index f5a261e..597729d 100644 --- a/skills/artemis-use/references/modes.md +++ b/skills/artemis-use/references/modes.md @@ -54,6 +54,22 @@ mobile_run_task( - `completed/failed/cancelled/interrupted/skipped` 为执行终态；检查 test_result 再说明通过、失败、未验证或阻塞。人工判定附来源和原 AI 判定，不改写原始证据。 - 实际耗时受模型、网络及设备影响，不能固定扣减 5/30 秒当作精确等待。最终外部测试脚本应使用明确等待条件。 +## 短时控件与精确拖动 + +Flash 与普通 Pro 共用原生 `action_sequence` 和 `drag` 动作。外层 MCP 仍通过 +`mobile_run_task` 提交任务；在任务目标中说明短时控件、已观察布局、目标刻度和验收标准， +不需要切换探索或通过外部 ADB 脚本绕开队列。 + +- `action_sequence(steps=[...])` 在一次调用内按序执行 click、wait、drag，不在成员之间 + 截图或请求模型。沿用 `pro.execution.max_burst_actions` 的连续动作数量限制（默认 4）。 + 先验证整组参数，失败后停止剩余成员；检查 `sequence_results`，不要重发已完成输入。 +- `drag` 接受 start、end、coordinate_space（normalized 或当前截图的 pixel）、 + duration_ms（默认 300）及 target_description。pixel 避免细刻度被整数归一化坐标量化。 +- 可用 calibration 替代 end：axis、mode（thumb/pan）、两个 anchors（position/value）、 + target_value；pan 还要提供固定指针 indicator_position。各值必须来自当前缩放下的真实观察， + 使用同一数值单位。工具返回计算出的端点、比例和量化误差，未自动识别当前选中值。 +- 拖动成功只表示派发成功。对照动作后同名刻度或选中值再判断；不得自动过冲或把慢拖当作精准。 + ## 安卓文件 任务内部支持 `android_file`，外层以任务目标交给 ARTEMIS，文件操作也走队列。截图原生 PNG 默认留存任务 trace；明确要求桌面时额外复制并校验，不需电脑插件。读写文本、列目录、pull/push 使用目标 Android 路径及任务文件边界，不能绕过应用私有目录权限。已有文件默认不覆盖。只有工具返回实际路径及校验结果才说明保存成功；「全部资料」导出包含任务文件。 diff --git a/tests/fixtures/action_surfaces/action_server_manifest.json b/tests/fixtures/action_surfaces/action_server_manifest.json index 968aa5d..6c46796 100644 --- a/tests/fixtures/action_surfaces/action_server_manifest.json +++ b/tests/fixtures/action_surfaces/action_server_manifest.json @@ -1,4 +1,570 @@ { + "action_sequence": { + "description": "Execute validated short gesture steps consecutively; stop at the first failure.", + "inputSchema": { + "$defs": { + "ClickStep": { + "additionalProperties": false, + "properties": { + "action": { + "const": "click", + "title": "Action", + "type": "string" + }, + "target": { + "items": { + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "title": "Target", + "type": "array" + }, + "target_description": { + "minLength": 1, + "title": "Target Description", + "type": "string" + }, + "coordinate_space": { + "default": "normalized", + "enum": [ + "normalized", + "pixel" + ], + "title": "Coordinate Space", + "type": "string" + } + }, + "required": [ + "action", + "target", + "target_description" + ], + "title": "ClickStep", + "type": "object" + }, + "DragCalibration": { + "additionalProperties": false, + "properties": { + "axis": { + "enum": [ + "x", + "y" + ], + "title": "Axis", + "type": "string" + }, + "mode": { + "description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.", + "enum": [ + "thumb", + "pan" + ], + "title": "Mode", + "type": "string" + }, + "anchors": { + "items": { + "$ref": "#/$defs/ScaleAnchor" + }, + "maxItems": 2, + "minItems": 2, + "title": "Anchors", + "type": "array" + }, + "target_value": { + "title": "Target Value", + "type": "number" + }, + "indicator_position": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Required for pan: the fixed indicator's observed axis position.", + "title": "Indicator Position" + } + }, + "required": [ + "axis", + "mode", + "anchors", + "target_value" + ], + "title": "DragCalibration", + "type": "object" + }, + "DragStep": { + "additionalProperties": false, + "properties": { + "start": { + "items": { + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "title": "Start", + "type": "array" + }, + "target_description": { + "minLength": 1, + "title": "Target Description", + "type": "string" + }, + "end": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Exact endpoint; MUST be null when calibration is supplied.", + "title": "End" + }, + "coordinate_space": { + "default": "normalized", + "enum": [ + "normalized", + "pixel" + ], + "title": "Coordinate Space", + "type": "string" + }, + "duration_ms": { + "default": 300, + "maximum": 10000, + "minimum": 1, + "title": "Duration Ms", + "type": "integer" + }, + "calibration": { + "anyOf": [ + { + "$ref": "#/$defs/DragCalibration" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Computes the endpoint from observed anchors; then set end=null." + }, + "action": { + "const": "drag", + "title": "Action", + "type": "string" + } + }, + "required": [ + "start", + "target_description", + "action" + ], + "title": "DragStep", + "type": "object" + }, + "ScaleAnchor": { + "additionalProperties": false, + "properties": { + "position": { + "description": "Observed axis coordinate in the selected coordinate space.", + "title": "Position", + "type": "number" + }, + "value": { + "description": "Observed scale value, in one consistent numeric unit.", + "title": "Value", + "type": "number" + } + }, + "required": [ + "position", + "value" + ], + "title": "ScaleAnchor", + "type": "object" + }, + "WaitStep": { + "additionalProperties": false, + "properties": { + "action": { + "const": "wait", + "title": "Action", + "type": "string" + }, + "time_in_ms": { + "maximum": 10000, + "minimum": 0, + "title": "Time In Ms", + "type": "integer" + } + }, + "required": [ + "action", + "time_in_ms" + ], + "title": "WaitStep", + "type": "object" + } + }, + "properties": { + "steps": { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/ClickStep" + }, + { + "$ref": "#/$defs/DragStep" + }, + { + "$ref": "#/$defs/WaitStep" + } + ] + }, + "title": "Steps", + "type": "array" + } + }, + "required": [ + "steps" + ], + "title": "action_sequenceArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } + }, + "drag": { + "description": "Drag with exact coordinates or an observed two-anchor calibration.", + "inputSchema": { + "$defs": { + "DragCalibration": { + "additionalProperties": false, + "properties": { + "axis": { + "enum": [ + "x", + "y" + ], + "title": "Axis", + "type": "string" + }, + "mode": { + "description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.", + "enum": [ + "thumb", + "pan" + ], + "title": "Mode", + "type": "string" + }, + "anchors": { + "items": { + "$ref": "#/$defs/ScaleAnchor" + }, + "maxItems": 2, + "minItems": 2, + "title": "Anchors", + "type": "array" + }, + "target_value": { + "title": "Target Value", + "type": "number" + }, + "indicator_position": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Required for pan: the fixed indicator's observed axis position.", + "title": "Indicator Position" + } + }, + "required": [ + "axis", + "mode", + "anchors", + "target_value" + ], + "title": "DragCalibration", + "type": "object" + }, + "ScaleAnchor": { + "additionalProperties": false, + "properties": { + "position": { + "description": "Observed axis coordinate in the selected coordinate space.", + "title": "Position", + "type": "number" + }, + "value": { + "description": "Observed scale value, in one consistent numeric unit.", + "title": "Value", + "type": "number" + } + }, + "required": [ + "position", + "value" + ], + "title": "ScaleAnchor", + "type": "object" + } + }, + "properties": { + "start": { + "items": { + "type": "integer" + }, + "title": "Start", + "type": "array" + }, + "target_description": { + "title": "Target Description", + "type": "string" + }, + "end": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "End" + }, + "coordinate_space": { + "default": "normalized", + "enum": [ + "normalized", + "pixel" + ], + "title": "Coordinate Space", + "type": "string" + }, + "duration_ms": { + "default": 300, + "title": "Duration Ms", + "type": "integer" + }, + "calibration": { + "anyOf": [ + { + "$ref": "#/$defs/DragCalibration" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "start", + "target_description" + ], + "title": "dragArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } + }, "click": { "description": "Tap at a 0-1000 normalized [x, y] coordinate.", "inputSchema": { @@ -99,6 +665,11 @@ ], "default": null, "title": "Duration Ms" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" } }, "required": [ @@ -209,6 +780,11 @@ ], "default": null, "title": "Duration Ms" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" } }, "required": [ @@ -316,6 +892,11 @@ ], "default": null, "title": "Duration Ms" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" } }, "required": [ @@ -435,6 +1016,11 @@ ], "default": null, "title": "Duration Ms" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" } }, "required": [ @@ -550,6 +1136,11 @@ ], "default": null, "title": "Duration Ms" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" } }, "required": [ @@ -649,6 +1240,11 @@ ], "default": null, "title": "Duration Ms" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" } }, "required": [ @@ -753,6 +1349,11 @@ ], "default": null, "title": "Duration Ms" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" } }, "required": [ @@ -852,6 +1453,11 @@ ], "default": null, "title": "Duration Ms" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" } }, "required": [ @@ -961,6 +1567,11 @@ ], "default": null, "title": "Duration Ms" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" } }, "required": [ @@ -1060,6 +1671,11 @@ ], "default": null, "title": "Duration Ms" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" } }, "required": [ @@ -1151,6 +1767,11 @@ ], "default": null, "title": "Duration Ms" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" } }, "required": [ @@ -1253,6 +1874,11 @@ ], "default": null, "title": "Duration Ms" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" } }, "required": [ diff --git a/tests/fixtures/action_surfaces/operator_shells.json b/tests/fixtures/action_surfaces/operator_shells.json index 6f3e6c8..a6324ba 100644 --- a/tests/fixtures/action_surfaces/operator_shells.json +++ b/tests/fixtures/action_surfaces/operator_shells.json @@ -50,6 +50,376 @@ } } }, + "action_sequence": { + "type": "function", + "function": { + "name": "action_sequence", + "description": "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, in order, without model turns or screenshots between members. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Use observed coordinates, or positions learned from an earlier observation of the same restored layout; element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Verify the final screen before claiming the intended result.", + "parameters": { + "properties": { + "steps": { + "description": "Ordered click, wait and drag members.", + "items": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "action": { + "const": "click", + "title": "Action", + "type": "string" + }, + "target": { + "items": { + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "title": "Target", + "type": "array" + }, + "target_description": { + "minLength": 1, + "title": "Target Description", + "type": "string" + }, + "coordinate_space": { + "default": "normalized", + "enum": [ + "normalized", + "pixel" + ], + "title": "Coordinate Space", + "type": "string" + } + }, + "required": [ + "action", + "target", + "target_description" + ], + "title": "ClickStep", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "start": { + "items": { + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "title": "Start", + "type": "array" + }, + "target_description": { + "minLength": 1, + "title": "Target Description", + "type": "string" + }, + "end": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Exact endpoint; MUST be null when calibration is supplied.", + "title": "End" + }, + "coordinate_space": { + "default": "normalized", + "enum": [ + "normalized", + "pixel" + ], + "title": "Coordinate Space", + "type": "string" + }, + "duration_ms": { + "default": 300, + "maximum": 10000, + "minimum": 1, + "title": "Duration Ms", + "type": "integer" + }, + "calibration": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "axis": { + "enum": [ + "x", + "y" + ], + "title": "Axis", + "type": "string" + }, + "mode": { + "description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.", + "enum": [ + "thumb", + "pan" + ], + "title": "Mode", + "type": "string" + }, + "anchors": { + "items": { + "additionalProperties": false, + "properties": { + "position": { + "description": "Observed axis coordinate in the selected coordinate space.", + "title": "Position", + "type": "number" + }, + "value": { + "description": "Observed scale value, in one consistent numeric unit.", + "title": "Value", + "type": "number" + } + }, + "required": [ + "position", + "value" + ], + "title": "ScaleAnchor", + "type": "object" + }, + "maxItems": 2, + "minItems": 2, + "title": "Anchors", + "type": "array" + }, + "target_value": { + "title": "Target Value", + "type": "number" + }, + "indicator_position": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Required for pan: the fixed indicator's observed axis position.", + "title": "Indicator Position" + } + }, + "required": [ + "axis", + "mode", + "anchors", + "target_value" + ], + "title": "DragCalibration", + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Computes the endpoint from observed anchors; then set end=null." + }, + "action": { + "const": "drag", + "title": "Action", + "type": "string" + } + }, + "required": [ + "start", + "target_description", + "action" + ], + "title": "DragStep", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "action": { + "const": "wait", + "title": "Action", + "type": "string" + }, + "time_in_ms": { + "maximum": 10000, + "minimum": 0, + "title": "Time In Ms", + "type": "integer" + } + }, + "required": [ + "action", + "time_in_ms" + ], + "title": "WaitStep", + "type": "object" + } + ] + }, + "type": "array" + } + }, + "required": [ + "steps" + ], + "type": "object" + } + } + }, + "drag": { + "type": "function", + "function": { + "name": "drag", + "description": "[ACTION] Precisely drag a local slider or timeline. Use explicit start/end or calibration from TWO observed numeric scale anchors. coordinate_space is normalized (0..1000) or pixel (the current screenshot's actual pixels); pixel avoids quantization on a zoomed scale. Default duration_ms=300: slower is not always more precise and may trigger long-press recognition. Do not overshoot the endpoint automatically. For calibration, axis=x/y; mode=thumb moves the handle to target_value, mode=pan shifts the scale under indicator_position. Use one numeric unit, e.g. seconds since midnight, and two anchors from the CURRENT zoom/layout. Provide exactly one of end or calibration. Returned geometry is a plan/dispatch receipt, not proof of the selected value: compare the post-action scale/value and correct only the observed residual; zoom further if the pixel resolution is insufficient.", + "parameters": { + "properties": { + "start": { + "description": "Observed handle or pan starting point [x, y].", + "items": { + "type": "integer" + }, + "type": "array" + }, + "target_description": { + "description": "The slider, handle or timeline being dragged.", + "type": "string" + }, + "end": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Exact endpoint. Set null (or omit) when calibration is supplied; never compute both." + }, + "coordinate_space": { + "default": "normalized", + "enum": [ + "normalized", + "pixel" + ], + "type": "string" + }, + "duration_ms": { + "default": 300, + "type": "integer" + }, + "calibration": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "axis": { + "enum": [ + "x", + "y" + ], + "title": "Axis", + "type": "string" + }, + "mode": { + "description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.", + "enum": [ + "thumb", + "pan" + ], + "title": "Mode", + "type": "string" + }, + "anchors": { + "items": { + "additionalProperties": false, + "properties": { + "position": { + "description": "Observed axis coordinate in the selected coordinate space.", + "title": "Position", + "type": "number" + }, + "value": { + "description": "Observed scale value, in one consistent numeric unit.", + "title": "Value", + "type": "number" + } + }, + "required": [ + "position", + "value" + ], + "title": "ScaleAnchor", + "type": "object" + }, + "maxItems": 2, + "minItems": 2, + "title": "Anchors", + "type": "array" + }, + "target_value": { + "title": "Target Value", + "type": "number" + }, + "indicator_position": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Required for pan: the fixed indicator's observed axis position.", + "title": "Indicator Position" + } + }, + "required": [ + "axis", + "mode", + "anchors", + "target_value" + ], + "title": "DragCalibration", + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed numeric anchors that compute the endpoint. When supplied, end MUST be null; otherwise set calibration null." + } + }, + "required": [ + "start", + "target_description" + ], + "type": "object" + } + } + }, "input_text": { "type": "function", "function": { @@ -105,7 +475,7 @@ "type": "function", "function": { "name": "swipe", - "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).", + "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Use observed endpoints for local slider/timeline movement. Do not automatically overshoot: slow short gestures can trigger long-press recognition. Compare the actual post-action value/scale before adjusting again. When available, drag supports pixel coordinates and two-anchor calibration for fine positioning.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).", "parameters": { "properties": { "direction": { diff --git a/tests/fixtures/action_surfaces/tool_declarations.json b/tests/fixtures/action_surfaces/tool_declarations.json index 33f5ac8..1984db4 100644 --- a/tests/fixtures/action_surfaces/tool_declarations.json +++ b/tests/fixtures/action_surfaces/tool_declarations.json @@ -161,7 +161,7 @@ "type": "function", "function": { "name": "swipe", - "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).", + "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Use observed endpoints for local slider/timeline movement. Do not automatically overshoot: slow short gestures can trigger long-press recognition. Compare the actual post-action value/scale before adjusting again. When available, drag supports pixel coordinates and two-anchor calibration for fine positioning.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n target_description: What is being dragged, in a few words (e.g. 'brightness slider knob'). REQUIRED for coordinate gestures ('start'/'end' or a coordinates list); ignored for directional scrolling.\n duration: Optional gesture duration in milliseconds (default 800).", "parameters": { "type": "object", "properties": { @@ -310,5 +310,375 @@ ] } } + }, + "DRAG_TOOL": { + "type": "function", + "function": { + "name": "drag", + "description": "[ACTION] Precisely drag a local slider or timeline. Use explicit start/end or calibration from TWO observed numeric scale anchors. coordinate_space is normalized (0..1000) or pixel (the current screenshot's actual pixels); pixel avoids quantization on a zoomed scale. Default duration_ms=300: slower is not always more precise and may trigger long-press recognition. Do not overshoot the endpoint automatically. For calibration, axis=x/y; mode=thumb moves the handle to target_value, mode=pan shifts the scale under indicator_position. Use one numeric unit, e.g. seconds since midnight, and two anchors from the CURRENT zoom/layout. Provide exactly one of end or calibration. Returned geometry is a plan/dispatch receipt, not proof of the selected value: compare the post-action scale/value and correct only the observed residual; zoom further if the pixel resolution is insufficient.", + "parameters": { + "properties": { + "start": { + "description": "Observed handle or pan starting point [x, y].", + "items": { + "type": "integer" + }, + "type": "array" + }, + "target_description": { + "description": "The slider, handle or timeline being dragged.", + "type": "string" + }, + "end": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Exact endpoint. Set null (or omit) when calibration is supplied; never compute both." + }, + "coordinate_space": { + "default": "normalized", + "enum": [ + "normalized", + "pixel" + ], + "type": "string" + }, + "duration_ms": { + "default": 300, + "type": "integer" + }, + "calibration": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "axis": { + "enum": [ + "x", + "y" + ], + "title": "Axis", + "type": "string" + }, + "mode": { + "description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.", + "enum": [ + "thumb", + "pan" + ], + "title": "Mode", + "type": "string" + }, + "anchors": { + "items": { + "additionalProperties": false, + "properties": { + "position": { + "description": "Observed axis coordinate in the selected coordinate space.", + "title": "Position", + "type": "number" + }, + "value": { + "description": "Observed scale value, in one consistent numeric unit.", + "title": "Value", + "type": "number" + } + }, + "required": [ + "position", + "value" + ], + "title": "ScaleAnchor", + "type": "object" + }, + "maxItems": 2, + "minItems": 2, + "title": "Anchors", + "type": "array" + }, + "target_value": { + "title": "Target Value", + "type": "number" + }, + "indicator_position": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Required for pan: the fixed indicator's observed axis position.", + "title": "Indicator Position" + } + }, + "required": [ + "axis", + "mode", + "anchors", + "target_value" + ], + "title": "DragCalibration", + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Observed numeric anchors that compute the endpoint. When supplied, end MUST be null; otherwise set calibration null." + } + }, + "required": [ + "start", + "target_description" + ], + "type": "object" + } + } + }, + "ACTION_SEQUENCE_TOOL": { + "type": "function", + "function": { + "name": "action_sequence", + "description": "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, in order, without model turns or screenshots between members. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Use observed coordinates, or positions learned from an earlier observation of the same restored layout; element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Verify the final screen before claiming the intended result.", + "parameters": { + "properties": { + "steps": { + "description": "Ordered click, wait and drag members.", + "items": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "action": { + "const": "click", + "title": "Action", + "type": "string" + }, + "target": { + "items": { + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "title": "Target", + "type": "array" + }, + "target_description": { + "minLength": 1, + "title": "Target Description", + "type": "string" + }, + "coordinate_space": { + "default": "normalized", + "enum": [ + "normalized", + "pixel" + ], + "title": "Coordinate Space", + "type": "string" + } + }, + "required": [ + "action", + "target", + "target_description" + ], + "title": "ClickStep", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "start": { + "items": { + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "title": "Start", + "type": "array" + }, + "target_description": { + "minLength": 1, + "title": "Target Description", + "type": "string" + }, + "end": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Exact endpoint; MUST be null when calibration is supplied.", + "title": "End" + }, + "coordinate_space": { + "default": "normalized", + "enum": [ + "normalized", + "pixel" + ], + "title": "Coordinate Space", + "type": "string" + }, + "duration_ms": { + "default": 300, + "maximum": 10000, + "minimum": 1, + "title": "Duration Ms", + "type": "integer" + }, + "calibration": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "axis": { + "enum": [ + "x", + "y" + ], + "title": "Axis", + "type": "string" + }, + "mode": { + "description": "thumb moves a handle to a value; pan moves a scale under a fixed indicator.", + "enum": [ + "thumb", + "pan" + ], + "title": "Mode", + "type": "string" + }, + "anchors": { + "items": { + "additionalProperties": false, + "properties": { + "position": { + "description": "Observed axis coordinate in the selected coordinate space.", + "title": "Position", + "type": "number" + }, + "value": { + "description": "Observed scale value, in one consistent numeric unit.", + "title": "Value", + "type": "number" + } + }, + "required": [ + "position", + "value" + ], + "title": "ScaleAnchor", + "type": "object" + }, + "maxItems": 2, + "minItems": 2, + "title": "Anchors", + "type": "array" + }, + "target_value": { + "title": "Target Value", + "type": "number" + }, + "indicator_position": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Required for pan: the fixed indicator's observed axis position.", + "title": "Indicator Position" + } + }, + "required": [ + "axis", + "mode", + "anchors", + "target_value" + ], + "title": "DragCalibration", + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Computes the endpoint from observed anchors; then set end=null." + }, + "action": { + "const": "drag", + "title": "Action", + "type": "string" + } + }, + "required": [ + "start", + "target_description", + "action" + ], + "title": "DragStep", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "action": { + "const": "wait", + "title": "Action", + "type": "string" + }, + "time_in_ms": { + "maximum": 10000, + "minimum": 0, + "title": "Time In Ms", + "type": "integer" + } + }, + "required": [ + "action", + "time_in_ms" + ], + "title": "WaitStep", + "type": "object" + } + ] + }, + "type": "array" + } + }, + "required": [ + "steps" + ], + "type": "object" + } + } } } diff --git a/tests/tools/operator/test_operator_tools.py b/tests/tools/operator/test_operator_tools.py index 8fe4092..aaaf9cf 100644 --- a/tests/tools/operator/test_operator_tools.py +++ b/tests/tools/operator/test_operator_tools.py @@ -25,6 +25,8 @@ async def test_operator_shell_tools(): "click": {"target": 1, "times": 1, "delay_ms": 100}, "input_text": {"text": "hello", "target": 1, "clear_exist": True}, "swipe": {"gesture": "up"}, + "drag": {"start": [100, 200], "end": [100, 220], "target_description": "timeline"}, + "action_sequence": {"steps": [{"action": "wait", "time_in_ms": 50}]}, "press_key": {"key": "ENTER"}, "manage_app": {"action": "launch", "app_name": "Settings"}, "wait_for_delay": {"time_in_ms": 1000}, diff --git a/tests/unit/agents/test_operator_transcript.py b/tests/unit/agents/test_operator_transcript.py index 84bf207..89cf837 100644 --- a/tests/unit/agents/test_operator_transcript.py +++ b/tests/unit/agents/test_operator_transcript.py @@ -54,8 +54,8 @@ from artemis.memory.transcript import ( # SHA-256 snapshots of the legacy system message with the fixed inputs below. # Update these when an intentional template change alters the rendered prompt. -GOLDEN_EMPTY_PLAN = "e83c2c6cc1e28b04898e6602194cb2fbac67fe5a3aec8e109872c375f4e59944" -GOLDEN_SENTINEL_PLAN = "dba4eb2cdcd9389148ac3228c6e9e69fdbe8d407dec76b58aa65509ad768c392" +GOLDEN_EMPTY_PLAN = "23c3d6b5fc3694275e7015f7bd0901bde54aae54fec7a8c68d4f8529c65ef4bc" +GOLDEN_SENTINEL_PLAN = "370c53f1d088a4120c63fbaea6d895f121c683b8d3c8cc7d61105d6e1d223128" SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8") diff --git a/tests/unit/agents/test_prompt_assembly.py b/tests/unit/agents/test_prompt_assembly.py index 78ee3c9..033beb2 100644 --- a/tests/unit/agents/test_prompt_assembly.py +++ b/tests/unit/agents/test_prompt_assembly.py @@ -131,10 +131,12 @@ def test_reduced_enums_stay_well_formed(): available_tools=OPERATOR_PROMPT_TOOLSET - {"manage_app", "wait_for_delay"}, ) assert ( - "Physical device actions (`click`, `input_text`, `swipe`, `press_key`, `long_press`)" in out + "Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`, `press_key`, `long_press`)" + in out ) assert ( - "Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`, or `press_key`)" in out + "Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`, `long_press`, or `press_key`)" + in out ) @@ -143,11 +145,11 @@ def test_full_set_enum_slots_render_verbatim(): template = load_operator_prompts()["main_template"] out = apply_operator_prompt_contract(template) assert ( - "Physical device actions (`click`, `input_text`, `swipe`, `press_key`," + "Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`, `press_key`," " `manage_app`, `wait_for_delay`, `long_press`)" in out ) assert ( - "Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`," + "Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`, `long_press`," " `press_key`, `manage_app`, or `wait_for_delay`)" in out ) assert "Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`)" in out diff --git a/tests/unit/mcp/test_action_specs.py b/tests/unit/mcp/test_action_specs.py index d89c277..a9570f6 100644 --- a/tests/unit/mcp/test_action_specs.py +++ b/tests/unit/mcp/test_action_specs.py @@ -83,6 +83,8 @@ def test_tool_declarations_match_fixture(): "LONG_PRESS_TOOL": "long_press", "INPUT_TEXT_TOOL": "input_text", "SWIPE_TOOL": "swipe", + "DRAG_TOOL": "drag", + "ACTION_SEQUENCE_TOOL": "action_sequence", "PRESS_KEY_TOOL": "press_key", "MANAGE_APP_TOOL": "manage_app", "WAIT_FOR_DELAY_TOOL": "wait_for_delay", @@ -121,6 +123,8 @@ def test_validator_declaration_order_is_stable(): "long_press", "input_text", "swipe", + "drag", + "action_sequence", "press_key", "read_note", "list_notes", @@ -159,7 +163,7 @@ def test_flash_declarations_are_the_operator_shells_projected(): decl_props = declaration.parameters["properties"] assert list(decl_props) == list(shell_props), spec.name for param, schema in shell_props.items(): - assert decl_props[param] == _without_optional_noise(schema), ( + assert _without_optional_noise(decl_props[param]) == _without_optional_noise(schema), ( f"'{spec.name}.{param}' diverges between the operator shell and the" " Flash declaration." ) diff --git a/tests/unit/mcp/test_gestures.py b/tests/unit/mcp/test_gestures.py new file mode 100644 index 0000000..479bd13 --- /dev/null +++ b/tests/unit/mcp/test_gestures.py @@ -0,0 +1,400 @@ +"""Behavioral regressions for short-lived controls and measured scale movement.""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from artemis.mcp.action_server import build_action_server +from artemis.mcp.action_session import ActionSession +from artemis.mcp.actuators import MockActuator +from artemis.mcp.gestures import DragRequest, plan_drag + + +def test_pan_uses_decreasing_observed_time_scale_without_overshoot(): + request = DragRequest( + start=[190, 1059], + coordinate_space="pixel", + target_description="timeline", + calibration={ + "axis": "y", + "mode": "pan", + "anchors": [ + {"position": 1075, "value": 38400}, + {"position": 1187, "value": 37800}, + ], + "target_value": 37800, + "indicator_position": 1215, + }, + ) + planned = plan_drag(request, 720, 1600) + assert planned["coordinates"] == [190, 1059, 190, 1087] + assert planned["calibration"]["value_per_pixel"] == pytest.approx(-600 / 112) + assert planned["calibration"]["verification"] == "pending_observation" + + +@pytest.mark.parametrize( + "axis,start,expected", [("x", [80, 200], [125, 200]), ("y", [200, 80], [200, 125])] +) +def test_thumb_maps_value_to_the_requested_axis(axis, start, expected): + planned = plan_drag( + DragRequest( + start=start, + coordinate_space="pixel", + target_description="slider", + calibration={ + "axis": axis, + "mode": "thumb", + "anchors": [ + {"position": 100, "value": 0}, + {"position": 200, "value": 100}, + ], + "target_value": 25, + }, + ), + 720, + 1600, + ) + assert planned["coordinates"] == [*start, *expected] + + +@pytest.mark.asyncio +async def test_pixel_drag_is_not_quantized_through_normalized_coordinates(): + actuator = MockActuator(width=720, height=1600) + session = await ActionSession(build_action_server(actuator)).start() + try: + result = await session.call( + "drag", + { + "start": [191, 1001], + "end": [191, 1032], + "coordinate_space": "pixel", + "target_description": "timeline", + }, + ) + assert result.ok + assert result.metadata["coordinates"] == [191, 1001, 191, 1032] + assert actuator.action_history == [ + { + "action": "swipe", + "start": (191, 1001), + "end": (191, 1032), + "duration_ms": 300, + } + ] + finally: + await session.aclose() + + +def steps(): + return [ + {"action": "click", "target": [200, 900], "target_description": "open scale"}, + {"action": "wait", "time_in_ms": 1}, + { + "action": "drag", + "start": [190, 1001], + "end": [190, 1032], + "coordinate_space": "pixel", + "target_description": "scale handle", + }, + ] + + +@pytest.mark.asyncio +async def test_sequence_has_no_observation_between_inputs_and_returns_member_receipts(): + actuator = MockActuator(width=720, height=1600) + actuator.controller.tap_at = AsyncMock(return_value=MagicMock(error=None)) + actuator.controller.swipe_coords = AsyncMock(return_value=None) + actuator.take_screenshot = AsyncMock() + result = await actuator.action_sequence(steps()) + assert result.ok + actuator.controller.tap_at.assert_awaited_once_with(144, 1440) + actuator.controller.swipe_coords.assert_awaited_once_with(190, 1001, 190, 1032, 300) + actuator.take_screenshot.assert_not_awaited() + receipts = result.metadata["sequence_results"] + assert [r["execution"] for r in receipts] == ["completed"] * 3 + assert receipts[0]["ended_ms"] <= receipts[1]["started_ms"] <= receipts[2]["started_ms"] + + +@pytest.mark.asyncio +async def test_late_invalid_member_prevents_even_the_first_input(): + actuator = MockActuator(width=720, height=1600) + actuator.controller.tap_at = AsyncMock() + invalid = steps() + invalid[-1]["end"] = [720, 1032] + result = await actuator.action_sequence(invalid) + assert not result.ok and result.code.value == "INVALID_ARGS" + actuator.controller.tap_at.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_failure_keeps_completed_members_and_does_not_dispatch_the_remainder(): + actuator = MockActuator(width=720, height=1600) + actuator.controller.tap_at = AsyncMock( + side_effect=[MagicMock(error=None), RuntimeError("transport lost")] + ) + member = {"action": "click", "target": [200, 900], "target_description": "control"} + result = await actuator.action_sequence([member, member, member]) + assert not result.ok + assert actuator.controller.tap_at.await_count == 2 + assert [r["execution"] for r in result.metadata["sequence_results"]] == [ + "completed", + "unknown", + "not_started", + ] + assert result.metadata["execution"] == "partial" + + +@pytest.mark.asyncio +async def test_cancellation_is_not_swallowed_or_retried(): + actuator = MockActuator(width=720, height=1600) + actuator.ctx.data_engine = MagicMock(current_step_id="step") + actuator.controller.swipe_coords = AsyncMock(side_effect=asyncio.CancelledError) + with pytest.raises(asyncio.CancelledError): + await actuator.action_sequence(steps()) + assert len(actuator.action_history) == 1 + assert actuator.controller.swipe_coords.await_count == 1 + receipts = [ + c.kwargs["payload"]["receipt"] + for c in actuator.ctx.data_engine.record_trace.call_args_list + if "receipt" in c.kwargs["payload"] + ] + assert [r["execution"] for r in receipts] == ["completed", "completed", "unknown"] + + +@pytest.mark.asyncio +async def test_pause_at_member_boundary_does_not_dispatch_later_inputs(tmp_path): + from artemis.runtime.task_control import TaskControl + + actuator = MockActuator(width=720, height=1600) + control = object.__new__(TaskControl) + control.directory = tmp_path + actuator.ctx.task_control = control + + async def tap(*_): + (tmp_path / "pause.request").write_text("{}") + return SimpleNamespace(error=None) + + actuator.controller.tap_at = AsyncMock(side_effect=tap) + actuator.controller.swipe_coords = AsyncMock() + result = await actuator.action_sequence(steps()) + assert not result.ok + assert result.metadata["completed_steps"] == 1 + assert result.metadata["sequence_results"][1]["execution"] == "not_started" + actuator.controller.swipe_coords.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_over_limit_is_rejected_before_dispatch(monkeypatch): + monkeypatch.setattr( + "artemis.config.agent.load_agent_config", + lambda: SimpleNamespace( + pro=SimpleNamespace(execution=SimpleNamespace(max_burst_actions=2)), + ), + ) + actuator = MockActuator(width=720, height=1600) + result = await actuator.action_sequence(steps()) + assert not result.ok and "limit of 2" in result.message + assert actuator.action_history == [] + + +@pytest.mark.parametrize("values", [(0, 0), (1e308, -1e308), (float("nan"), 1), (True, 5)]) +def test_degenerate_and_nonfinite_calibration_is_rejected(values): + with pytest.raises(ValueError): + request = DragRequest( + start=[100, 100], + coordinate_space="pixel", + target_description="timeline", + calibration={ + "axis": "y", + "mode": "pan", + "indicator_position": 100, + "anchors": [ + {"position": 100, "value": values[0]}, + {"position": 200, "value": values[1]}, + ], + "target_value": 3, + }, + ) + plan_drag(request, 720, 1600) + + +def test_flash_and_pro_schemas_preserve_null_for_calculated_endpoints(): + from langchain_core.utils.function_calling import convert_to_openai_tool + from artemis.mcp.action_specs import tool_declaration, operator_shell_tool + + flash = tool_declaration("drag").parameters + pro = convert_to_openai_tool(operator_shell_tool("drag"))["function"]["parameters"] + assert flash == pro + assert {"type": "null"} in flash["properties"]["end"]["anyOf"] + + +@pytest.mark.asyncio +async def test_conflicting_endpoint_has_actionable_error_and_no_input(): + actuator = MockActuator(width=720, height=1600) + result = await actuator.drag( + start=[190, 1200], + end=[190, 1100], + coordinate_space="pixel", + target_description="timeline", + calibration={ + "axis": "y", + "mode": "pan", + "anchors": [{"position": 1000, "value": 60}, {"position": 1100, "value": 0}], + "target_value": 30, + "indicator_position": 1050, + }, + ) + assert not result.ok and "set end=null" in result.message + assert actuator.action_history == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["flash", "pro"]) +async def test_each_mode_exposes_the_receipt_to_its_next_model_turn(mode): + from artemis.agents.validator.action_execution import exec_action + from artemis.mcp.action_executor import McpActionExecutor + from artemis.mcp.action_types import ActionResult, ObserveResult + + ctx = SimpleNamespace( + data_engine=None, device=SimpleNamespace(device_width=720, device_height=1600) + ) + receipt = { + "sequence_results": [{"action": "click", "execution": "completed"}], + "execution": "partial", + } + session = MagicMock() + session.call = AsyncMock( + return_value=ActionResult.failure("action_sequence", "Interrupted", metadata=receipt) + ) + args = {"steps": steps()} + if mode == "pro": + action = {"action": "action_sequence", "args": args} + ok, _ = await exec_action(ctx, session, action) + assert not ok + assert action["execution"] == "partial" + from artemis.utils.task_tree import format_result_clean + + text = format_result_clean({"status": "failed", "execution": [action]}) + else: + executor = McpActionExecutor(ctx, actuator=MagicMock()) + executor._session_or_start = AsyncMock(return_value=session) + executor._observe_after_action = AsyncMock(return_value=(ObserveResult(ok=True), True)) + result = await executor._execute_device_action( + "action_sequence", + "action_sequence", + args, + "gesture", + SimpleNamespace(latest_screenshot=None, indexed_elements=[]), + ) + assert result.status == "error" + executor._observe_after_action.assert_awaited_once() + text = result.text_summary + assert '"execution": "partial"' in text + assert '"execution": "completed"' in text + + +@pytest.mark.asyncio +async def test_pro_translation_and_wire_dispatch_preserve_exact_pixel_drag(): + from artemis.agents.operator.operator import OperatorNode + from artemis.agents.validator.action_execution import exec_action + + actuator = MockActuator(width=720, height=1600) + node = object.__new__(OperatorNode) + node.ctx = actuator.ctx + request = { + "start": [191, 1001], + "end": [191, 1032], + "coordinate_space": "pixel", + "target_description": "timeline", + } + state = SimpleNamespace(operator_raw_data={"width": 720, "height": 1600}) + actions, error = node._translate_and_validate_tool_inner( + {"name": "drag", "args": request}, state + ) + assert error is None + assert actions[0]["coordinates"] == [191, 1001, 191, 1032] + assert actions[0]["normalized_coordinates"] == [265, 626, 265, 645] + session = await ActionSession(build_action_server(actuator)).start() + try: + ok, _ = await exec_action(actuator.ctx, session, actions[0]) + assert ok + assert actuator.action_history[-1]["end"] == (191, 1032) + assert actions[0]["execution"] == "completed" + finally: + await session.aclose() + + +@pytest.mark.parametrize( + "change", + [ + {"start": [True, 10]}, + {"duration_ms": 0}, + {"target_description": " "}, + {"end": None}, + {"coordinate_space": "screen"}, + ], +) +def test_bad_drag_arguments_are_rejected(change): + args = {"start": [10, 20], "end": [20, 30], "target_description": "slider", **change} + with pytest.raises(ValueError): + DragRequest.model_validate(args) + + +def test_pixel_history_under_1000_still_uses_its_actual_screen_geometry(): + from artemis.utils.coordinates import normalize_step_actions + + action = { + "action": "drag", + "coordinates": [191, 900, 191, 920], + "coordinate_space": "pixel", + "screen_size": [720, 1600], + } + step = normalize_step_actions({"action_taken": action}) + assert step["action_taken"]["coordinates"] == [191, 900, 191, 920] + assert step["action_taken"]["normalized_coordinates"] == [265, 562, 265, 575] + + +def test_overlay_uses_executed_pixels_instead_of_original_request_units(): + import io + from PIL import Image + from artemis.utils.visualization import overlay_action_on_screenshot + + buffer = io.BytesIO() + Image.new("RGB", (720, 1600), "white").save(buffer, format="PNG") + action = { + "action": "drag", + "coordinates": [190, 900, 190, 930], + "coordinate_space": "pixel", + "args": {"start": [264, 563], "end": [264, 581], "coordinate_space": "normalized"}, + } + drawn = overlay_action_on_screenshot(buffer.getvalue(), action) + assert drawn is not None + result = Image.open(io.BytesIO(drawn)) + red, green, blue = result.getpixel((190, 915)) + assert red > 150 and green < 120 and blue < 120 + assert min(result.getpixel((137, 1450))) > 240 + + +def test_sequence_overlay_does_not_draw_unexecuted_targets(): + from artemis.utils.visualization import draw_action_overlay_on_image + + raw = b"unused original image" + assert ( + draw_action_overlay_on_image( + raw, + "action_sequence", + { + "sequence_results": [ + { + "action": "click", + "coordinates": [190, 900], + "coordinate_space": "pixel", + "execution": "not_started", + }, + ] + }, + ) + == raw + ) diff --git a/tests/unit/runtime/test_server_process_lifecycle.py b/tests/unit/runtime/test_server_process_lifecycle.py index afa2864..72d3b05 100644 --- a/tests/unit/runtime/test_server_process_lifecycle.py +++ b/tests/unit/runtime/test_server_process_lifecycle.py @@ -121,11 +121,20 @@ def test_stop_server_when_not_running(monkeypatch): """Verify stop_server returns early when no server is running.""" monkeypatch.setattr("artemis.runtime.server_lifecycle.is_port_in_use", lambda port, **k: False) monkeypatch.setattr("artemis.runtime.server_lifecycle.find_server_pids", lambda port: []) + # A simulated offline server must not touch the user's live metadata or leases. + clear_info = MagicMock() + cleanup_locks = MagicMock(return_value=0) + monkeypatch.setattr("artemis.runtime.server_lifecycle.clear_server_info", clear_info) + monkeypatch.setattr( + "artemis.runtime.server_lifecycle.DeviceExecutionLock.cleanup_stale_locks", cleanup_locks + ) success, msg, stopped = stop_server(8000) assert success is True assert stopped == [] assert "No active server" in msg + clear_info.assert_called_once_with(port=8000) + cleanup_locks.assert_called_once_with() def test_server_main_parses_non_default_bind_address(monkeypatch):