From 4bc79fdec8178b8939a766ebfa0002c3890aa468 Mon Sep 17 00:00:00 2001 From: somew1nd Date: Thu, 3 Sep 2026 10:10:22 -0700 Subject: [PATCH] feat: improve operator progress and recording accuracy --- .../agent-stream/agent-stream.component.scss | 55 +++- .../src/app/pages/home/home.component.scss | 14 + artemis/agents/operator/operator.json | 2 +- artemis/agents/operator/operator.py | 94 +++++- artemis/agents/operator/prompts.py | 88 ++++-- artemis/config/agent.py | 15 + artemis/controllers/unified_controller.py | 84 ++++-- artemis/utils/plan_grammar.py | 41 ++- artemis/utils/task_tree.py | 12 +- artemis/utils/video.py | 43 +++ tests/unit/agents/test_operator.py | 134 --------- .../unit/agents/test_operator_plan_ledger.py | 277 ++++++++++++++++++ tests/unit/agents/test_operator_transcript.py | 9 +- tests/unit/test_unified_controller_video.py | 127 ++++++++ tests/unit/utils/test_task_tree.py | 2 +- 15 files changed, 773 insertions(+), 224 deletions(-) create mode 100644 tests/unit/agents/test_operator_plan_ledger.py diff --git a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss index 9695191..54372bb 100644 --- a/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss +++ b/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss @@ -37,14 +37,13 @@ $stream-row-height: 25px; background: #f8f9fa; box-sizing: border-box; position: relative; + container-type: inline-size; + container-name: stream-panel; - - - /* Light VS Code / IDE Style Top Action Bar */ - /* Light VS Code / IDE Style Top Action Bar */ .vscode-nav-header { position: relative; height: 60px; + flex-shrink: 0; background: #ffffff; border-bottom: 1px solid rgba(0, 0, 0, 0.08); display: flex; @@ -99,17 +98,6 @@ $stream-row-height: 25px; outline: none; white-space: nowrap; - @media (max-width: 1150px) { - padding: 0 9px; - gap: 0; - } - - .tab-label { - @media (max-width: 1150px) { - display: none !important; - } - } - .tab-icon { font-size: 16px; color: inherit; @@ -1032,7 +1020,6 @@ $stream-row-height: 25px; } } - /* Floating Top-Right Action Toolbar (Model Info + Screen Recording Button) */ .stream-floating-toolbar { position: absolute; top: 12px; @@ -3717,3 +3704,39 @@ $stream-row-height: 25px; } } } + +@media (min-width: 1151px) { + @container stream-panel (max-width: 660px) { + .agent-stream-container .stream-floating-toolbar { + .model-info-chip { + padding: 0 9px; + + .model-label { + display: none; + } + } + + .btn-screen-recording { + padding: 0 9px; + gap: 0; + + .btn-text { + display: none; + } + } + } + } +} + +@media (max-width: 1150px) { + @container stream-panel (max-width: 900px) { + .agent-stream-container .vscode-nav-header .vscode-tab-btn { + padding: 0 9px; + gap: 0; + + .tab-label { + display: none; + } + } + } +} diff --git a/apps/showcase_ui/src/app/pages/home/home.component.scss b/apps/showcase_ui/src/app/pages/home/home.component.scss index a6c6921..dbfde67 100644 --- a/apps/showcase_ui/src/app/pages/home/home.component.scss +++ b/apps/showcase_ui/src/app/pages/home/home.component.scss @@ -431,6 +431,18 @@ $text-light: #94a3b8; transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1); outline: none; white-space: nowrap; + min-width: 0; + + @media (max-width: 560px) { + white-space: normal; + height: auto; + min-height: 32px; + padding: 6px 10px; + gap: 5px; + font-size: 12px; + line-height: 1.2; + text-align: center; + } .tab-icon { font-size: 16px; @@ -440,6 +452,7 @@ $text-light: #94a3b8; align-items: center; justify-content: center; color: #64748b; + flex-shrink: 0; transition: color 0.15s ease, transform 0.15s ease; &.icon-ready { @@ -454,6 +467,7 @@ $text-light: #94a3b8; .tab-label { color: #475569; + min-width: 0; transition: color 0.15s ease; } diff --git a/artemis/agents/operator/operator.json b/artemis/agents/operator/operator.json index 25dbcce..d894269 100644 --- a/artemis/agents/operator/operator.json +++ b/artemis/agents/operator/operator.json @@ -1,3 +1,3 @@ { - "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] into note (Exit: ; Interval: )` or `[Loop:continuous] into note (Interval: )` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snapshot into ``; that note is the persistent baseline and diffing ledger you cross-reference and append to on every later cycle.\n - **Step 2 (Pacing & Refresh)**: Adhere strictly to the exact duration declared in ``, then execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view. Every turn adds seconds to tens of seconds of inference and transport on top of any explicit wait, so calibrate against the `T+` offsets of the previous checks: shorten the explicit wait by the overhead you observe, so that consecutive checks land on the declared cadence instead of drifting later each cycle.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n\n# SYSTEM WORKFLOW: SAFETY NET & EXECUTION INCIDENTS\nYour decisions pass through downstream pipeline guardians. Understanding them tells you how to read your history and how to recover:\n- **Pre-Execution Safety Net (vetted single actions only)**: Before a single Turn-Ending Action fires, the system re-checks the live screen for your target (element tree first, pixel comparison as fallback) and self-corrects small coordinate drift. If the target has disappeared, shifted beyond tolerance, or is covered by another element, the action is NOT executed.\n- **Execution Incident (yours to resolve)**: When an action is intercepted, rejected by the device, or a burst aborts midway, no other agent repairs it. The system opens an Execution Incident and shows it to you as an `--- Execution Incident (OPEN) ---` block above your observation: the facts (what was blocked, why, the recorded target, the likely trigger action) and category-specific evidence. The incident stays open, escalating on each consecutive failed turn, until one of your Turn-Ending Actions executes successfully; closing means the execution channel works again, not that the blocked intent was achieved, and a one-turn `--- Execution Incident (CLOSED ...) ---` notice then asks you to settle that intent against the plan. Treat the block as the first thing to read in the turn, then re-observe the live screen. Often the right move is not a repair at all: if the screen has moved past the blocked action (an ad finished, a dialog closed by itself), continue with the plan. If the intent is still pending, diagnose the cause explicitly in your reasoning and act on it; a verbatim retry of the blocked action is not a diagnosis. Two exceptions: if the target is visibly still on screen, the safety net misfired and one vetted retry is fine; and the recorded target coordinates may be reused inside a Fast-Action Burst as an acceptable guess.\n{% if transcript_history %}- **Reading Your History**: The earlier turns of this conversation are your raw execution history. Each committed turn ends with a `--- Action Execution Result (T+mm:ss) ---` message: `Status: success` means your Turn-Ending Action executed; `Status: failed` with an `Error:` line (starting `Intercepted by Pre-Execution Safety Net` or `Execution failed`) is an execution incident, and for a burst it names which member failed and that the rest were not executed. Screenshots older than a few turns are replaced by short visual summaries and old UI element lists are dropped. Compressed spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks: ① a merged synopsis (entry/exit state, verified facts, unresolved items, failed paths, entities), ② segment titles, ③ a per-step action ledger; a recall-only era shows a period paragraph plus a `(Step-level ledger via recall_history for steps a–b)` line. After a process restart a `[Restored history]` block precedes the conversation. Every timestamp is a session-relative `T+mm:ss` offset.\n{% else %}- **History Markers**: In '--- Execution History ---', an intercepted step shows `(Intercepted by Pre-Execution Safety Net)`, a burst shows `[Planned Fast-Action Burst]` with each member's outcome, and a blocked step's `[Result]` line starts with `Error:`. Use them to trace how the current screen state was reached. Step times are session-relative offsets.\n{% endif %}\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Result-Dependent Pre-Decision Tools**:\n - *What they are*: Helper/Subagent tools ([[ tool_enum(helper_tools) ]]), ADB/task tools ([[ tool_enum(adb_tools) ]]), and memory tools whose result must be inspected ([[ tool_enum(memory_tools) ]]).\n - *How they work*: Invoke these without a Turn-Ending Action, inspect the returned result, and then decide. Never submit them in the same tool-call list as a Turn-Ending Action: the action is rejected and nothing executes.\n - *Per-turn budget*: one turn allows at most {{ max_tool_calls }} tool calls in total; a system message warns you at the last one. Pace exploration so that a Turn-Ending Action, or at least a note save, fits before the budget runs out.\n2. **Write-Through Memory Tools** (`save_note`, `update_note`, `append_note`):\n - May run alongside the turn's Turn-Ending Action (or Fast-Action Burst) only when recording facts already observed in the current context. If the note content depends on the action's result, wait for the next observation before writing it. Never write an action's expected outcome as if it had already been observed.\n3. **Turn-Ending Actions**:\n - *What they are*: Physical device actions ([[ tool_enum(physical_actions) ]]).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn. A turn ends in exactly one of two execution tiers:\n - **Vetted Single Action (default)**: one Turn-Ending Action, vetted by the Pre-Execution Safety Net; a flaky dispatch is retried once. Use this for every ordinary step.\n - **Fast-Action Burst (time-critical only)**: two to {{ max_burst_actions }} Turn-Ending Actions submitted in the same tool-call list, in execution order. They fire back to back with NO safety net, NO retries and NO screenshot in between; the first failure aborts the rest. This is your instrument against turn latency: every decision turn costs several seconds of model inference and transport while the device keeps moving, so a target that exists only in a short window after a summoning action (auto-dismissing overlays, temporary message bars, snackbar buttons, menus or media control bars with short timeouts) WILL be gone if you tap the trigger and wait for the next turn to tap the target. Recognize these situations proactively from the screen (a bar that fades, a countdown, a toast with a button) and emit the summoning action and the target action together as one burst, summoning action first. Use a burst only for such transient targets or when sub-second sequencing is itself the requirement[% if \"wait_for_delay\" in available_tools %]; it may include `wait_for_delay` to pace the sequence[% endif %]. Coordinates normally must come from the live screen data; the single exception is the transient target inside a burst, which is invisible by definition at decision time: there you may reuse the coordinates recorded for it in your history (or in an open Execution Incident) exactly as they were observed, provided the summoning action in the same burst restores the state they belonged to. Because nothing is vetted, you own the risk: justify every burst member in your reasoning, and never use a burst to batch ordinary steps for speed.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, proceed through these three phases in sequence.\n\n## Phase 1: Observation & Diagnosis\n1. **Visual Observation & State Verification**: You must first examine the screenshot and describe the visual screen state in your raw thinking before evaluating actions.\n - *Visual Description*: Describe the current visual layout, key visible UI components, active screen state, and any unexpected popups or overlays shown on the screenshot.\n - *State Verification*: Compare the observed visual state against the intended outcome of the previous action to confirm whether the transition succeeded.\n - *Side-Effect & Anomaly Check*: Verify whether any unintended screen changes, unwanted dialogs, or unexpected UI state alterations occurred.\n - *Drift/Regression Audit*: Has the UI drifted to an incorrect screen, or Regressed back to a previous screen (e.g. from an accidental browser back-navigation)?\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen screen (static stuck)?\n2. **Path Decision (Recover vs. Adapt)**: Encourage flexible problem-solving. First, evaluate if the current path can still be recovered or advanced (e.g., dismissing a blocker, retrying, or restoring the view). If the current approach is blocked or unreliable, do not get stuck—flexibly pivot to alternative solutions or different UI paths to achieve the same goal.\n3. **Verify & Diagnose (Recommended Diagnosis Triggers & Loop Exemptions)**: Consider the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.[% if \"recall_history\" in available_tools %] Choose the tool by the question you need answered: `ask_diagnoser` when the question is why the action is not taking effect (logs, video), `recall_history` when the question is what exactly you did or saw earlier.[% endif %]\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and do NOT call for diagnosis, provided you are making progress or adhering to the planned loop pacing.\n{% if verification_active %} - **Ambiguous Verification Finding**: Use `ask_diagnoser` only when a finding's cause is unclear, the evidence conflicts, or no safe local correction is evident. A verification finding alone is not a diagnosis trigger.\n{% endif %} - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n\n## Phase 2: Planning & Memory Preservation\n1. **Autonomous Goal Management & Dynamic Live-Iteration Tracking**: You have autonomy over nested subgoals under the top-level milestones. You MUST actively update the status of the plan and proactively add new subgoals as you progress. Use `[/]` to denote the currently active milestone and subgoal. Only mark a step as completed (`[x]`) AFTER visual confirmation. Keep the plan as a pure Markdown checklist in `task_plan`. Its grammar is machine-enforced:\n{{ plan_grammar }}\n - *Surgical Plan Edits*: Modify `task_plan` with targeted `update_note` (target/replacement) edits. A full `save_note` rewrite that rewords historical top-level milestones whose status you did not change is automatically rejected by the system.\n - *Dynamic Live-Iteration Ledger in Loops*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), dynamically log each concrete iteration item in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle with the session-relative time from the `# CURRENT OBSERVATION [T+mm:ss]` header (e.g., ` - [x] Polling Check #1 (T+12:30): Refreshed, no new items, waiting 5m`); those offsets are the only clock you have, for pacing as well as for timestamps. This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a bounded `[Loop]`, evaluate whether its Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit); once it is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**{% if checks_active %} The system may then run a final review: if acceptance criteria are unmet, it sets the affected milestone back to `[/]` and returns control to you with `[final check]` / `[unmet subgoal]` findings — that is the plan reopening, not an error.{% endif %}\n2. **Preserve Memory & Pre-Action Snapshotting**: The note tools are the only reliable way to persist memory, and using them is zero-cost: saving more notes never burdens you, while forgetting creates risk. Before any UI interaction, perform this risk assessment: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain you will need these details, immediately save the visible critical values to the notes. Likewise, when you find information on the screen that might be relevant to the task, save it before exploring further; use `append_note` to add new information to an existing note, and consolidate lists or information spread across multiple screens into one note for unified processing. Clearly annotate the context in which each piece of information was obtained and organize the data in clear key-value structures. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated.\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data.\n2. **Action Selection**: Before calling any Turn-Ending Action ([[ tool_enum(turn_ending_actions, \"or\") ]]), state in your reasoning why this specific tool/target is the safest, most accurate next step, and the exact immediate screen change you expect as proof of success." + "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] into note (Exit: ; Interval: )` or `[Loop:continuous] into note (Interval: )` (the milestone grammar itself is defined under Task Plan Grammar below), run this 5-step cyclic state machine:\n - **Step 1 (Baseline Sync)**: On the initial cycle, observe all visible entries/metrics in the primary monitoring view and save the full snapshot into ``; that note is the persistent baseline and diffing ledger you cross-reference and append to on every later cycle.\n - **Step 2 (Pacing & Refresh)**: Adhere strictly to the exact duration declared in ``, then execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view. Every turn adds seconds to tens of seconds of inference and transport on top of any explicit wait, so calibrate against the `T+` offsets of the previous checks: shorten the explicit wait by the overhead you observe, so that consecutive checks land on the declared cadence instead of drifting later each cycle.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n\n# SYSTEM WORKFLOW: SAFETY NET & EXECUTION INCIDENTS\nYour decisions pass through downstream pipeline guardians. Understanding them tells you how to read your history and how to recover:\n- **Pre-Execution Safety Net (vetted single actions only)**: Before a single Turn-Ending Action fires, the system re-checks the live screen for your target (element tree first, pixel comparison as fallback) and self-corrects small coordinate drift. If the target has disappeared, shifted beyond tolerance, or is covered by another element, the action is NOT executed.\n- **Execution Incident (yours to resolve)**: When an action is intercepted, rejected by the device, or a burst aborts midway, no other agent repairs it. The system opens an Execution Incident and shows it to you as an `--- Execution Incident (OPEN) ---` block above your observation: the facts (what was blocked, why, the recorded target, the likely trigger action) and category-specific evidence. The incident stays open, escalating on each consecutive failed turn, until one of your Turn-Ending Actions executes successfully; closing means the execution channel works again, not that the blocked intent was achieved, and a one-turn `--- Execution Incident (CLOSED ...) ---` notice then asks you to settle that intent against the plan. Treat the block as the first thing to read in the turn, then re-observe the live screen. Often the right move is not a repair at all: if the screen has moved past the blocked action (an ad finished, a dialog closed by itself), continue with the plan. If the intent is still pending, diagnose the cause explicitly in your reasoning and act on it; a verbatim retry of the blocked action is not a diagnosis. Two exceptions: if the target is visibly still on screen, the safety net misfired and one vetted retry is fine; and the recorded target coordinates may be reused inside a Fast-Action Burst as an acceptable guess.\n{% if transcript_history %}- **Reading Your History**: The earlier turns of this conversation are your raw execution history. Each committed turn ends with a `--- Action Execution Result (T+mm:ss) ---` message: `Status: success` means your Turn-Ending Action executed; `Status: failed` with an `Error:` line (starting `Intercepted by Pre-Execution Safety Net` or `Execution failed`) is an execution incident, and for a burst it names which member failed and that the rest were not executed. Screenshots older than a few turns are replaced by short visual summaries and old UI element lists are dropped. Compressed spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks: ① a merged synopsis (entry/exit state, verified facts, unresolved items, failed paths, entities), ② segment titles, ③ a per-step action ledger; a recall-only era shows a period paragraph plus a `(Step-level ledger via recall_history for steps a–b)` line. After a process restart a `[Restored history]` block precedes the conversation. Every timestamp is a session-relative `T+mm:ss` offset.\n{% else %}- **History Markers**: In '--- Execution History ---', an intercepted step shows `(Intercepted by Pre-Execution Safety Net)`, a burst shows `[Planned Fast-Action Burst]` with each member's outcome, and a blocked step's `[Result]` line starts with `Error:`. Use them to trace how the current screen state was reached. Step times are session-relative offsets.\n{% endif %}\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Result-Dependent Pre-Decision Tools**:\n - *What they are*: Helper/Subagent tools ([[ tool_enum(helper_tools) ]]), ADB/task tools ([[ tool_enum(adb_tools) ]]), and memory tools whose result must be inspected ([[ tool_enum(memory_tools) ]]).\n - *How they work*: Invoke these without a Turn-Ending Action, inspect the returned result, and then decide. Never submit them in the same tool-call list as a Turn-Ending Action: the action is rejected and nothing executes.\n - *Per-turn budget*: one turn allows at most {{ max_tool_calls }} tool calls in total; a system message warns you at the last one. Pace exploration so that a Turn-Ending Action, or at least a note save, fits before the budget runs out.\n2. **Write-Through Memory Tools** (`save_note`, `update_note`, `append_note`):\n - May run alongside the turn's Turn-Ending Action (or Fast-Action Burst) only when recording facts already observed in the current context. If the note content depends on the action's result, wait for the next observation before writing it. Never write an action's expected outcome as if it had already been observed.\n3. **Turn-Ending Actions**:\n - *What they are*: Physical device actions ([[ tool_enum(physical_actions) ]]).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn. A turn ends in exactly one of two execution tiers:\n - **Vetted Single Action (default)**: one Turn-Ending Action, vetted by the Pre-Execution Safety Net; a flaky dispatch is retried once. Use this for every ordinary step.\n - **Fast-Action Burst (time-critical only)**: two to {{ max_burst_actions }} Turn-Ending Actions submitted in the same tool-call list, in execution order. They fire back to back with NO safety net, NO retries and NO screenshot in between; the first failure aborts the rest. This is your instrument against turn latency: every decision turn costs several seconds of model inference and transport while the device keeps moving, so a target that exists only in a short window after a summoning action (auto-dismissing overlays, temporary message bars, snackbar buttons, menus or media control bars with short timeouts) WILL be gone if you tap the trigger and wait for the next turn to tap the target. Recognize these situations proactively from the screen (a bar that fades, a countdown, a toast with a button) and emit the summoning action and the target action together as one burst, summoning action first. Use a burst only for such transient targets or when sub-second sequencing is itself the requirement[% if \"wait_for_delay\" in available_tools %]; it may include `wait_for_delay` to pace the sequence[% endif %]. Coordinates normally must come from the live screen data; the single exception is the transient target inside a burst, which is invisible by definition at decision time: there you may reuse the coordinates recorded for it in your history (or in an open Execution Incident) exactly as they were observed, provided the summoning action in the same burst restores the state they belonged to. Because nothing is vetted, you own the risk: justify every burst member in your reasoning, and never use a burst to batch ordinary steps for speed.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, proceed through these three phases in sequence.\n\n## Phase 1: Observation & Diagnosis\n1. **Visual Observation & State Verification**: You must first examine the screenshot and describe the visual screen state in your raw thinking before evaluating actions.\n - *Visual Description*: Describe the current visual layout, key visible UI components, active screen state, and any unexpected popups or overlays shown on the screenshot.\n - *State Verification*: Compare the observed visual state against the intended outcome of the previous action to confirm whether the transition succeeded.\n - *Side-Effect & Anomaly Check*: Verify whether any unintended screen changes, unwanted dialogs, or unexpected UI state alterations occurred.\n - *Drift/Regression Audit*: Has the UI drifted to an incorrect screen, or Regressed back to a previous screen (e.g. from an accidental browser back-navigation)?\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen screen (static stuck)?\n2. **Path Decision (Recover vs. Adapt)**: Encourage flexible problem-solving. First, evaluate if the current path can still be recovered or advanced (e.g., dismissing a blocker, retrying, or restoring the view). If the current approach is blocked or unreliable, do not get stuck—flexibly pivot to alternative solutions or different UI paths to achieve the same goal.\n3. **Verify & Diagnose (Recommended Diagnosis Triggers & Loop Exemptions)**: Consider the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.[% if \"recall_history\" in available_tools %] Choose the tool by the question you need answered: `ask_diagnoser` when the question is why the action is not taking effect (logs, video), `recall_history` when the question is what exactly you did or saw earlier.[% endif %]\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and do NOT call for diagnosis, provided you are making progress or adhering to the planned loop pacing.\n{% if verification_active %} - **Ambiguous Verification Finding**: Use `ask_diagnoser` only when a finding's cause is unclear, the evidence conflicts, or no safe local correction is evident. A verification finding alone is not a diagnosis trigger.\n{% endif %} - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n\n## Phase 2: Planning & Memory Preservation\n1. **Live Sub-goal Ledger**: `task_plan` has two layers. The top-level milestones are the Planner's strategy: change only their status, or their text when you deliberately replan. Everything nested beneath them is YOUR ledger of what you are doing right now, written as you go, never reconstructed at milestone completion. Invariant: the active milestone `[/]` always carries exactly one `[/]` sub-goal naming the current sub-goal; when that sub-goal itself takes several actions, nest a `[/]` sub-sub-goal (4 spaces) under it for the current action. Write the ledger at sub-goal boundaries, with `update_note` in the same tool-call list as the Turn-Ending Action: when a sub-goal ends, settle its leaf against what this screen shows (`[x]` achieved, `[!]` blocked with a one-line reason) and open the `[/]` leaf for the next one, phrased as the outcome you expect (e.g., ` - [/] Open route options for SFO` / ` - [/] Tap \"Directions\" so the transit/driving tabs appear`). Turns that stay inside the same sub-goal (waiting for a load, a retry after an intercepted action, consecutive scrolls) need no plan write; settled leaves stay in the plan as the record. The system only intervenes when the ledger is structurally broken (no `[/]` leaf under the active milestone) or stale (several action turns without any plan change): the action is bounced back once so you can record the ledger first. Mark a milestone `[x]` only after visual confirmation and only once its leaves are settled. Keep the plan a pure Markdown checklist; its grammar is machine-enforced:\n{{ plan_grammar }}\n - *Surgical Plan Edits*: Modify `task_plan` with targeted `update_note` (target/replacement) edits. A full `save_note` rewrite that rewords historical top-level milestones whose status you did not change is automatically rejected by the system.\n - *Loop Form of the Ledger*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), the sub-goals are the iteration items: log each concrete iteration in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle with the session-relative time from the `# CURRENT OBSERVATION [T+mm:ss]` header (e.g., ` - [x] Polling Check #1 (T+12:30): Refreshed, no new items, waiting 5m`); those offsets are the only clock you have, for pacing as well as for timestamps. This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a bounded `[Loop]`, evaluate whether its Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit); once it is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**{% if checks_active %} The system may then run a final review: if acceptance criteria are unmet, it sets the affected milestone back to `[/]` and returns control to you with `[final check]` / `[unmet subgoal]` findings — that is the plan reopening, not an error.{% endif %}\n2. **Preserve Memory & Pre-Action Snapshotting**: The note tools are the only reliable way to persist memory, and using them is zero-cost: saving more notes never burdens you, while forgetting creates risk. Before any UI interaction, perform this risk assessment: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain you will need these details, immediately save the visible critical values to the notes. Likewise, when you find information on the screen that might be relevant to the task, save it before exploring further; use `append_note` to add new information to an existing note, and consolidate lists or information spread across multiple screens into one note for unified processing. Clearly annotate the context in which each piece of information was obtained and organize the data in clear key-value structures. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated.\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data.\n2. **Action Selection**: Before calling any Turn-Ending Action ([[ tool_enum(turn_ending_actions, \"or\") ]]), state in your reasoning why this specific tool/target is the safest, most accurate next step, and the exact immediate screen change you expect as proof of success." } diff --git a/artemis/agents/operator/operator.py b/artemis/agents/operator/operator.py index 359fc87..379416b 100644 --- a/artemis/agents/operator/operator.py +++ b/artemis/agents/operator/operator.py @@ -46,6 +46,7 @@ from artemis.utils.decorators import wrap_with_callbacks from artemis.utils.element_hit_test import find_element_at_point from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path +from artemis.utils.plan_grammar import parse_plan from artemis.memory.context_policy import build_history_for from artemis.utils.task_tree import get_active_subgoal_hashes from artemis.utils.visualization import format_minimal_list_with_elements @@ -77,7 +78,8 @@ from artemis.agents.operator.prompts import ( ExecutionIncidentPromptComponent, CheckItemsExplainerPromptComponent, BackgroundTasksPromptComponent, - TaskPlanWarningPromptComponent, + render_plan_ledger_bounce, + unwritten_action_streak, ToolLimitWarningPromptComponent, InjectedInstructionPromptComponent, render_transcript_static_system, @@ -120,6 +122,8 @@ class OperatorNode: # Index into the rendered message list where this turn's tail begins; # set by the transcript build and consumed after the tool loop. self._transcript_turn_base: int | None = None + self._turn_plan_baseline: str | None = None + self._turn_ledger_streak: int = 0 def _available_device_actions(self) -> frozenset[str]: """Device actions the installed actuator backend provides. @@ -216,7 +220,6 @@ class OperatorNode: (InjectedInstructionPromptComponent(), {}), (BackgroundTasksPromptComponent(), {}), (FeedbackPromptComponent(), {}), - (TaskPlanWarningPromptComponent(), {}), (ToolLimitWarningPromptComponent(), {}), ] else: @@ -230,6 +233,7 @@ class OperatorNode: kwargs_to_pass = { "prompts": self.prompts, "plan_and_history": plan_and_history, + "task_plan": task_plan, "latest_screenshot_b64": latest_screenshot_b64, "minimal_list": minimal_list, "current_step_num": current_step_num, @@ -378,7 +382,6 @@ class OperatorNode: (InjectedInstructionPromptComponent(), {}), (BackgroundTasksPromptComponent(), {}), (FeedbackPromptComponent(), {}), - (TaskPlanWarningPromptComponent(), {}), (ToolLimitWarningPromptComponent(), {}), ] for component, extra_kwargs in components: @@ -433,6 +436,7 @@ class OperatorNode: raw_thoughts = [] native_thoughts = [] tool_limit_exceeded = False + plan_gate_bounced = False for iteration in range(max_iterations): if iteration == max_iterations - 1: @@ -709,7 +713,25 @@ class OperatorNode: ) actions_list = [] burst_cap = self._max_burst_actions() - if len(action_calls) > burst_cap: + bounce = None + if ( + not plan_gate_bounced + and iteration < max_iterations - 1 + and len(action_calls) == 1 + and not getattr(state, "open_incident", None) + and self._plan_changed_this_turn() is False + ): + bounce = self._plan_ledger_bounce_reason() + if bounce: + plan_gate_bounced = True + validation_errors = True + logger.info(f"Plan ledger gate: bouncing action. {bounce}") + for tc in action_calls: + tool_outputs.append( + ToolMessage(tool_call_id=tc["id"], content=bounce, status="error") + ) + action_calls_to_translate = [] + elif len(action_calls) > burst_cap: # A multi-action turn is a fast-action burst that the Validator # fires without the safety net; cap its length before it runs. validation_errors = True @@ -730,6 +752,11 @@ class OperatorNode: ) action_calls_to_translate = [] else: + if plan_gate_bounced and self._plan_changed_this_turn() is False: + logger.warning( + "Plan ledger gate: executing actions after one bounce without" + " a task_plan change (ledger gap this turn)." + ) action_calls_to_translate = action_calls for tc in action_calls_to_translate: translated_actions, error = self._translate_and_validate_tool(tc, state) @@ -838,6 +865,8 @@ class OperatorNode: # 3. Get history and plan steps, task_plan = self._get_history_and_plan() + self._turn_plan_baseline = self._plan_ledger_baseline(task_plan) + self._turn_ledger_streak = unwritten_action_streak(steps) # 4. Format minimal list @@ -940,6 +969,63 @@ class OperatorNode: "operator_tool_limit_exceeded": tool_limit_exceeded, } + def _plan_ledger_gate_enabled(self) -> bool: + try: + from artemis.config import load_agent_config + + return bool(load_agent_config().pro.execution.plan_ledger_gate) + except (AttributeError, OSError, TypeError, ValueError): + return True + + def _read_task_plan_text(self) -> str | None: + """Current ``task_plan`` note text, or ``None`` when it cannot be read.""" + engine = getattr(self.ctx, "data_engine", None) + if not engine: + return None + try: + path = get_note_file_path(engine.base_dir, "task_plan") + if not path.exists(): + return None + return path.read_text(encoding="utf-8") + except (AttributeError, OSError, TypeError) as e: + logger.warning(f"Plan ledger gate: failed to read task_plan: {e}") + return None + + def _plan_ledger_baseline(self, task_plan: str) -> str | None: + """Return the plan text used as the current turn's baseline.""" + if not self._plan_ledger_gate_enabled() or not getattr(self.ctx, "data_engine", None): + return None + if not parse_plan(task_plan).has_top_level: + return None + return task_plan + + def _plan_ledger_stale_turns(self) -> int: + try: + from artemis.config import load_agent_config + + return int(load_agent_config().pro.execution.plan_ledger_stale_turns) + except (AttributeError, OSError, TypeError, ValueError): + return 4 + + def _plan_ledger_bounce_reason(self) -> str | None: + """Bounce message when the (unchanged) ledger is broken or stale, else ``None``.""" + current = self._read_task_plan_text() + if current is None: + return None + return render_plan_ledger_bounce( + current, self._turn_ledger_streak, self._plan_ledger_stale_turns() + ) + + def _plan_changed_this_turn(self) -> bool | None: + """Whether ``task_plan`` differs from the turn's baseline (``None``: gate inactive).""" + baseline = self._turn_plan_baseline + if baseline is None: + return None + current = self._read_task_plan_text() + if current is None: + return None + return current != baseline + def _get_active_subgoal_hash(self) -> str: """Parses task_plan.md to find the active top-level subgoal hash.""" diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index 13aa0af..66c30f9 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -652,41 +652,69 @@ class BackgroundTasksPromptComponent(PromptComponent): builder.add_human_content("\n".join(lines) + "\n") -class TaskPlanWarningPromptComponent(PromptComponent): - """Suggests a task_plan update after an action turn that did not touch it. +_PLAN_NOTE_TOOLS = ("update_note", "save_note", "append_note") - Only the most recent step is inspected: if it executed a Turn-Ending - Action and no note tool wrote to ``task_plan``, a soft suggestion is - injected into the next turn. Pure diagnosis / waiting / observation turns - legitimately leave the plan alone and get no reminder. - """ - NOTE_TOOLS = ("update_note", "save_note", "append_note") +def step_updated_task_plan(step: dict) -> bool: + """Whether a recorded step's tool calls wrote the ``task_plan`` note.""" + for tc in step.get("tool_calls", []): + if tc.get("name") not in _PLAN_NOTE_TOOLS: + continue + args = tc.get("args", {}) + note_key = args.get("key") or args.get("name") + if note_key == "task_plan": + return True + return False - @classmethod - def _step_updated_task_plan(cls, step: dict) -> bool: - for tc in step.get("tool_calls", []): - if tc.get("name") not in cls.NOTE_TOOLS: - continue - args = tc.get("args", {}) - note_key = args.get("key") or args.get("name") - if note_key == "task_plan": - return True - return False - async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisContext, **kwargs): - steps = kwargs.get("steps", []) - if not steps: - return - last = steps[-1] - if not last.get("action_taken") or self._step_updated_task_plan(last): - return - builder.add_human_content( - "\nReminder: your last turn executed a Turn-Ending Action without" - " updating task_plan. If progress was made, record it now with" - " surgical `update_note` edits: mark what completed and add the" - " subtasks you are now pursuing." +def unwritten_action_streak(steps: list[dict]) -> int: + """Count trailing action steps since the last ``task_plan`` write.""" + streak = 0 + for step in reversed(steps or []): + if step_updated_task_plan(step): + break + if step.get("action_taken"): + streak += 1 + return streak + + +def _clip(text: str, limit: int = 80) -> str: + text = " ".join(text.split()) + return text if len(text) <= limit else text[: limit - 1] + "\u2026" + + +_LEDGER_BOUNCE_TAIL = ( + " with `update_note` (key `task_plan`), then re-issue the action in the same" + " tool-call list (Live Sub-goal Ledger)." +) + + +def render_plan_ledger_bounce(task_plan: str, streak: int, stale_turns: int) -> str | None: + """Return a plan-ledger validation error, if any.""" + snapshot = parse_plan(task_plan or "") + if not snapshot.has_top_level or snapshot.all_top_level_done: + return None + milestone = snapshot.active_milestone() + if milestone is None: + return ( + "Not executed: no milestone is marked in progress. Mark the one you are" + " executing `[/]` and open its `[/]` sub-goal" + _LEDGER_BOUNCE_TAIL ) + leaf = snapshot.active_leaf(milestone) + if leaf is None: + return ( + f'Not executed: the active milestone "{_clip(milestone.text)}" has no' + " in-progress `[/]` sub-goal beneath it. Open one for what you are doing now" + + _LEDGER_BOUNCE_TAIL + ) + turns = streak + 1 + if stale_turns > 0 and turns >= stale_turns: + return ( + f'Not executed: the sub-goal "{_clip(leaf.text)}" has stayed in progress for' + f" {turns} action turns without any plan change. Settle it (`[x]` / `[!]`) or" + " record its progress and open the next leaf" + _LEDGER_BOUNCE_TAIL + ) + return None class ToolLimitWarningPromptComponent(PromptComponent): diff --git a/artemis/config/agent.py b/artemis/config/agent.py index 9f81888..027f14b 100644 --- a/artemis/config/agent.py +++ b/artemis/config/agent.py @@ -663,6 +663,21 @@ class ExecutionConfig(BaseModel): " rejected before execution and fed back to the Operator." ), ) + plan_ledger_gate: bool = Field( + default=True, + description=( + "Require active plan sub-goals and periodically prompt the Operator to" + " update stale plan progress." + ), + ) + plan_ledger_stale_turns: int = Field( + default=4, + ge=0, + description=( + "Action turns allowed without a task_plan change before the ledger gate" + " prompts for an update. Set to 0 to disable staleness checks." + ), + ) class ProProfileConfig(BaseModel): diff --git a/artemis/controllers/unified_controller.py b/artemis/controllers/unified_controller.py index a8a6a8b..4dd03d0 100644 --- a/artemis/controllers/unified_controller.py +++ b/artemis/controllers/unified_controller.py @@ -39,6 +39,7 @@ from artemis.utils.video import ( DEFAULT_MAX_DURATION_SECONDS, RecordingSession, VideoRecordingResult, + await_scrcpy_first_frame, build_scrcpy_record_command, cleanup_video_segments, concatenate_videos, @@ -346,7 +347,8 @@ class UnifiedMobileController: # Strict timeline alignment: # DataEngine session start time is the reference T0 (0.0s). - # Video recording start time is session.start_time. + # session.start_time is the wall-clock time of the first frame in + # the recording (the file's t=0), estimated at recording start. offset = 0.0 if session.data_engine_start_time is not None: offset = max(0.0, session.start_time - session.data_engine_start_time) @@ -517,14 +519,19 @@ class UnifiedMobileController: process = await self._spawn_scrcpy( build_scrcpy_record_command("scrcpy", session.device_id, new_video_path) ) - await asyncio.sleep(0.8) + spawned_at = time.time() + first_frame_at = await await_scrcpy_first_frame(process, spawned_at) if process.returncode is not None: stderr = await process.stderr.read() logger.error(f"Failed to start scrcpy segment: {stderr.decode(errors='replace')}") return False session.process = process session.local_video_path = new_video_path - session.android_segment_started_at = time.time() + session.android_segment_started_at = first_frame_at + logger.info( + f"scrcpy segment {session.android_segment_index} first frame estimated " + f"{first_frame_at - spawned_at:.2f}s after spawn" + ) if display_state: session.android_rotation, session.capture_width, session.capture_height = display_state return True @@ -621,7 +628,52 @@ class UnifiedMobileController: if display_state: session.capture_width, session.capture_height = display_state[1:] - # Persist to local database if Data Engine is active + # Start scrcpy in background + cmd = build_scrcpy_record_command("scrcpy", device_id, local_video_path) + + process = await self._spawn_scrcpy(cmd) + spawned_at = time.time() + + session.process = process + set_active_session(device_id, session) + + logger.info(f"Started scrcpy recording on {device_id}, saving to {local_video_path}") + + # The file's t=0 is the first captured frame, about a second after + # spawn. Anchor the recording timeline there: segment offsets and + # analyzer clip trimming all treat session.start_time as that t=0. + first_frame_at = await await_scrcpy_first_frame(process, spawned_at) + if process.returncode is not None: + stderr = await process.stderr.read() + err_msg = stderr.decode() + logger.error(f"scrcpy failed to start on {device_id}: {err_msg}") + if self.ctx and self.ctx.data_engine: + self.ctx.data_engine.record_video_start( + video_id=video_id, + device_id=device_id, + local_video_path=local_video_path, + start_time=session.start_time, + ) + self._record_recording_failure(session, f"scrcpy failed to start: {err_msg}") + remove_active_session(device_id) + return VideoRecordingResult( + success=False, + message=f"scrcpy failed to start: {err_msg}", + ) + + session.start_time = first_frame_at + session.android_segment_started_at = first_frame_at + timeline_note = "" + if session.data_engine_start_time is not None: + timeline_note = ( + "; recording timeline offset from session start: " + f"{first_frame_at - session.data_engine_start_time:.2f}s" + ) + logger.info( + f"scrcpy first frame estimated {first_frame_at - spawned_at:.2f}s after spawn" + f"{timeline_note}" + ) + if self.ctx and self.ctx.data_engine: self.ctx.data_engine.record_video_start( video_id=video_id, @@ -630,30 +682,6 @@ class UnifiedMobileController: start_time=session.start_time, ) - # Start scrcpy in background - cmd = build_scrcpy_record_command("scrcpy", device_id, local_video_path) - - process = await self._spawn_scrcpy(cmd) - - session.process = process - set_active_session(device_id, session) - - logger.info(f"Started scrcpy recording on {device_id}, saving to {local_video_path}") - - # Check if process failed immediately - await asyncio.sleep(0.8) - if process.returncode is not None: - stderr = await process.stderr.read() - err_msg = stderr.decode() - logger.error(f"scrcpy failed to start on {device_id}: {err_msg}") - self._record_recording_failure(session, f"scrcpy failed to start: {err_msg}") - remove_active_session(device_id) - return VideoRecordingResult( - success=False, - message=f"scrcpy failed to start: {err_msg}", - ) - - # Start background watchdog to auto-recover if scrcpy terminates session.watchdog_task = asyncio.create_task(self._recording_watchdog(device_id)) return VideoRecordingResult( diff --git a/artemis/utils/plan_grammar.py b/artemis/utils/plan_grammar.py index 10bd75a..6117d17 100644 --- a/artemis/utils/plan_grammar.py +++ b/artemis/utils/plan_grammar.py @@ -185,6 +185,45 @@ class PlanSnapshot: parent = candidate return parent + def children_of(self, item: PlanItem) -> tuple[PlanItem, ...]: + """Nested items under a top-level item (up to the next top-level line).""" + if not item.is_top_level: + return () + children: list[PlanItem] = [] + for candidate in self.items: + if candidate.line_no <= item.line_no: + continue + if candidate.is_top_level: + break + children.append(candidate) + return tuple(children) + + def active_milestone(self) -> PlanItem | None: + """The milestone currently being executed. + + An explicitly active top-level item wins; otherwise the parent of the + bottom-most active nested item; otherwise, when every item is still + untouched, the first pending milestone (the same fallback the step + stamping uses). ``None`` when nothing is in progress. + """ + for item in self.top_level: + if item.is_active: + return item + nested_active = self.last_active() + if nested_active is not None: + return self.parent_of(nested_active) + if self.top_level and all(item.is_pending for item in self.top_level): + return self.top_level[0] + return None + + def active_leaf(self, milestone: PlanItem) -> PlanItem | None: + """Bottom-most ``[/]`` nested item under ``milestone`` (the current sub-goal).""" + leaf = None + for child in self.children_of(milestone): + if child.is_active: + leaf = child + return leaf + def check_items_of(self, item: PlanItem) -> tuple[CheckItem, ...]: """Check lines anchored to the given top-level subgoal.""" return tuple(ci for ci in self.check_items if ci.parent_key == item.key) @@ -394,7 +433,7 @@ def unintended_milestone_edits(before: PlanSnapshot, after: PlanSnapshot) -> lis _PLAN_GRAMMAR_BASE = f"""### Task Plan Grammar (machine-enforced contract) Every checklist line MUST match `- [] `: - Status characters: `[{STATUS_PENDING}]` pending, `[{STATUS_ACTIVE}]` in progress, `[{STATUS_DONE}]` completed, `[{STATUS_BLOCKED}]` blocked. -- Top-level lines (no indentation) are strategic milestones; their sub-tasks are indented by 2 spaces. All other text in the note is free-form context. +- Top-level lines (no indentation) are strategic milestones. Nested lines are their sub-goals: 2 spaces for a sub-goal, 4 spaces for a sub-sub-goal beneath it (deeper nesting is allowed). Only zero-indent lines count as milestones for termination and validation; nested lines are the executor's live ledger. All other text in the note is free-form context. - `{LOOP_TAG}` tags a BOUNDED iterative milestone: declare its exit boundary (e.g., `(Exit: ; Interval: )`) and mark it `[{STATUS_DONE}]` only once that exit condition is verifiably met. - `{CONTINUOUS_LOOP_TAG}` tags an UNBOUNDED continuous-monitoring milestone: it must stay `[{STATUS_ACTIVE}]` and can never be marked `[{STATUS_DONE}]`, deleted, or untagged by you — the system mechanically rejects such edits. Only an explicit external stop signal injected by the user unlocks its completion; that signal is an external interruption delivered by the system, never something inferred from the screen or the plan. - The task terminates only when every top-level milestone is `[{STATUS_DONE}]`.""" diff --git a/artemis/utils/task_tree.py b/artemis/utils/task_tree.py index b8b14f2..59634d8 100644 --- a/artemis/utils/task_tree.py +++ b/artemis/utils/task_tree.py @@ -825,11 +825,13 @@ def build_plan_and_history( def get_active_subgoal_hashes(task_plan: str) -> tuple[str, str | None]: - """Parses task_plan content to find the active top-level subgoal hash. + """Parses task_plan content to find the active (top-level, nested) subgoal hashes. - Consolidates all active subtasks under their parent Level 1 subgoal, - returning (parent_hash, None). Falls back to the first pending subgoal if no - active subgoal is found and all are pending. + Returns ``(parent_hash, sub_hash)``: the parent is the active top-level + milestone (all active nested items consolidate under it); the sub hash is + the bottom-most active nested item — the live sub-goal ledger leaf — or + ``None`` when the milestone has no in-progress sub-goal. Falls back to the + first pending subgoal if no active subgoal is found and all are pending. """ if not task_plan: return "default", None @@ -843,7 +845,7 @@ def get_active_subgoal_hashes(task_plan: str) -> tuple[str, str | None]: if not active_item.is_top_level: parent = snapshot.parent_of(active_item) if parent is not None: - return parent.key, None + return parent.key, active_item.key return active_item.key, None # No active subgoal found. Check if all are completed. diff --git a/artemis/utils/video.py b/artemis/utils/video.py index 72dc484..ccec2dc 100644 --- a/artemis/utils/video.py +++ b/artemis/utils/video.py @@ -24,6 +24,7 @@ from pathlib import Path import platform import re import shutil +import time import subprocess from typing import Any from uuid import UUID @@ -37,6 +38,11 @@ logger = get_logger(__name__) DEFAULT_MAX_DURATION_SECONDS = 900 # 15 minutes ANDROID_RECORDING_SEGMENT_SECONDS = 1800 +# The recording marker closely tracks the first frame; startup measurements +# put the fallback near three quarters of a second after process creation. +SCRCPY_RECORDING_STARTED_MARKER = "Recording started" +SCRCPY_STARTUP_FALLBACK_SECONDS = 0.75 +SCRCPY_STARTUP_TIMEOUT_SECONDS = 6.0 VIDEO_READY_DELAY_SECONDS = 1 ANDROID_DEVICE_VIDEO_PATH = "/sdcard/screen_recording.mp4" ANDROID_MAX_RECORDING_DURATION_SECONDS = 180 # Android screenrecord limit @@ -78,6 +84,43 @@ def build_scrcpy_record_command( return command +async def await_scrcpy_first_frame( + process: Any, + spawned_at: float, + *, + timeout: float = SCRCPY_STARTUP_TIMEOUT_SECONDS, +) -> float: + """Estimate when the first frame of a scrcpy recording reached the host.""" + fallback = spawned_at + SCRCPY_STARTUP_FALLBACK_SECONDS + reader = getattr(process, "stdout", None) + if not isinstance(reader, asyncio.StreamReader): + await asyncio.sleep(0.8) + return fallback + + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + logger.warning( + f"scrcpy did not report '{SCRCPY_RECORDING_STARTED_MARKER}' within " + f"{timeout:.1f}s; assuming the first frame at spawn + " + f"{SCRCPY_STARTUP_FALLBACK_SECONDS:.2f}s" + ) + return fallback + try: + line = await asyncio.wait_for(reader.readline(), timeout=remaining) + except TimeoutError: + continue + if not line: + try: + await asyncio.wait_for(process.wait(), timeout=1.0) + except (AttributeError, ProcessLookupError, TimeoutError): + pass + return fallback + if SCRCPY_RECORDING_STARTED_MARKER in line.decode(errors="replace"): + return max(spawned_at, time.time()) + + class _CyFunctionDetectorMeta(type): def __instancecheck__(self, instance): name = type(instance).__name__ diff --git a/tests/unit/agents/test_operator.py b/tests/unit/agents/test_operator.py index 8536187..1104f44 100644 --- a/tests/unit/agents/test_operator.py +++ b/tests/unit/agents/test_operator.py @@ -1188,140 +1188,6 @@ async def test_operator_background_tasks_prompt_injection(): adb_registry.background.pop("task_99999", None) -@pytest.mark.asyncio -async def test_operator_task_plan_warning_prompt_injection(): - from artemis.agents.operator.operator import OperatorNode - from artemis.context import ArtemisContext - from unittest.mock import AsyncMock, MagicMock, patch - - mock_ctx = MagicMock(spec=ArtemisContext) - mock_ctx.execution_setup = None - mock_ctx.data_engine = None - - mock_state = MagicMock() - mock_state.subagent_calls = [] - mock_state.initial_goal = "Test goal" - - mock_controller = MagicMock() - mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=") - mock_device_data = MagicMock() - mock_device_data.elements = [] - mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data) - - mock_llm = MagicMock() - mock_response = MagicMock() - mock_response.tool_calls = [] - - mock_llm.ainvoke = AsyncMock(return_value=mock_response) - mock_llm.bind_tools.return_value = mock_llm - - # Case 1: task_plan updated in the last step (using the "key" argument) - steps_updated = [ - { - "step_id": "step_1", - "step_number": 1, - "tool_calls": [{"name": "click", "args": {"target": 1}}], - }, - { - "step_id": "step_2", - "step_number": 2, - "tool_calls": [ - { - "name": "update_note", - "args": {"key": "task_plan", "target": "a", "replacement": "b"}, - } - ], - }, - ] - - captured_messages_1 = [] - - async def mock_invoke_llm_loop_1(*args, **kwargs): - messages = await node._build_prompt( - state=mock_state, - latest_screenshot_b64="YmFzZTY0X3NjcmVlbnNob3Q=", - fused_xml=[], - minimal_list="", - current_step_num=3, - steps=steps_updated, - task_plan="", - ) - captured_messages_1.extend(messages) - return [], None, None, False - - with ( - patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm), - patch( - "artemis.agents.operator.operator.OperatorNode._invoke_llm_loop", - side_effect=mock_invoke_llm_loop_1, - ), - ): - node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) - node_update = await node(mock_state) - - full_content_1 = "" - for m in captured_messages_1: - if isinstance(m.content, list): - for part in m.content: - if isinstance(part, dict) and "text" in part: - full_content_1 += part["text"] + "\n" - elif isinstance(m.content, str): - full_content_1 += m.content + "\n" - - assert "Reminder: your last turn executed a Turn-Ending Action" not in full_content_1 - - # Case 2: an action turn that did NOT update task_plan - steps_not_updated = [ - { - "step_id": "step_1", - "step_number": 1, - "tool_calls": [{"name": "click", "args": {"target": 1}}], - }, - { - "step_id": "step_2", - "step_number": 2, - "tool_calls": [{"name": "click", "args": {"target": 2}}], - "action_taken": {"action": "click", "target": 2}, - }, - ] - - captured_messages_2 = [] - - async def mock_invoke_llm_loop_2(*args, **kwargs): - messages = await node._build_prompt( - state=mock_state, - latest_screenshot_b64="YmFzZTY0X3NjcmVlbnNob3Q=", - fused_xml=[], - minimal_list="", - current_step_num=3, - steps=steps_not_updated, - task_plan="", - ) - captured_messages_2.extend(messages) - return [], None, None, False - - with ( - patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm), - patch( - "artemis.agents.operator.operator.OperatorNode._invoke_llm_loop", - side_effect=mock_invoke_llm_loop_2, - ), - ): - node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) - node_update = await node(mock_state) - - full_content_2 = "" - for m in captured_messages_2: - if isinstance(m.content, list): - for part in m.content: - if isinstance(part, dict) and "text" in part: - full_content_2 += part["text"] + "\n" - elif isinstance(m.content, str): - full_content_2 += m.content + "\n" - - assert "Reminder: your last turn executed a Turn-Ending Action" in full_content_2 - - @pytest.mark.asyncio async def test_operator_tool_limit_exceeded_warning(): from artemis.agents.operator.operator import OperatorNode diff --git a/tests/unit/agents/test_operator_plan_ledger.py b/tests/unit/agents/test_operator_plan_ledger.py new file mode 100644 index 0000000..acf1e79 --- /dev/null +++ b/tests/unit/agents/test_operator_plan_ledger.py @@ -0,0 +1,277 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for operator plan-ledger handling.""" + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from artemis.agents.operator.operator import OperatorNode +from artemis.agents.operator.prompts import ( + apply_operator_prompt_contract, + load_operator_prompts, + render_plan_ledger_bounce, + unwritten_action_streak, +) +from artemis.config.agent import MemoryTranscriptConfig +from artemis.context import ArtemisContext +from artemis.utils.plan_grammar import parse_plan, render_plan_grammar_spec + +LEGACY_TRANSCRIPT = MemoryTranscriptConfig(enabled=False) + +PLAN_NO_LEAF = """- [x] Open Google Maps and search for SFO +- [/] Read the commute duration and record it into note `commute_eta_info` + - verify: duration recorded +- [ ] Draft the ETA message +""" + +PLAN_WITH_LEAF = """- [x] Open Google Maps and search for SFO +- [/] Read the commute duration and record it into note `commute_eta_info` + - [x] Open route options for SFO + - [/] Read the driving duration + - [/] Tap the "Driving" tab so the fastest route is highlighted + - verify: duration recorded +- [ ] Draft the ETA message +""" + + +def test_grammar_spec_teaches_deeper_nesting(): + spec = render_plan_grammar_spec(include_checks=False) + assert "4 spaces for a sub-sub-goal" in spec + assert "Only zero-indent lines count as milestones" in spec + + +def test_active_milestone_and_leaf(): + snap = parse_plan(PLAN_WITH_LEAF) + milestone = snap.active_milestone() + assert milestone is not None and milestone.text.startswith("Read the commute duration") + children = snap.children_of(milestone) + assert [c.text for c in children] == [ + "Open route options for SFO", + "Read the driving duration", + 'Tap the "Driving" tab so the fastest route is highlighted', + ] + leaf = snap.active_leaf(milestone) + assert leaf is not None and leaf.text.startswith('Tap the "Driving" tab') + assert leaf.indent == 4 + + snap = parse_plan(PLAN_NO_LEAF) + milestone = snap.active_milestone() + assert milestone is not None + assert snap.active_leaf(milestone) is None + assert snap.children_of(milestone) == () + + +def test_active_milestone_fallbacks(): + all_pending = parse_plan("- [ ] First\n- [ ] Second\n") + assert all_pending.active_milestone().text == "First" + nested_only = parse_plan("- [ ] First\n - [/] Doing it\n- [ ] Second\n") + assert nested_only.active_milestone().text == "First" + assert parse_plan("- [x] First\n- [x] Second\n").active_milestone() is None + + +def _action_step(n, wrote_plan=False, action=True): + tool_calls = [] + if wrote_plan: + tool_calls.append( + {"name": "update_note", "args": {"key": "task_plan", "target": "a", "replacement": "b"}} + ) + step = {"step_id": f"s{n}", "step_number": n, "tool_calls": tool_calls} + if action: + step["action_taken"] = [{"action": "tap"}] + return step + + +def test_streak_counts_trailing_action_turns_without_plan_write(): + assert unwritten_action_streak([]) == 0 + assert unwritten_action_streak([_action_step(1, wrote_plan=True)]) == 0 + assert unwritten_action_streak([_action_step(1)]) == 1 + steps = [ + _action_step(1), + _action_step(2, wrote_plan=True), + _action_step(3), + _action_step(4, action=False), + _action_step(5), + ] + assert unwritten_action_streak(steps) == 2 + + +def test_bounce_only_when_ledger_is_broken_or_stale(): + assert render_plan_ledger_bounce(PLAN_WITH_LEAF, streak=0, stale_turns=4) is None + assert render_plan_ledger_bounce(PLAN_WITH_LEAF, streak=2, stale_turns=4) is None + stale = render_plan_ledger_bounce(PLAN_WITH_LEAF, streak=3, stale_turns=4) + assert stale and "stayed in progress for 4 action turns" in stale + assert 'Tap the "Driving" tab' in stale + assert render_plan_ledger_bounce(PLAN_WITH_LEAF, streak=9, stale_turns=0) is None + + broken = render_plan_ledger_bounce(PLAN_NO_LEAF, streak=0, stale_turns=4) + assert broken and "has no in-progress `[/]` sub-goal beneath it" in broken + assert "Read the commute duration" in broken + + assert render_plan_ledger_bounce("No task plan yet.", 0, 4) is None + assert render_plan_ledger_bounce("- [x] Done\n- [x] Also done\n", 0, 4) is None + + +def test_prompt_teaches_the_ledger_once_without_per_turn_reminder(): + prompt = apply_operator_prompt_contract(load_operator_prompts()["main_template"]) + assert prompt.count("**Live Sub-goal Ledger**") == 1 + assert "mandatory every turn" not in prompt + assert "need no plan write" in prompt + assert "sub-sub-goal (4 spaces)" in prompt + assert "structurally broken" in prompt + + +def _make_node(tmp_path, plan_text, steps=None): + notes = tmp_path / "notes" + notes.mkdir(parents=True) + (notes / "task_plan.md").write_text(plan_text, encoding="utf-8") + + ctx = MagicMock(spec=ArtemisContext) + ctx.execution_setup = None + ctx.data_engine = MagicMock() + ctx.data_engine.base_dir = str(tmp_path) + ctx.data_engine.current_session_id = "s" + ctx.data_engine.get_agent_friendly_steps.return_value = steps or [] + + state = MagicMock() + state.subagent_calls = [] + state.initial_goal = "goal" + state.open_incident = None + node = OperatorNode(ctx, transcript_config=LEGACY_TRANSCRIPT) + return node, state, notes / "task_plan.md" + + +def _click_response(n_actions=1): + response = MagicMock() + response.content = "" + response.tool_calls = [ + {"name": "click", "args": {"target": [50, 50]}, "id": f"call_{i}"} for i in range(n_actions) + ] + return response + + +def _llm(side_effect=None, return_value=None): + llm = MagicMock() + llm.ainvoke = AsyncMock(side_effect=side_effect, return_value=return_value) + llm.bind_tools.return_value = llm + return llm + + +@pytest.mark.asyncio +async def test_gate_bounces_broken_ledger_once_then_executes(tmp_path): + node, state, _ = _make_node(tmp_path, PLAN_NO_LEAF) + seen = [] + + async def ainvoke(messages, *args, **kwargs): + seen.append(list(messages)) + return _click_response() + + with patch("artemis.agents.operator.operator.get_llm", return_value=_llm(ainvoke)): + update = await node(state) + + assert len(seen) == 2, "first submission bounced, second executed" + bounce = [ + m + for m in seen[1] + if isinstance(getattr(m, "content", None), str) + and m.content.startswith("Not executed: the active milestone") + ] + assert bounce and bounce[0].status == "error" + assert json.loads(update["structured_decisions"])[0]["action"] == "tap" + + +@pytest.mark.asyncio +async def test_gate_lets_healthy_ledger_through_without_a_write(tmp_path): + node, state, _ = _make_node(tmp_path, PLAN_WITH_LEAF, steps=[_action_step(1)]) + llm = _llm(return_value=_click_response()) + with patch("artemis.agents.operator.operator.get_llm", return_value=llm): + update = await node(state) + assert llm.ainvoke.await_count == 1 + assert update["structured_decisions"] is not None + + +@pytest.mark.asyncio +async def test_gate_bounces_stale_ledger(tmp_path): + stale_history = [_action_step(1), _action_step(2), _action_step(3)] + node, state, _ = _make_node(tmp_path, PLAN_WITH_LEAF, steps=stale_history) + seen = [] + + async def ainvoke(messages, *args, **kwargs): + seen.append(list(messages)) + return _click_response() + + with ( + patch("artemis.agents.operator.operator.get_llm", return_value=_llm(ainvoke)), + patch.object(OperatorNode, "_plan_ledger_stale_turns", return_value=4), + ): + await node(state) + assert len(seen) == 2 + assert any( + isinstance(getattr(m, "content", None), str) + and "stayed in progress for 4 action turns" in m.content + for m in seen[1] + ) + + +@pytest.mark.asyncio +async def test_gate_passes_when_plan_changed_in_turn(tmp_path): + node, state, plan_path = _make_node(tmp_path, PLAN_NO_LEAF) + + async def ainvoke(messages, *args, **kwargs): + plan_path.write_text(PLAN_WITH_LEAF, encoding="utf-8") + return _click_response() + + llm = _llm(ainvoke) + with patch("artemis.agents.operator.operator.get_llm", return_value=llm): + update = await node(state) + assert llm.ainvoke.await_count == 1 + assert update["structured_decisions"] is not None + + +@pytest.mark.asyncio +async def test_gate_exempts_bursts_and_open_incidents(tmp_path): + node, state, _ = _make_node(tmp_path / "burst", PLAN_NO_LEAF) + llm = _llm(return_value=_click_response(n_actions=2)) + with patch("artemis.agents.operator.operator.get_llm", return_value=llm): + update = await node(state) + assert llm.ainvoke.await_count == 1 + assert len(json.loads(update["structured_decisions"])) == 2 + + node, state, _ = _make_node(tmp_path / "incident", PLAN_NO_LEAF) + state.open_incident = {"category": "intercepted"} + llm = _llm(return_value=_click_response()) + with patch("artemis.agents.operator.operator.get_llm", return_value=llm): + await node(state) + assert llm.ainvoke.await_count == 1 + + +@pytest.mark.asyncio +async def test_gate_inactive_without_plan_or_when_disabled(tmp_path): + llm = _llm(return_value=_click_response()) + + node, state, _ = _make_node(tmp_path / "a", "No task plan yet.") + with patch("artemis.agents.operator.operator.get_llm", return_value=llm): + await node(state) + assert llm.ainvoke.await_count == 1 + + llm.ainvoke.reset_mock() + node, state, _ = _make_node(tmp_path / "b", PLAN_NO_LEAF) + with ( + patch("artemis.agents.operator.operator.get_llm", return_value=llm), + patch.object(OperatorNode, "_plan_ledger_gate_enabled", return_value=False), + ): + await node(state) + assert llm.ainvoke.await_count == 1 diff --git a/tests/unit/agents/test_operator_transcript.py b/tests/unit/agents/test_operator_transcript.py index ca1e754..2f51cbf 100644 --- a/tests/unit/agents/test_operator_transcript.py +++ b/tests/unit/agents/test_operator_transcript.py @@ -60,10 +60,11 @@ from artemis.memory.transcript import ( # Incident workflow section and the Phase 3 burst-discipline rule; and again # on 2026-09-02 after the prompt consolidation (every rule stated once, the # transcript/legacy history-marker split, the recited tool budget, the -# recommended diagnosis triggers). Any other drift here means the flag-off -# path is no longer byte-identical. -GOLDEN_EMPTY_PLAN = "a20046ff46058fa331fa3b8f2d3ffa4a018be567bf264588b9a29bc64e5b11b8" -GOLDEN_SENTINEL_PLAN = "8f4456524c8a964d1614d26d2fafab809bd5bd3f30334fd40498999e4b27638b" +# recommended diagnosis triggers); and again on 2026-09-03 for the nested +# plan-ledger contract and its validation gate. Any other drift means the +# flag-off path is no longer byte-identical. +GOLDEN_EMPTY_PLAN = "2ed35be503e3872f008ae5a12616af6f90cde65f23b63adbb6da782079ae4e7c" +GOLDEN_SENTINEL_PLAN = "8bd6aa60cf3fad8d21cf5c284a01f586c57d209fcd1d2b4a517a791736f9b1de" SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8") diff --git a/tests/unit/test_unified_controller_video.py b/tests/unit/test_unified_controller_video.py index 9516a91..2bbcc59 100644 --- a/tests/unit/test_unified_controller_video.py +++ b/tests/unit/test_unified_controller_video.py @@ -14,6 +14,7 @@ """Unit tests for UnifiedMobileController video recording and playback features.""" +import asyncio import subprocess import time from pathlib import Path @@ -494,3 +495,129 @@ async def test_unified_controller_timeline_alignment(mock_ctx, tmp_path): assert abs(res.actual_start_relative_time - 5.0) < 0.1 remove_active_session("emulator-5554") + + +class _FakeScrcpyProcess: + """Minimal stand-in for an asyncio subprocess with a real stdout reader.""" + + def __init__(self, lines: list[bytes], *, close: bool = False): + self.stdout = asyncio.StreamReader() + for line in lines: + self.stdout.feed_data(line) + if close: + self.stdout.feed_eof() + self.stderr = MagicMock() + self.stderr.read = AsyncMock(return_value=b"") + self.returncode = None + self.waited = False + + async def wait(self): + self.waited = True + self.returncode = 1 + return self.returncode + + +@pytest.mark.asyncio +async def test_await_scrcpy_first_frame_uses_recording_started_marker(): + from artemis.utils.video import await_scrcpy_first_frame + + proc = _FakeScrcpyProcess( + [ + b"scrcpy 4.1 \n", + b"INFO: ADB device found:\n", + b"INFO: Recording started to matroska file: C:\\rec\\recording.mkv\n", + ] + ) + spawned_at = time.time() - 2.0 + before = time.time() + estimate = await await_scrcpy_first_frame(proc, spawned_at) + after = time.time() + + assert before <= estimate <= after + assert after - before < 0.5 + + +@pytest.mark.asyncio +async def test_await_scrcpy_first_frame_never_precedes_spawn(): + from artemis.utils.video import await_scrcpy_first_frame + + proc = _FakeScrcpyProcess([b"INFO: Recording started to matroska file: x.mkv\n"]) + spawned_at = time.time() + estimate = await await_scrcpy_first_frame(proc, spawned_at) + assert estimate >= spawned_at + + +@pytest.mark.asyncio +async def test_await_scrcpy_first_frame_falls_back_when_stdout_closes(): + from artemis.utils.video import SCRCPY_STARTUP_FALLBACK_SECONDS, await_scrcpy_first_frame + + proc = _FakeScrcpyProcess([b"scrcpy 4.1\n"], close=True) + spawned_at = time.time() + estimate = await await_scrcpy_first_frame(proc, spawned_at) + + assert estimate == pytest.approx(spawned_at + SCRCPY_STARTUP_FALLBACK_SECONDS) + assert proc.waited is True + assert proc.returncode == 1 + + +@pytest.mark.asyncio +async def test_await_scrcpy_first_frame_falls_back_on_timeout(): + from artemis.utils.video import SCRCPY_STARTUP_FALLBACK_SECONDS, await_scrcpy_first_frame + + proc = _FakeScrcpyProcess([b"scrcpy 4.1\n"]) # marker never arrives, stdout stays open + spawned_at = time.time() + estimate = await await_scrcpy_first_frame(proc, spawned_at, timeout=0.1) + assert estimate == pytest.approx(spawned_at + SCRCPY_STARTUP_FALLBACK_SECONDS) + + +@pytest.mark.asyncio +async def test_start_recording_anchors_timeline_to_first_frame(mock_ctx, tmp_path): + controller = UnifiedMobileController(mock_ctx) + remove_active_session("emulator-5554") + + proc = _FakeScrcpyProcess([b"INFO: Recording started to matroska file: x.mkv\n"]) + before = time.time() + with ( + patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)), + patch( + "artemis.controllers.unified_controller.get_android_display_state", + AsyncMock(return_value=(0, 1080, 2424)), + ), + ): + res = await controller.start_video_recording(output_dir=tmp_path) + + try: + assert res.success is True + session = get_active_session("emulator-5554") + assert session is not None + assert session.start_time >= before + assert session.android_segment_started_at == session.start_time + kwargs = mock_ctx.data_engine.record_video_start.call_args.kwargs + assert kwargs["start_time"] == session.start_time + finally: + session = get_active_session("emulator-5554") + if session and session.watchdog_task: + session.watchdog_task.cancel() + remove_active_session("emulator-5554") + + +@pytest.mark.asyncio +async def test_next_segment_anchors_at_first_frame(mock_ctx, tmp_path): + controller = UnifiedMobileController(mock_ctx) + session = RecordingSession( + video_id=uuid4(), + device_id="emulator-5554", + start_time=time.time() - 30.0, + data_engine_start_time=time.time() - 30.0, + local_video_path=tmp_path / "recording.mkv", + ) + proc = _FakeScrcpyProcess([b"INFO: Recording started to matroska file: y.mkv\n"]) + before = time.time() + with patch("asyncio.create_subprocess_exec", AsyncMock(return_value=proc)): + ok = await controller._start_next_recording_segment(session, (1, 2424, 1080)) + + assert ok is True + assert session.android_segment_started_at is not None + assert session.android_segment_started_at >= before + assert session.android_rotation == 1 + assert session.local_video_path == tmp_path / "recording_001.mkv" diff --git a/tests/unit/utils/test_task_tree.py b/tests/unit/utils/test_task_tree.py index e9a7b36..7d5f340 100644 --- a/tests/unit/utils/test_task_tree.py +++ b/tests/unit/utils/test_task_tree.py @@ -146,7 +146,7 @@ def test_get_active_subgoal_hashes_fallback(): parent, sub = get_active_subgoal_hashes(plan_indented) assert parent == h_system - assert sub is None + assert sub == hashlib.md5(b"Scroll down to find System settings").hexdigest() # Scenario 3: Safe Fallback (All pending "[ ]", no active subgoal) plan_fallback = """- [ ] Open Settings app