From cbe8f4f0c67c2c8e1ad3fc53fda9b9ce98fb0a86 Mon Sep 17 00:00:00 2001 From: somew1nd Date: Mon, 31 Aug 2026 20:42:24 -0700 Subject: [PATCH] feat: harden agent verification and runtime contracts Introduce two-gate checker settlement, explicit graph visibility and checkpoints, structured LLM reliability, centralized action specifications, and corresponding tests and design documentation. --- .gitignore | 1 + artemis/agents/checker/checker.json | 9 +- artemis/agents/checker/checker.py | 1020 +++++++----- artemis/agents/diagnoser/diagnoser.py | 36 +- artemis/agents/flash/runner.py | 113 +- .../history_analyzer/history_analyzer.py | 16 +- artemis/agents/log_analyzer/log_analyzer.py | 16 +- .../agents/object_detector/object_detector.py | 59 +- artemis/agents/operator/operator.json | 3 +- artemis/agents/operator/operator.py | 508 +----- artemis/agents/operator/prompts.py | 323 ++-- artemis/agents/outputter/outputter.py | 63 +- artemis/agents/planner/planner.json | 8 +- artemis/agents/planner/planner.py | 65 +- artemis/agents/summarizer/summarizer.py | 32 +- artemis/agents/validator/tool_declarations.py | 237 +-- artemis/agents/validator/validator.py | 50 +- artemis/agents/video_analyzer/reliability.py | 151 +- artemis/config/agent.py | 53 +- artemis/config/settings.py | 10 + artemis/constants.py | 2 - artemis/context.py | 60 +- artemis/core/context.py | 3 + artemis/data_engine/engine.py | 37 +- artemis/engine/graph_runner.py | 21 +- artemis/engine/reactive_runner.py | 20 +- artemis/graph/checkpoints.py | 649 ++++++++ artemis/graph/graph.py | 1079 ++++++------ artemis/graph/perception.py | 36 +- artemis/graph/state.py | 142 +- artemis/graph/visibility.py | 227 +++ artemis/llm/reliability.py | 252 +++ artemis/llm/structured.py | 194 +++ artemis/mcp/action_executor.py | 61 +- artemis/mcp/action_manifest.py | 4 +- artemis/mcp/action_server.py | 203 +-- artemis/mcp/action_specs.py | 961 +++++++++++ artemis/sdk/agent.py | 178 +- artemis/sdk/builders/agent_config_builder.py | 78 +- artemis/sdk/types/agent.py | 10 +- artemis/services/llm.py | 648 ++++++-- artemis/tools/actions/__init__.py | 45 - artemis/tools/actions/device_actions.py | 250 --- artemis/tools/committee_tool.py | 7 - artemis/tools/mobile/exec_tools.py | 59 +- artemis/tools/mobile/launch_app.py | 15 +- artemis/tools/mobile/ocr.py | 15 +- artemis/tools/object_detection_tool.py | 6 +- artemis/tools/scratchpad.py | 92 +- artemis/tools/tool_wrapper.py | 14 +- artemis/tools/video_tool.py | 15 +- artemis/utils/file.py | 28 - artemis/utils/plan_grammar.py | 362 +++++ artemis/utils/task_tree.py | 68 +- artemis/utils/verification.py | 62 - config/artemis.jsonc | 14 +- docs/plans/checker-two-gate-redesign.md | 370 +++++ docs/plans/context-visibility-redesign.md | 586 +++++++ mcp_server/README.md | 2 +- mcp_server/tools/task_manager.py | 55 +- .../action_server_manifest.json | 1443 +++++++++++++++++ .../action_surfaces/operator_shells.json | 303 ++++ .../action_surfaces/tool_declarations.json | 412 +++++ ...est_failure_analyzer_standalone_youtube.py | 5 - tests/tools/conftest.py | 3 + tests/tools/operator/test_operator_tools.py | 67 +- .../test_video_analyzer_tools.py | 38 +- .../unit/agents/test_check_prompt_assembly.py | 176 ++ tests/unit/agents/test_checker.py | 303 +++- tests/unit/agents/test_diagnoser.py | 17 +- tests/unit/agents/test_operator.py | 198 +-- tests/unit/agents/test_operator_prompts.py | 17 +- .../agents/test_plan_mutation_guardrails.py | 288 +++- tests/unit/agents/test_planner_node.py | 3 - tests/unit/agents/test_snapshot_rollback.py | 112 -- tests/unit/agents/test_validator.py | 6 +- .../test_latency_optimizations_deep.py | 47 - tests/unit/graph/test_checkpoints.py | 545 +++++++ tests/unit/graph/test_exit_settlement.py | 446 +++++ tests/unit/graph/test_graph.py | 409 ++--- tests/unit/graph/test_state_contract.py | 110 ++ tests/unit/mcp/test_action_manifest.py | 27 +- tests/unit/mcp/test_action_specs.py | 229 +++ tests/unit/test_config_package.py | 55 +- tests/unit/test_llm_gateway.py | 206 +++ tests/unit/test_llm_reliability.py | 68 + tests/unit/test_llm_structured.py | 73 + tests/unit/test_tools.py | 2 +- tests/unit/tools/test_append_note_tool.py | 24 +- tests/unit/tools/test_click_tool.py | 24 +- tests/unit/tools/test_committee_tool.py | 5 - tests/unit/tools/test_input_text_tool.py | 24 +- tests/unit/tools/test_launch_app_tool.py | 24 +- tests/unit/tools/test_list_notes_tool.py | 26 +- tests/unit/tools/test_long_press_tool.py | 24 +- .../unit/tools/test_object_detection_tool.py | 13 +- .../unit/tools/test_object_detection_trace.py | 1 - tests/unit/tools/test_ocr_tool.py | 37 +- tests/unit/tools/test_press_key_tool.py | 24 +- tests/unit/tools/test_read_note_tool.py | 26 +- tests/unit/tools/test_save_note_tool.py | 24 +- tests/unit/tools/test_swipe_tool.py | 24 +- tests/unit/tools/test_update_note_tool.py | 24 +- tests/unit/tools/test_video_analyzer_tool.py | 34 +- tests/unit/utils/test_plan_grammar.py | 241 +++ tests/unit/utils/test_verification.py | 77 - 106 files changed, 11542 insertions(+), 4445 deletions(-) create mode 100644 artemis/graph/checkpoints.py create mode 100644 artemis/graph/visibility.py create mode 100644 artemis/llm/reliability.py create mode 100644 artemis/llm/structured.py create mode 100644 artemis/mcp/action_specs.py delete mode 100644 artemis/tools/actions/__init__.py delete mode 100644 artemis/tools/actions/device_actions.py create mode 100644 artemis/utils/plan_grammar.py delete mode 100644 artemis/utils/verification.py create mode 100644 docs/plans/checker-two-gate-redesign.md create mode 100644 docs/plans/context-visibility-redesign.md create mode 100644 tests/fixtures/action_surfaces/action_server_manifest.json create mode 100644 tests/fixtures/action_surfaces/operator_shells.json create mode 100644 tests/fixtures/action_surfaces/tool_declarations.json create mode 100644 tests/unit/agents/test_check_prompt_assembly.py delete mode 100644 tests/unit/agents/test_snapshot_rollback.py create mode 100644 tests/unit/graph/test_checkpoints.py create mode 100644 tests/unit/graph/test_exit_settlement.py create mode 100644 tests/unit/graph/test_state_contract.py create mode 100644 tests/unit/mcp/test_action_specs.py create mode 100644 tests/unit/test_llm_gateway.py create mode 100644 tests/unit/test_llm_reliability.py create mode 100644 tests/unit/test_llm_structured.py create mode 100644 tests/unit/utils/test_plan_grammar.py delete mode 100644 tests/unit/utils/test_verification.py diff --git a/.gitignore b/.gitignore index ce68482..31cf3e9 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ coverage.xml .nox/ tests/**/outputs/ **/outputs/ +tests/**/trace_output/ # Linters & Formatters .ruff_cache/ diff --git a/artemis/agents/checker/checker.json b/artemis/agents/checker/checker.json index c90df2f..d418a92 100644 --- a/artemis/agents/checker/checker.json +++ b/artemis/agents/checker/checker.json @@ -1,4 +1,9 @@ { - "checker_system": "You are a precise and reliable verification expert (Checker). Your task is to determine whether the Operator has successfully completed the active subgoal by combining visual evidence from the current screenshot with data stored in persistent notes.", - "checker_prompt": "# Inputs\n- Active Subgoal: {{ subgoal }}\n- Execution History:\n{{ plan_and_history }}\n- Dialogue History: {{ dialogue_history }}\n- Visual State: The provided screenshot.\n\n# Guidelines\n- **Visual + Data Verification**: You must combine the screenshot with note contents. If the subgoal involves extracting, saving, or recording information (e.g., \"save contacts\", \"record address\"), you **MUST** verify that the content actually exists in the note. Do not assume success just because the information is visible on the screen. If the signs of success are not obvious, you can check the history, especially the most recent steps, to see if the operator performed their own check and whether that check was reasonable, to determine if the subgoal was truly successful.\n- **Visual Priority (Anti-History-Bias)**: Your primary ground truth is the visual evidence on the **Current Screenshot**. The `Execution History` lists previous successes or failures, which may be outdated due to dynamic background recovery or self-healing. If the screenshot demonstrably shows the active subgoal's target state is achieved, you **MUST** mark `success: true`, regardless of previous failed steps. Trust what you see, not past FAILED texts.\n- **Constraint Verification**: For subgoals containing specific parameters (such as names, brands, topics, events, numbers, or dates), confirm that the visible text precisely reflects these specified details. A successful state is defined by a clear, direct match with the subgoal's constraints.\n- **Iteration & Monitoring Subgoals Validation**: For concrete iteration sub-tasks (e.g., inspecting Candidate #1) or periodic polling subgoals (e.g., Polling Round #1 check), verify whether the Operator correctly executed the step and logged the progress/findings into notes or the task plan. Do NOT mark a monitoring or iteration step as failed simply because the overall monitoring session is ongoing or no new incoming events were detected in that specific polling cycle. As long as the discrete check was performed and recorded, mark `success: true`.\n- **Realistic Expectations**: You cannot scroll the screen. If a subgoal requires saving all items but only some are visible on the screen, verify that at least the **visible items** are recorded in the note. As long as the visible part is recorded and there are no obvious errors, consider it a success. Do not fail the task simply because off-screen content cannot be verified.\n- **Success Criteria**: Set `success` to `true` if the core objective of the subgoal is met and it does not block subsequent steps. Set `success` to `false` if the state is incorrect, required data was not saved, or the operator is stuck/looping.\n\n# Output Protocol\nInvoke the `submit_verification` tool with `success` (boolean) and `reason` (string) to submit your final decision. If you need to inspect note data first, use `read_note`." + "system": "You are a precise, evidence-driven verification expert (Checker). You audit declared check items against recorded execution evidence and issue verdicts. You never operate the device, never modify any state, and never negotiate with the executing agent — you only observe, probe read-only state where allowed, and judge.", + "base_rules": "# Verdict Rules (always apply)\n- Quote every check item verbatim in `item_text` — never paraphrase.\n- Every verdict MUST carry concrete evidence: a specific probe output, a specific note excerpt, or a specific on-screen/screenshot observation. A `failed` verdict without concrete evidence is invalid and will be downgraded.\n- `inconclusive` means the evidence needed for a judgment was never captured or is genuinely ambiguous. When evidence was never collected, output `inconclusive` and state explicitly that the evidence is missing — do NOT output `failed` for lack of evidence.\n- Judge only what the check item declares. Do not invent additional criteria and do not relax the declared ones.", + "verify_semantics": "# `verify` items (completion acceptance criteria)\n- A `verify` item asks: was the anchored subgoal actually accomplished to this standard?\n- `failed` verify verdicts trigger a repair loop, so a failed verdict MUST include a `suggestion`: a concrete, actionable hint telling the executor what is missing or wrong.\n- The release decision (whether execution proceeds) is computed outside of you and is fail-open on errors; your job is only the truthful verdict value.", + "assert_semantics": "# `assert` items (test assertions)\n- An `assert` item is a test expectation, not an instruction. A failed assertion is a legitimate, valuable test result: record it truthfully with evidence. Nothing will (or should) try to repair it.\n- Never mark an assert `failed` merely because evidence is absent — absence of captured evidence is `inconclusive` with an explicit note that the evidence is missing.\n- `suggestion` is not needed for assert verdicts.", + "anchor_guide": "# Evidence anchoring (checkpoint audit)\n- You are auditing a subgoal at its completion moment. The authoritative evidence is the execution history around the anchor step (the anchor step's pre-action screenshot and the preceding step's post-action state) plus persistently probeable device state.\n- For transient expectations (toasts, dialogs, momentary states) and ordering expectations, you MUST inspect the recorded history near the anchor step — the live device has already moved on and is NOT valid evidence for them.\n- Prefer persistent state probes and recorded notes over inference from summaries.", + "final_guide": "# Final review (task exit audit)\n- Audit against the USER'S ORIGINAL GOAL, not merely the plan's checkboxes.\n- `@end` items: judge them using the FINAL device state provided to you.\n- `on_complete` items: cite the verdict ledger — an already-recorded PASS/FAIL from the item's checkpoint attempt is authoritative (an assert that ever failed stays failed). For items never checked, judge from the historical evidence near their subgoal's completion; if that evidence was never captured, output `inconclusive` stating the evidence is missing. NEVER re-judge an `on_complete` item from the current screen: the current screen only proves the present, not the past moment the item was declared for.\n- `unmet_subgoals`: list, verbatim, any plan subgoal marked complete whose verify criteria are demonstrably not met. Leave it empty otherwise; failed asserts alone do NOT make a subgoal unmet.", + "probe_guide": "# Device probes (read-only)\n- `probe_device` runs a fixed, enumerated set of read-only state probes. Prefer probing persistent state over inferring it: alarms (`alarms`), battery status (`battery`), foreground activities (`foreground`), notifications (`notifications`), a settings value (`setting` with `namespace` + `key`), a content provider query (`content` with `uri`), installed packages (`packages`), a system property (`prop` with `key`).\n- Probes outside this table do not exist; you cannot run arbitrary commands." } diff --git a/artemis/agents/checker/checker.py b/artemis/agents/checker/checker.py index e2af85b..16d779b 100644 --- a/artemis/agents/checker/checker.py +++ b/artemis/agents/checker/checker.py @@ -12,36 +12,48 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Checker: independent, zero-side-effect verdict agent. + +Two entry points share one tool loop: + +- :func:`run_checkpoint_check` — audits a completed subgoal's ``on_complete`` + check items against the evidence anchored at completion time. Deliberately + has NO live-screen access: the device has moved on, and the tool table (not + prompt persuasion) enforces the evidence discipline. +- :func:`run_final_check` — audits the user's original goal plus all declared + check items at task exit, with the final screen state available. + +Both entries only ever read: step history, notes, and an enumerated table of +read-only device probes. No device actions, no note writes, no sub-agents. +The release decision is computed by the caller from the verdicts +(:func:`verdicts_allow_release`) — releasing never rewrites a verdict. +""" + +from __future__ import annotations + import asyncio -import hashlib +import base64 import json +import re +from dataclasses import dataclass, field from pathlib import Path -import re -from typing import Any +from typing import Any, Literal -from jinja2 import Template from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage -from langchain_core.tools import tool +from langchain_core.tools import BaseTool, tool from pydantic import BaseModel, Field from artemis.constants import CHECKER_MAX_ITERATIONS from artemis.context import ArtemisContext -from artemis.controllers.unified_controller import UnifiedMobileController -from artemis.data_engine.trace import ( - TraceSpan, - trace, - trace_langchain_tool, -) -from artemis.services.llm import get_llm, invoke_llm_with_timeout_message -from artemis.tools.index import get_tool_by_name +from artemis.data_engine.trace import trace, trace_langchain_tool +from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message from artemis.tools.scratchpad import get_read_note_tool_pure from artemis.tools.tool_wrapper import ( get_tool_result_content, invoke_tool_with_injection, ) from artemis.utils.logger import get_logger -from artemis.utils.notes import get_note_file_path, get_notes_dir from artemis.utils.ocr_api import is_ocr_configured, perform_ocr from artemis.utils.ocr_xml_fusion import ( _crop_image_remove_status_bar, @@ -49,413 +61,641 @@ from artemis.utils.ocr_xml_fusion import ( _map_coordinates_back, fuse_ocr_with_xml, ) -from artemis.utils.task_tree import ( - build_plan_and_history, - get_recent_subgoal_hashes, -) -from artemis.utils.verification import ( - append_verification_chat, - get_verification_chat_path, - get_verification_chat_rounds, - read_verification_chat, -) -from artemis.utils.text import safe_extract_text from artemis.utils.visualization import format_minimal_list_with_elements logger = get_logger(__name__) +_PROBE_OUTPUT_LIMIT = 8000 -class CheckerResult(BaseModel): - """Structured output contract for Checker verification.""" - success: bool = Field( - description="Whether the active subgoal was successfully completed based on visual and note evidence." +# --- Structured output ---------------------------------------------------------------- + + +class CheckVerdict(BaseModel): + """One check item's verdict. The verdict value is never altered by the + release decision (fail-open affects release only).""" + + item_text: str = Field(description="The check line's text, quoted verbatim.") + kind: Literal["verify", "assert"] + status: Literal["passed", "failed", "inconclusive"] + evidence: str = Field( + description=( + "Concrete observation backing the verdict: a probe output, note" + " excerpt, or screenshot element. Vague evidence invalidates a" + " failed verdict." + ) ) - reason: str = Field( - description="Detailed explanation of the result. If failed, specify what is missing or wrong and give actionable hints for the Operator." + suggestion: str = Field( + default="", + description="For failed verify items only: actionable repair hint.", ) +class CheckReport(BaseModel): + verdicts: list[CheckVerdict] = Field(default_factory=list) + unmet_subgoals: list[str] = Field( + default_factory=list, + description=( + "Final check only: plan subgoals (quoted verbatim) marked complete" + " whose verify criteria are demonstrably unmet." + ), + ) + + +def verdicts_allow_release(report: CheckReport) -> bool: + """Node-side release decision: every verify verdict passed or inconclusive. + + Computed outside the model so that releasing (fail-open) can never rewrite + a verdict value. Assert failures never block release. + """ + return all( + v.status in ("passed", "inconclusive") for v in report.verdicts if v.kind == "verify" + ) + + +# --- Read-only device probes (enumerated; no free-form command channel) -------------- + + +_KEY_RE = re.compile(r"^[A-Za-z0-9._-]+$") +_URI_RE = re.compile(r"^content://[A-Za-z0-9./_-]+$") +_SETTING_NAMESPACES = frozenset({"system", "secure", "global"}) + + +@dataclass(frozen=True) +class ProbeParam: + name: str + pattern: re.Pattern + allowed: frozenset[str] | None = None + + +@dataclass(frozen=True) +class ProbeSpec: + """A fixed argv template. ```` placeholders are substituted with + regex-validated parameters; argv is built as a list and never passes + through shell string concatenation.""" + + argv: tuple[str, ...] + params: tuple[ProbeParam, ...] = field(default=()) + + +PROBES: dict[str, ProbeSpec] = { + "alarms": ProbeSpec(argv=("dumpsys", "alarm")), + "battery": ProbeSpec(argv=("dumpsys", "battery")), + "foreground": ProbeSpec(argv=("dumpsys", "activity", "activities")), + "notifications": ProbeSpec(argv=("dumpsys", "notification")), + "setting": ProbeSpec( + argv=("settings", "get", "", ""), + params=( + ProbeParam("namespace", _KEY_RE, _SETTING_NAMESPACES), + ProbeParam("key", _KEY_RE), + ), + ), + "content": ProbeSpec( + argv=("content", "query", "--uri", ""), + params=(ProbeParam("uri", _URI_RE),), + ), + "packages": ProbeSpec(argv=("pm", "list", "packages")), + "prop": ProbeSpec(argv=("getprop", ""), params=(ProbeParam("key", _KEY_RE),)), +} + + +def build_probe_argv(kind: str, params: dict[str, Any] | None = None) -> list[str]: + """Builds the argv list for an enumerated probe. + + Raises ``ValueError`` for unknown probe kinds, missing/extra parameters, or + parameter values that fail their regex (whitespace and shell metacharacters + can never pass). ``dumpsys battery set``-style mutating subcommands are + structurally inexpressible: the no-parameter probes accept no arguments at + all. + """ + spec = PROBES.get(kind) + if spec is None: + raise ValueError(f"Unknown probe kind '{kind}'. Allowed: {', '.join(sorted(PROBES))}.") + params = dict(params or {}) + expected = {p.name for p in spec.params} + extra = set(params) - expected + if extra: + raise ValueError(f"Probe '{kind}' accepts no parameter(s): {sorted(extra)}") + missing = expected - set(params) + if missing: + raise ValueError(f"Probe '{kind}' requires parameter(s): {sorted(missing)}") + + values: dict[str, str] = {} + for p in spec.params: + raw = params[p.name] + if not isinstance(raw, str) or not p.pattern.match(raw): + raise ValueError(f"Probe '{kind}' parameter '{p.name}' has an invalid value.") + if p.allowed is not None and raw not in p.allowed: + raise ValueError( + f"Probe '{kind}' parameter '{p.name}' must be one of {sorted(p.allowed)}." + ) + values[p.name] = raw + + argv: list[str] = [] + for token in spec.argv: + if token.startswith("<") and token.endswith(">"): + argv.append(values[token[1:-1]]) + else: + argv.append(token) + return argv + + +async def _execute_probe(ctx: ArtemisContext, argv: list[str]) -> str: + """Executes a validated probe argv over the ADB device channel.""" + try: + adb_client = ctx.get_adb_client() + device = adb_client.device(serial=ctx.device.device_id) + output = await asyncio.to_thread(device.shell, argv) + text = str(output) + if len(text) > _PROBE_OUTPUT_LIMIT: + text = text[:_PROBE_OUTPUT_LIMIT] + "\n... [output truncated]" + return text + except Exception as e: + return f"Error executing probe: {e}" + + +def get_probe_tool(ctx: ArtemisContext) -> BaseTool: + @tool + async def probe_device(kind: str, params: dict | None = None) -> str: + """Runs one enumerated read-only device state probe. + + kind: one of 'alarms', 'battery', 'foreground', 'notifications', + 'setting' (params: namespace in system|secure|global, key), + 'content' (params: uri, a content:// URI), 'packages', + 'prop' (params: key). Anything else is rejected. + """ + try: + argv = build_probe_argv(kind, params) + except ValueError as e: + return f"Error: {e}" + return await _execute_probe(ctx, argv) + + return probe_device + + +# --- History / step evidence tools --------------------------------------------------- + + +def get_step_detail_tool(ctx: ArtemisContext) -> BaseTool: + @tool + def get_step_detail(step_no: int) -> str: + """Returns the recorded details of one execution step (action taken, + summary, metadata) by its step number.""" + if not ctx.data_engine: + return "Error: no execution history available." + record = ctx.data_engine.get_step_record(step_no) + if record is None: + return f"Error: step {step_no} not found." + payload = { + "step_number": record.step_number, + "timestamp": record.timestamp, + "summary": record.summary, + "action_taken": record.action_taken, + "last_execution_result": record.last_execution_result, + "extra_metadata": record.extra_metadata, + "has_pre_screenshot": bool(record.pre_image_name), + "has_post_screenshot": bool(record.post_image_name), + } + try: + return json.dumps(payload, ensure_ascii=False, default=str) + except Exception as e: + return f"Error serializing step: {e}" + + return get_step_detail + + +class _ScreenshotRequest: + """Sentinel result carrying an image to splice into the conversation.""" + + def __init__(self, description: str, image_b64: str | None): + self.description = description + self.image_b64 = image_b64 + + +def get_step_screenshot_tool(ctx: ArtemisContext) -> BaseTool: + @tool + def get_step_screenshot(step_no: int, which: str = "pre") -> str: + """Attaches the recorded screenshot of a step to the conversation. + `which` is 'pre' (state observed at the start of the step) or 'post' + (state after the step's action).""" + # The real work happens in the loop's interception; this body only + # documents the contract for schema generation. + return f"screenshot request: step {step_no} ({which})" + + return get_step_screenshot + + +def _load_step_screenshot(ctx: ArtemisContext, step_no: int, which: str) -> _ScreenshotRequest: + if which not in ("pre", "post"): + return _ScreenshotRequest(f"Error: 'which' must be 'pre' or 'post', got '{which}'.", None) + if not ctx.data_engine: + return _ScreenshotRequest("Error: no execution history available.", None) + path = ctx.data_engine.get_step_image_path(step_no, which) + if path is None: + return _ScreenshotRequest( + f"No {which}-action screenshot recorded for step {step_no}.", None + ) + try: + image_b64 = base64.b64encode(Path(path).read_bytes()).decode("utf-8") + except Exception as e: + return _ScreenshotRequest(f"Error reading screenshot: {e}", None) + return _ScreenshotRequest( + f"Screenshot of step {step_no} ({which}-action) is attached below.", image_b64 + ) + + +# --- Prompt assembly (segments x entry x content x config) --------------------------- + + +def _load_prompts() -> dict[str, str]: + prompts_path = Path(__file__).parent / "checker.json" + try: + return json.loads(prompts_path.read_text(encoding="utf-8")) + except Exception as e: + logger.error(f"Failed to load checker prompts: {e}") + return {} + + +def probes_enabled(ctx: ArtemisContext) -> bool: + setup = getattr(ctx, "execution_setup", None) + return not (setup and getattr(setup, "disable_device_probes", False)) + + +def assemble_checker_prompt_segments( + entry: Literal["checkpoint", "final"], + check_items: list, + probe_tool_registered: bool, + prompts: dict[str, str] | None = None, +) -> str: + """Assembles the system prompt segments for one run. + + Segment table (assembly discipline: every rule present corresponds to an + active mechanism, nothing more): + - base rules: always + - verify semantics: only when a verify item is under audit + - assert semantics: only when an assert item is under audit + - anchor guide: checkpoint entry only + - final guide: final entry only + - probe guide: only when the probe tool is actually registered + """ + prompts = prompts if prompts is not None else _load_prompts() + segments = [prompts.get("base_rules", "")] + kinds = {getattr(ci, "kind", None) for ci in check_items} + if "verify" in kinds: + segments.append(prompts.get("verify_semantics", "")) + if "assert" in kinds: + segments.append(prompts.get("assert_semantics", "")) + if entry == "checkpoint": + segments.append(prompts.get("anchor_guide", "")) + else: + segments.append(prompts.get("final_guide", "")) + if probe_tool_registered: + segments.append(prompts.get("probe_guide", "")) + return "\n\n".join(s for s in segments if s) + + +def build_checker_tools( + ctx: ArtemisContext, entry: Literal["checkpoint", "final"] +) -> list[BaseTool]: + """The tool table is the single source of Checker authority — all read-only. + + The checkpoint entry deliberately gets NO live-screen capability: its + evidence is anchored history plus persistent-state probes; the current + screen has already moved past the audited moment. The final entry receives + the final screen through its prompt (not a tool), so the table is identical + for both entries; the difference lives in what the prompt provides. + Device actions, note writes, and sub-agents are registered for neither. + """ + tools: list[BaseTool] = [ + get_step_detail_tool(ctx), + get_step_screenshot_tool(ctx), + get_read_note_tool_pure(ctx), + ] + if probes_enabled(ctx): + tools.append(get_probe_tool(ctx)) + return tools + + +# --- History formatting -------------------------------------------------------------- + + +def _format_history(ctx: ArtemisContext, limit: int = 60) -> str: + if not ctx.data_engine: + return "No execution history available." + try: + steps = ctx.data_engine.get_agent_friendly_steps() or [] + except Exception as e: + return f"Failed to load execution history: {e}" + if not steps: + return "No execution history available." + lines = [] + for s in steps[-limit:]: + num = s.get("step_number") + rel = s.get("relative_time", "") + summary = s.get("summary") or "" + action = s.get("action_taken") + action_str = "" + if action and not summary: + try: + action_str = json.dumps(action, ensure_ascii=False)[:160] + except Exception: + action_str = str(action)[:160] + lines.append(f"- Step {num} ({rel}): {summary or action_str}".rstrip()) + return "\n".join(lines) + + +def _format_check_items(check_items: list) -> str: + lines = [] + for ci in check_items: + lines.append(f"- [{ci.kind}] ({ci.when}) {ci.text}") + return "\n".join(lines) if lines else "(none)" + + +# --- Shared tool loop ---------------------------------------------------------------- + + +async def _structured_report(llm, messages) -> CheckReport: + structured_llm = llm.with_structured_output(CheckReport) + result = await invoke_llm_with_timeout_message(structured_llm.ainvoke(messages)) + if isinstance(result, CheckReport): + return result + if isinstance(result, dict): + try: + return CheckReport.model_validate(result) + except Exception: + pass + return CheckReport(verdicts=[]) + + +def _normalize_report(report: CheckReport, check_items: list) -> CheckReport: + """Node-side hygiene: vague failed verdicts downgrade to inconclusive; every + expected item gets a verdict (missing ones are inconclusive).""" + verdicts: list[CheckVerdict] = [] + for v in report.verdicts: + if v.status == "failed" and not v.evidence.strip(): + verdicts.append( + CheckVerdict( + item_text=v.item_text, + kind=v.kind, + status="inconclusive", + evidence="failed verdict lacked concrete evidence; downgraded", + suggestion=v.suggestion, + ) + ) + else: + verdicts.append(v) + + covered = {(v.kind, v.item_text) for v in verdicts} + for ci in check_items: + if (ci.kind, ci.text) not in covered: + verdicts.append( + CheckVerdict( + item_text=ci.text, + kind=ci.kind, + status="inconclusive", + evidence="no verdict produced for this item", + ) + ) + return CheckReport(verdicts=verdicts, unmet_subgoals=list(report.unmet_subgoals)) + + +async def _run_check_loop( + ctx: ArtemisContext, + messages: list, + tools: list[BaseTool], + check_items: list, +) -> CheckReport: + llm = get_llm(ctx=ctx, name="checker") + traced_tools = [trace_langchain_tool(t, ctx) for t in tools] + + max_iterations = ( + getattr(ctx.execution_setup, "checker_max_iterations", CHECKER_MAX_ITERATIONS) + if ctx.execution_setup + else CHECKER_MAX_ITERATIONS + ) + + report: CheckReport | None = None + for i in range(max_iterations): + if i == max_iterations - 1: + messages.append( + HumanMessage( + content=( + "This is your final iteration; provide your structured verdict report now." + ) + ) + ) + report = await _structured_report(llm, messages) + break + + active_llm = llm.bind_tools(tools=traced_tools) + response = await invoke_llm_with_timeout_message(acomplete(active_llm, messages)) + + if not response.tool_calls: + report = await _structured_report(llm, messages + [response]) + break + + messages.append(response) + for tc in response.tool_calls: + tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"] + args = dict(tc["args"]) + if tool_name == "get_step_screenshot": + shot = _load_step_screenshot( + ctx, int(args.get("step_no", -1)), str(args.get("which", "pre")) + ) + messages.append(ToolMessage(tool_call_id=tc["id"], content=shot.description)) + if shot.image_b64: + messages.append( + HumanMessage( + content=[ + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{shot.image_b64}" + }, + } + ] + ) + ) + continue + + tool_to_run = next((t for t in traced_tools if t.name == tool_name), None) + if tool_to_run is None: + messages.append( + ToolMessage( + tool_call_id=tc["id"], + content=f"Error: Tool {tool_name} not supported", + status="error", + ) + ) + continue + try: + result_obj = await invoke_tool_with_injection( + tool=tool_to_run, args=args, tool_call_id=tc["id"] + ) + result_str = get_tool_result_content(result_obj) + status = "error" if str(result_str).startswith("Error") else "success" + except Exception as e: + result_str = f"Error running tool {tool_name}: {e}" + status = "error" + messages.append(ToolMessage(tool_call_id=tc["id"], content=result_str, status=status)) + + if report is None: + report = CheckReport(verdicts=[]) + return _normalize_report(report, check_items) + + +# --- Entry points -------------------------------------------------------------------- + + @trace(type="agent", name="checker") -async def run_async_check( +async def run_checkpoint_check( ctx: ArtemisContext, + check_items, + anchor, + goal: str, subgoal_text: str, - subgoal_hash: str | None = None, - raw_perception_data: dict[str, Any] | None = None, - latest_ui_hierarchy: list[Any] | None = None, -): - """Runs the checker verification as a background task.""" - logger.info(f"Starting async Checker for: {subgoal_text}") +) -> CheckReport: + """Audits one completed subgoal's on_complete check items. - if not ctx.data_engine: - logger.warning("DataEngine not available, skipping checker.") - return {"status": "success", "reason": "DataEngine unavailable."} + Evidence discipline is enforced by the tool table: no live-screen access — + the anchored history (anchor step's pre screenshot, previous step's post + state) and persistent-state probes are the only valid sources. + """ + items = list(check_items) + logger.info(f"Starting checkpoint check for '{subgoal_text}' ({len(items)} item(s))") + prompts = _load_prompts() + tools = build_checker_tools(ctx, "checkpoint") + probe_registered = any(t.name == "probe_device" for t in tools) + system_content = "\n\n".join( + s + for s in ( + prompts.get("system", "You are the Checker."), + assemble_checker_prompt_segments("checkpoint", items, probe_registered, prompts), + ) + if s + ) - notes_dir = get_notes_dir(ctx.data_engine.base_dir) - status_path = notes_dir / "checker_status.json" - - if ctx.execution_setup and ctx.execution_setup.disable_checker: - logger.info("Checker is disabled via flag. Skipping.") + anchor_step_no = None + if ctx.data_engine and anchor.anchor_step_id: try: - status_path.write_text( - json.dumps({"status": "success", "reason": "Checker disabled."}), - encoding="utf-8", - ) + anchor_step_no = ctx.data_engine.get_step_number(anchor.anchor_step_id) except Exception: - pass - return {"status": "success", "reason": "Checker disabled."} + anchor_step_no = None - # 1. Set status to running + human_text = ( + f"# User's Original Goal\n{goal}\n\n" + f"# Audited Subgoal (just marked complete)\n{subgoal_text}\n\n" + f"# Check Items\n{_format_check_items(items)}\n\n" + f"# Evidence Anchor\nThe subgoal was marked complete at step" + f" {anchor_step_no if anchor_step_no is not None else 'unknown'}." + " The anchor step's pre-action screenshot and the preceding step's" + " post-action state capture the completion moment.\n\n" + f"# Task Plan at Completion Time\n{anchor.plan_text}\n\n" + f"# Execution History (summaries)\n{_format_history(ctx)}" + ) + + messages = [ + SystemMessage(content=system_content), + HumanMessage(content=human_text), + ] + return await _run_check_loop(ctx, messages, tools, items) + + +async def _capture_final_screen(ctx: ArtemisContext) -> tuple[str | None, str]: + """Best-effort capture of the final live screen (final entry only).""" try: - status_path.write_text( - json.dumps({"status": "running", "subgoal": subgoal_text}), - encoding="utf-8", - ) - logger.info(f"Wrote checker status 'running' to {status_path}") + from artemis.controllers.unified_controller import UnifiedMobileController + + controller = UnifiedMobileController(ctx) + device_data = await controller.get_screen_data() + screenshot_b64 = device_data.base64 + xml_hierarchy = device_data.elements + width = device_data.width + height = device_data.height + + ocr_results = [] + if is_ocr_configured(): + try: + status_bar_height = _detect_status_bar_height(xml_hierarchy, height) + if status_bar_height > 0: + cropped_b64, _, _ = _crop_image_remove_status_bar( + screenshot_b64, status_bar_height + ) + raw_ocr = await perform_ocr(cropped_b64) + ocr_results = _map_coordinates_back(raw_ocr, status_bar_height) + else: + ocr_results = await perform_ocr(screenshot_b64) + except Exception as e: + logger.warning(f"Final check OCR failed, proceeding without OCR: {e}") + + fused_xml = fuse_ocr_with_xml(xml_hierarchy, ocr_results) + minimal_list, _, _ = format_minimal_list_with_elements(fused_xml, width, height) + return screenshot_b64, minimal_list except Exception as e: - logger.error(f"Failed to write checker status: {e}") - return {"status": "success", "reason": f"Status write error: {e}"} + logger.warning(f"Failed to capture final screen state: {e}") + return None, "Final screen state unavailable." - # 2. Run Checker logic - try: - # Retrieve perception data: directly reuse upstream perception if available - if isinstance(raw_perception_data, dict) and raw_perception_data.get("screenshot_b64"): - screenshot_b64 = raw_perception_data["screenshot_b64"] - width = raw_perception_data.get("width", 1080) - height = raw_perception_data.get("height", 2400) - fused_xml = latest_ui_hierarchy or raw_perception_data.get("xml_hierarchy") or [] - elif ( - hasattr(ctx, "latest_perception_data") - and isinstance(ctx.latest_perception_data, dict) - and ctx.latest_perception_data - ): - screenshot_b64 = ctx.latest_perception_data.get("screenshot_b64") - width = ctx.latest_perception_data.get("width", 1080) - height = ctx.latest_perception_data.get("height", 2400) - fused_xml = ctx.latest_perception_data.get("latest_ui_hierarchy") or [] - else: - # Fallback to direct controller query - controller = UnifiedMobileController(ctx) - device_data = await controller.get_screen_data() - screenshot_b64 = device_data.base64 - xml_hierarchy = device_data.elements - width = device_data.width - height = device_data.height - ocr_results = [] - if is_ocr_configured(): - try: - status_bar_height = _detect_status_bar_height(xml_hierarchy, height) - if status_bar_height > 0: - cropped_b64, _, _ = _crop_image_remove_status_bar( - screenshot_b64, status_bar_height - ) - raw_ocr_results = await perform_ocr(cropped_b64) - ocr_results = _map_coordinates_back(raw_ocr_results, status_bar_height) - else: - ocr_results = await perform_ocr(screenshot_b64) - except Exception as e: - logger.warning(f"Checker OCR failed, proceeding without OCR: {e}") - - fused_xml = fuse_ocr_with_xml(xml_hierarchy, ocr_results) - - minimal_list, elements, labels = format_minimal_list_with_elements(fused_xml, width, height) - - # Build task tree - task_plan = "No task plan yet." - task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") - if task_plan_path.exists(): - try: - task_plan = task_plan_path.read_text(encoding="utf-8") - except Exception as e: - logger.error(f"Failed to read task plan for Checker: {e}") - - steps = ctx.data_engine.get_agent_friendly_steps() if ctx.data_engine else [] - - # Calculate exact subgoal hash from subgoal_text directly to avoid 'default' placeholder - if not subgoal_hash or subgoal_hash == "default": - clean_subgoal = safe_extract_text(subgoal_text).strip() - subgoal_hash = hashlib.md5(clean_subgoal.encode("utf-8")).hexdigest() - - keep_hashes = get_recent_subgoal_hashes(steps, subgoal_hash, ctx.data_engine.base_dir) - - plan_and_history = build_plan_and_history( - task_plan, steps, subgoal_hash, keep_subgoal_hashes=keep_hashes +def _format_ledger(ledger: list[dict]) -> str: + if not ledger: + return "(no checkpoint verdicts were recorded)" + lines = [] + for r in ledger: + lines.append( + f"- [{r.get('kind')}] '{r.get('item_text')}' -> {r.get('status')}" + f" (attempt {r.get('attempt_id')}): {r.get('evidence', '')}" ) + return "\n".join(lines) - chat_path = get_verification_chat_path(ctx.data_engine.base_dir, subgoal_hash) - turns = read_verification_chat(chat_path) - # Calculate round number - max_op, max_chk = get_verification_chat_rounds(turns) - round_num = max(max_op, max_chk + 1) - if round_num == 0: - round_num = 1 - - # Render dialogue history for prompt - dialogue_lines = [] - for t in turns: - role = "Operator" if t["role"] == "operator" else "Checker" - dialogue_lines.append(f"**{role} (Round {t['round']})**:\n{t['content']}") - dialogue_history = ( - "\n\n".join(dialogue_lines) if dialogue_lines else "No previous dialogue." +@trace(type="agent", name="checker") +async def run_final_check( + ctx: ArtemisContext, + goal: str, + plan_text: str, + ledger: list[dict], + check_items, +) -> CheckReport: + """Final review at task exit: audits the user's original goal and all + declared check items, with the final screen state available.""" + items = list(check_items) + logger.info(f"Starting final check ({len(items)} declared item(s))") + prompts = _load_prompts() + tools = build_checker_tools(ctx, "final") + probe_registered = any(t.name == "probe_device" for t in tools) + system_content = "\n\n".join( + s + for s in ( + prompts.get("system", "You are the Checker."), + assemble_checker_prompt_segments("final", items, probe_registered, prompts), ) + if s + ) - prompts = {} - prompts_path = Path(__file__).parent / "checker.json" - if prompts_path.exists(): - try: - prompts = json.loads(prompts_path.read_text(encoding="utf-8")) - except Exception as e: - logger.error(f"Failed to load checker prompts: {e}") + screenshot_b64, minimal_list = await _capture_final_screen(ctx) - prompt_template = prompts.get( - "checker_prompt", - "You are the Checker. Your job is to verify if the current subtask" - " was successfully completed.", - ) + human_text = ( + f"# User's Original Goal\n{goal}\n\n" + f"# Final Task Plan\n{plan_text or '(no plan)'}\n\n" + f"# Declared Check Items\n{_format_check_items(items)}\n\n" + f"# Verdict Ledger (checkpoint results, authoritative for on_complete" + f" items)\n{_format_ledger(ledger)}\n\n" + f"# Execution History (summaries)\n{_format_history(ctx)}\n\n" + f"# Final Screen Elements\n{minimal_list}" + ) - full_prompt = Template(prompt_template).render( - subgoal=subgoal_text, - plan_and_history=plan_and_history, - dialogue_history=dialogue_history if dialogue_history else "No previous dialogue.", - minimal_list=minimal_list, - ) - - llm = get_llm(ctx=ctx, name="checker") - read_note = get_read_note_tool_pure(ctx) - - @tool - def submit_verification( - success: bool, - reason: str, - ) -> str: - """[TERMINAL] Submit the final verification outcome (success or failure with reason). This completes verification.""" - return f"Verification decision: success={success}, reason={reason}" - - tools_to_bind = [ - trace_langchain_tool(read_note, ctx), - submit_verification, - ] - - content = [ - {"type": "text", "text": full_prompt}, - {"type": "text", "text": "--- Current Screenshot ---"}, + content: list = [{"type": "text", "text": human_text}] + if screenshot_b64: + content.append({"type": "text", "text": "--- Final Screenshot ---"}) + content.append( { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"}, - }, - ] - - messages = [ - SystemMessage( - content=prompts.get( - "checker_system", - "You are the Checker agent. Your objective is to accurately" - " assess whether the active subgoal was successfully" - " completed.", - ) - ), - HumanMessage(content=content), - ] - - max_iterations = ( - getattr( - ctx.execution_setup, - "checker_max_iterations", - CHECKER_MAX_ITERATIONS, - ) - if ctx.execution_setup - else CHECKER_MAX_ITERATIONS + } ) - success = True - reason = "Checker completed." - - for i in range(max_iterations): - is_final_round = i == max_iterations - 1 - - if is_final_round: - messages.append( - HumanMessage( - content=( - "This is your final iteration; all tools are" - " stripped, and you must provide your final structured verification result." - ) - ) - ) - structured_llm = llm.with_structured_output(CheckerResult) - structured_res: CheckerResult = await invoke_llm_with_timeout_message( - structured_llm.ainvoke(messages) - ) - if isinstance(structured_res, CheckerResult): - success = structured_res.success - reason = structured_res.reason - elif isinstance(structured_res, dict): - success = structured_res.get("success", True) - reason = structured_res.get("reason", "Verification completed.") - break - else: - if i > 0: - messages.append( - HumanMessage( - content=( - "You have not completed verification yet" - f" (iteration {i + 1} of {max_iterations})." - ) - ) - ) - active_llm = llm.bind_tools(tools=tools_to_bind) - - async def run_stream(): - full_response = None - async for chunk in active_llm.astream(messages): - if full_response is None: - full_response = chunk - else: - full_response += chunk - return full_response - - response = await invoke_llm_with_timeout_message(run_stream()) - - # Check if submit_verification was invoked - submit_tc = next( - ( - tc - for tc in response.tool_calls - if tc["name"].endswith("submit_verification") - ), - None, - ) - if submit_tc: - tc_args = dict(submit_tc["args"]) - success = bool(tc_args.get("success", True)) - reason = str(tc_args.get("reason", "Verification submitted.")) - break - - if not response.tool_calls: - # Model answered in plain text; use structured_llm directly - structured_llm = llm.with_structured_output(CheckerResult) - structured_res: CheckerResult = await invoke_llm_with_timeout_message( - structured_llm.ainvoke(messages + [response]) - ) - if isinstance(structured_res, CheckerResult): - success = structured_res.success - reason = structured_res.reason - elif isinstance(structured_res, dict): - success = structured_res.get("success", True) - reason = structured_res.get("reason", "Verification completed.") - break - - messages.append(response) - - async def run_tool(tc): - tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"] - logger.info(f"Checker requested tool: {tool_name}") - - result_str = "" - tool_status = "success" - try: - if ":" in tool_name: - tool_to_run = get_tool_by_name(tool_name, tools_to_bind) - else: - tool_to_run = next( - (t for t in tools_to_bind if t.name == tool_name), - None, - ) - if tool_to_run: - args = dict(tc["args"]) - with TraceSpan(name=tool_name, ctx=ctx) as span: - span.payload = {"args": args} - try: - result_obj = await invoke_tool_with_injection( - tool=tool_to_run, - args=args, - tool_call_id=tc["id"], - ) - result_str = get_tool_result_content(result_obj) - span.result = result_str - if result_str.startswith("Error"): - tool_status = "error" - span.status = "failed" - span.error = result_str - except Exception as err: - tool_status = "error" - span.status = "failed" - span.error = str(err) - raise err - else: - result_str = f"Error: Tool {tool_name} not supported" - tool_status = "error" - except Exception as err: - logger.error(f"Error running tool {tool_name}: {err}") - result_str = f"Error running tool {tool_name}: {err}" - tool_status = "error" - - return ToolMessage( - tool_call_id=tc["id"], - content=result_str, - status=tool_status, - ) - - tool_outputs = await asyncio.gather(*(run_tool(tc) for tc in response.tool_calls)) - for tm in tool_outputs: - messages.append(tm) - - logger.info( - f"Async Checker result for '{subgoal_text}': Success={success}, Reason={reason}" - ) - - # Update status file - status_path.write_text( - json.dumps({"status": "success" if success else "failed", "reason": reason}), - encoding="utf-8", - ) - - # Clean up dialogue file if successful - if success: - if chat_path.exists(): - logger.info(f"Checker succeeded. Cleaning up dialogue file: {chat_path}") - chat_path.unlink() - - # If failed, revert status in task_plan.md and write feedback - if not success: - task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") - if task_plan_path.exists(): - task_plan = task_plan_path.read_text(encoding="utf-8") - lines = task_plan.split("\n") - - for i in range(len(lines) - 1, -1, -1): - if lines[i].startswith("- ["): - task_content = safe_extract_text(lines[i][5:]).strip() - safe_subgoal = safe_extract_text(subgoal_text).strip() - if task_content == safe_subgoal: - lines[i] = re.sub(r"^(- \[)[x/ ](\])", r"\g<1>/\g<2>", lines[i]) - logger.info( - "Matched top-level subgoal by content." - f" Reverted status to [/]: {subgoal_text}" - ) - break - task_plan_path.write_text("\n".join(lines), encoding="utf-8") - - append_verification_chat( - chat_path, - "checker", - f"Status: Failed.\nReason: {reason}", - round_num, - ) - - return {"status": "success" if success else "failed", "reason": reason} - - except Exception as e: - logger.error(f"Async Checker encountered exception (fail-open): {e}") - status_path.write_text( - json.dumps( - { - "status": "success", - "reason": f"Checker Fail-Open: {e}", - } - ), - encoding="utf-8", - ) - return {"status": "success", "reason": f"Checker Fail-Open: {e}"} + messages = [ + SystemMessage(content=system_content), + HumanMessage(content=content), + ] + return await _run_check_loop(ctx, messages, tools, items) diff --git a/artemis/agents/diagnoser/diagnoser.py b/artemis/agents/diagnoser/diagnoser.py index 2fd5239..4b5df7c 100644 --- a/artemis/agents/diagnoser/diagnoser.py +++ b/artemis/agents/diagnoser/diagnoser.py @@ -30,7 +30,7 @@ from artemis.data_engine.trace import ( trace_langchain_tool, ) from artemis.graph.state import State -from artemis.services.llm import get_llm, invoke_llm_with_timeout_message +from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message from artemis.tools.command_tool import get_run_short_adb_command_tool from artemis.tools.diagnoser_submit_answer_tool import get_submit_answer_tool from artemis.tools.index import get_tool_by_name @@ -173,21 +173,11 @@ class Diagnoser: logger.error(f"Failed to build plan and history in Diagnoser: {e}") plan_and_history = f"Error building plan and history: {e}" - # Get Checker Feedback + # Get verification findings injected by the checkpoint harvest, if any checker_feedback = "" - if self.ctx.data_engine: - notes_dir = Path(self.ctx.data_engine.base_dir) / "notes" - verification_chat_path = notes_dir / f"verification_chat_{subgoal_hash}.json" - if verification_chat_path.exists(): - try: - turns = json.loads(verification_chat_path.read_text(encoding="utf-8")) - dialogue_lines = [] - for t in turns: - role = "Operator" if t["role"] == "operator" else "Checker" - dialogue_lines.append(f"**{role} (Round {t['round']})**:\n{t['content']}") - checker_feedback = "\n\n".join(dialogue_lines) - except Exception as e: - logger.error(f"Error reading verification chat in Diagnoser: {e}") + findings = getattr(state, "operator_feedback", None) + if findings: + checker_feedback = "\n".join(f"- {f}" for f in findings) # Get Current Screenshot and UI Hierarchy latest_screenshot_b64 = None @@ -309,18 +299,6 @@ class Diagnoser: ) llm = base_llm.bind_tools(tools=traced_tools) - async def run_stream(): - full_response = None - trace_id = CURRENT_TRACE_ID.get() - async for chunk in llm.astream(current_messages): - if full_response is None: - full_response = chunk - else: - full_response += chunk - - pass - return full_response - with TraceSpan(name="gemini_diagnoser_call", ctx=self.ctx) as span: span.payload = { "context_components": { @@ -331,7 +309,9 @@ class Diagnoser: "diagnostic_query": prompt, } } - response = await invoke_llm_with_timeout_message(run_stream()) + response = await invoke_llm_with_timeout_message( + acomplete(llm, current_messages) + ) span.result = ( response.content if hasattr(response, "content") else str(response) ) diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index 788b74f..892d5f3 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -49,9 +49,11 @@ from artemis.controllers.unified_controller import UnifiedMobileController from artemis.data_engine.trace import trace from artemis.graph.perception import _check_injected_instruction_file from artemis.graph.state import State +from artemis.llm.structured import ParseFailure, parse_structured from artemis.mcp.action_executor import McpActionExecutor from artemis.services.llm import ( RobustChatModelWrapper, + acomplete, get_google_llm, get_llm, invoke_llm_with_timeout_message, @@ -182,20 +184,23 @@ class FlashRunner: # Check for real-time injected instructions if self.ctx.data_engine and self.ctx.data_engine.base_dir: try: - injected_instruction = await asyncio.to_thread( + injected_payload = await asyncio.to_thread( _check_injected_instruction_file, str(self.ctx.data_engine.base_dir), ) - if injected_instruction: - messages.append( - HumanMessage( - content=( - "[REAL-TIME INJECTED INSTRUCTION from user]:" - f" {injected_instruction}\nYou MUST immediately follow" - " this instruction and adjust your plan/actions." - ) - ) + if injected_payload and injected_payload.get("instruction"): + injected_text = ( + "[REAL-TIME INJECTED INSTRUCTION from user]:" + f" {injected_payload['instruction']}\nYou MUST immediately" + " follow this instruction and adjust your plan/actions." ) + if injected_payload.get("release_loop"): + injected_text += ( + "\nThe user has explicitly authorized stopping any" + " ongoing monitoring loop; you may now wrap up and" + " complete the task." + ) + messages.append(HumanMessage(content=injected_text)) except Exception as e: logger.warning(f"Failed to check injected instruction in FlashRunner: {e}") @@ -224,61 +229,12 @@ class FlashRunner: # Bind active tools bound_llm = llm.bind_tools(current_tools) - # Invoke Model with Streaming & Retry - response = None - max_retries = 3 - - for attempt in range(max_retries): - try: - stream_exec_id = uuid.uuid4() - - async def run_stream(): - full_response = None - async for chunk in bound_llm.astream(messages): - if full_response is None: - full_response = chunk - else: - full_response += chunk - - # Real-time token streaming to DataEngine for live UI display - if ( - getattr(self, "ctx", None) - and getattr(self.ctx, "data_engine", None) - and getattr(chunk, "content", None) - ): - text_to_stream = "" - thinking_to_stream = "" - if isinstance(chunk.content, str): - text_to_stream = chunk.content - elif isinstance(chunk.content, list): - for item in chunk.content: - if isinstance(item, str): - text_to_stream += item - elif isinstance(item, dict): - if item.get("type") == "text": - text_to_stream += item.get("text", "") - elif item.get("type") == "thinking": - thinking_to_stream += item.get("thinking", "") - if text_to_stream: - self.ctx.data_engine.stream_output( - stream_exec_id, text_to_stream, is_thinking=False - ) - if thinking_to_stream: - self.ctx.data_engine.stream_output( - stream_exec_id, thinking_to_stream, is_thinking=True - ) - return full_response - - response = await invoke_llm_with_timeout_message( - run_stream(), timeout_seconds=10, hard_timeout=180 - ) - if response: - break - except Exception as e: - logger.warning(f"FlashRunner LLM turn failed on attempt {attempt + 1}: {e}") - if attempt == max_retries - 1: - raise e - await asyncio.sleep(2.0**attempt) + # Invoke Model. Streaming, live-token UI deltas, classified + # retries, and pause/resume are all owned by the LLM gateway + # (acomplete); a typed LLMCallError propagates if it gives up. + response = await invoke_llm_with_timeout_message( + acomplete(bound_llm, messages), timeout_seconds=10, hard_timeout=180 + ) if response is None: break @@ -372,19 +328,20 @@ class FlashRunner: # Fallback text parsing if tool_calls not parsed natively if not tool_calls and "```json" in raw_text: - try: - snippet = raw_text.split("```json")[1].split("```")[0].strip() - parsed = json.loads(snippet) - if isinstance(parsed, dict) and "name" in parsed: - tool_calls = [ - { - "name": parsed["name"], - "args": parsed.get("args", {}), - "id": str(uuid.uuid4()), - } - ] - except Exception: - pass + parsed = parse_structured(raw_text) + if isinstance(parsed, ParseFailure): + logger.warning( + "FlashRunner response contained a JSON block that could" + f" not be parsed: {parsed.error}" + ) + elif isinstance(parsed, dict) and "name" in parsed: + tool_calls = [ + { + "name": parsed["name"], + "args": parsed.get("args", {}), + "id": str(uuid.uuid4()), + } + ] if not tool_calls: logger.info( diff --git a/artemis/agents/history_analyzer/history_analyzer.py b/artemis/agents/history_analyzer/history_analyzer.py index 3689951..656229c 100644 --- a/artemis/agents/history_analyzer/history_analyzer.py +++ b/artemis/agents/history_analyzer/history_analyzer.py @@ -25,7 +25,7 @@ from langchain_core.messages import ( from langchain_core.tools import BaseTool, tool from artemis.context import ArtemisContext from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace -from artemis.services.llm import get_llm, invoke_llm_with_timeout_message +from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message from artemis.tools.scratchpad import ( get_list_notes_tool_pure, get_read_note_tool_pure, @@ -164,19 +164,7 @@ class HistoryAnalyzer: for turn in range(max_turns): logger.info(f"HistoryAnalyzer ReAct turn {turn + 1}") - async def run_stream(): - full_response = None - trace_id = CURRENT_TRACE_ID.get() - async for chunk in llm.astream(messages): - if full_response is None: - full_response = chunk - else: - full_response += chunk - - pass - return full_response - - response = await invoke_llm_with_timeout_message(run_stream()) + response = await invoke_llm_with_timeout_message(acomplete(llm, messages)) messages.append(response) if not response.tool_calls: diff --git a/artemis/agents/log_analyzer/log_analyzer.py b/artemis/agents/log_analyzer/log_analyzer.py index fd1325f..dc66e59 100644 --- a/artemis/agents/log_analyzer/log_analyzer.py +++ b/artemis/agents/log_analyzer/log_analyzer.py @@ -58,7 +58,7 @@ from artemis.data_engine.trace import ( trace, ) from artemis.graph.state import State -from artemis.services.llm import get_llm, invoke_llm_with_timeout_message +from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message from artemis.tools.index import get_tool_by_name from artemis.tools.mobile.log_utils import fetch_and_filter_logs from artemis.tools.mobile.read_logs import get_read_logs_tool @@ -220,19 +220,7 @@ class LogAnalyzerNode: ) llm = base_llm.bind_tools(tools=tools_for_binding) - async def run_stream(): - full_response = None - trace_id = CURRENT_TRACE_ID.get() - async for chunk in llm.astream(current_messages): - if full_response is None: - full_response = chunk - else: - full_response += chunk - - pass - return full_response - - response = await invoke_llm_with_timeout_message(run_stream()) + response = await invoke_llm_with_timeout_message(acomplete(llm, current_messages)) if response.content: outcome = response.content diff --git a/artemis/agents/object_detector/object_detector.py b/artemis/agents/object_detector/object_detector.py index f0605ca..22b34f2 100644 --- a/artemis/agents/object_detector/object_detector.py +++ b/artemis/agents/object_detector/object_detector.py @@ -22,9 +22,8 @@ from pathlib import Path import re from langchain_core.messages import HumanMessage, ToolMessage -from langgraph.types import Command -from artemis.constants import VALIDATOR_MESSAGES_KEY +from artemis.llm.structured import ParseFailure, parse_structured from artemis.services.llm import get_llm from artemis.utils.logger import get_logger @@ -78,22 +77,24 @@ async def _detect_single_label( ) logger.info(f"Received response for '{label}' (Attempt {i + 1})") - cleaned_initial = output.strip() - code_block_match = re.search(r"```(?:json)?\s*([\s\S]*?)\s*```", cleaned_initial) - if code_block_match: - cleaned_initial = code_block_match.group(1).strip() - - try: - res_json = json.loads(cleaned_initial) - if isinstance(res_json, list): - valid_results = [] - for item in res_json: - if isinstance(item, dict): - item["label"] = label - valid_results.append(item) - return valid_results - except json.JSONDecodeError: - pass + res_json = parse_structured(output) + if isinstance(res_json, list): + valid_results = [] + for item in res_json: + if isinstance(item, dict): + item["label"] = label + valid_results.append(item) + return valid_results + if isinstance(res_json, ParseFailure): + logger.warning( + f"Object detector response for '{label}' was not valid" + f" JSON (attempt {i + 1}): {res_json.error}" + ) + else: + logger.warning( + f"Object detector response for '{label}' parsed to" + f" {type(res_json).__name__} instead of a list (attempt {i + 1})." + ) except Exception as e: logger.warning(f"Detection attempt {i + 1} failed for '{label}': {e}") @@ -194,35 +195,17 @@ async def _run_object_detection( async def _create_error_command(ctx, state, tool_call_id, error_message, wrapper): - tool_message = ToolMessage( + return ToolMessage( tool_call_id=tool_call_id or "default_tool_call", content=wrapper.on_failure_fn(error_message), additional_kwargs={"error": error_message}, status="error", ) - return Command( - update=await state.asanitize_update( - ctx=ctx, - update={ - VALIDATOR_MESSAGES_KEY: [tool_message], - }, - agent="validator", - ), - ) async def _create_success_command(ctx, state, tool_call_id, output, wrapper): - tool_message = ToolMessage( + return ToolMessage( tool_call_id=tool_call_id or "default_tool_call", content=wrapper.on_success_fn(output), status="success", ) - return Command( - update=await state.asanitize_update( - ctx=ctx, - update={ - VALIDATOR_MESSAGES_KEY: [tool_message], - }, - agent="validator", - ), - ) diff --git a/artemis/agents/operator/operator.json b/artemis/agents/operator/operator.json index 5179ad1..0c3b004 100644 --- a/artemis/agents/operator/operator.json +++ b/artemis/agents/operator/operator.json @@ -1,4 +1,3 @@ { - "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen. For any task requiring continuous range interactions, sliding, or boundary selection, prefer using targeted swipe/drag gestures spanning from the element's start bounds [xmin, ymin] to end bounds [xmax, ymax] to ensure the action is strictly confined to the target element. For drag-and-drop, list reordering, or sliding sliders/SeekBars, set duration >= 1000 (e.g. 1500) and always drag slightly PAST the target position to overcome touch slop and reliably trigger the drop/snap/value update. When setting a slider to Maximum (100%) or Minimum (0%), always swipe fully to the extreme boundary to guarantee reaching the absolute limits.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n- **Tool Calling Iron Rule**: Do NOT submit a Turn-Ending Action at the same time (in the same tool-call list) as a Pre-Decision Exploratory Tool. You must first gather information (e.g., finding coordinates or diagnosing a state), manage ADB background tasks, or update memory notes. You can use pre-decision exploratory tools to complete this task. You will be prompted again with the tool's result, at which point you can output your final physical/turn-ending action.\n- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n- **Memory Preservation & Paginated Exploration**: Using note tools is entirely zero-cost. Actively use this tool to maintain goals and logs. The note tool is the only reliable way to persist memory. Therefore, when you find information on the screen that might be relevant to the task, even if you are not certain, you should first save the information currently visible on the screen before exploring further. If you have already recorded some information, you can use `append_note` to append the new information to the existing content. The note tool can also help you organize lists or information across multiple screens; you can consolidate all information into notes for unified processing.\n- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor. Calibrate your scroll distance so that this anchor remains visible near the top of the subsequent screen, creating a seamless visual overlap that guarantees zero missed items. Terminate exploration when a definitive boundary is reached (e.g., list bottom reached or no new content appears after scrolling).\n 2. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 3. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\". Only perform incremental re-verification if the list has been refreshed or content has mutated, while maintaining strict single-pass handling of new events.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] into note (Exit: ; Interval: )`, adhere strictly to this grammar contract and 5-step cyclic state machine:\n 1. **Grammar & Invariant Contract**:\n - `[Loop]`: Represents a multi-turn lifecycle container. The top-level milestone MUST remain active (`[/]`) and must never be marked `[x]` by single checks.\n - ``: The persistent baseline and diffing ledger. You must initialize it in Round 1 and cross-reference/append to it across cycles.\n - `Exit: Continuous until manually stopped`: Strict unbounded mode. Never self-terminate or mark the top-level milestone as `[x]`; cycle continuously until external stop instruction is injected.\n - `Exit: `: Bounded mode. Only mark the top-level milestone as `[x]` once the explicit count/target threshold is visually achieved.\n - `Interval: `: Declared polling cadence. You MUST strictly adhere to the exact duration declared in `` (convert minutes to milliseconds accurately for `wait_for_delay`).\n 2. **The 5-Step Cyclic State Machine (SOP)**:\n - **Step 1 (Baseline Sync)**: On initial cycle, observe all visible entries/metrics in the primary monitoring view and save full snapshot into ``.\n - **Step 2 (Pacing & Refresh)**: Respect `` and execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n- **Short-Term Memory**: short_term_memory is a supplement to the task plan. If needed, use the tag in your raw thinking text to record it. Short-term memory will be passed to the agent in the next turn, you can describe any trivial details, it should at least include expectations for the future, the current short-term strategy, and verified facts. Example:\n\nExpectation: Duplicate targets were not found in e-h, now start looking for items starting with i-l. (Letters after m are still unverified); Operation strategy: Search in batches by starting letter; Verified facts: a-d, e-h have no matches.\n\n\n# SYSTEM WORKFLOW: SAFETY NET & AUTO-RECOVERY\nYour decisions are processed through downstream pipeline guardians to guarantee accuracy:\n- **Pre-Execution Safety Net**: Automatically checks that your target is visible and safe to interact with prior to action execution.\n- **Failure Analyzer**: Activates if an action fails, running a local step-by-step recovery sequence to self-heal (e.g., closing keyboards or handling wait states).\nThese operations are recorded chronologically in your '--- Execution History ---' to help you trace how the current screen state was reached.\n\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Pre-Decision Exploratory Tools**:\n - *What they are*: Helper/Subagent tools (such as `ask_explorer`, `ask_diagnoser`, `video_analyzer`), ADB command tools (`run_adb_command`, `manage_task`), and memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`, `append_note`).\n - *How they work*: These tools gather details or update/read memory. When you invoke these tools, the framework will immediately execute them and return the results, allowing you to continue thinking and make your final decision.\n2. **Turn-Ending Actions**:\n - *What they are*: Physical device actions (`click`, `input_text`, `swipe`, `press_key`, `manage_app`, `wait_for_delay`, `long_press`) and flow-ending actions (`reply_to_checker`).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, proceed through these three phases in sequence.\n\n## Phase 1: Observation & Diagnosis\n1. **Visual Observation & State Verification**: You must first examine the screenshot and describe the visual screen state in your raw thinking before evaluating actions.\n - *Visual Description*: Describe the current visual layout, key visible UI components, active screen state, and any unexpected popups or overlays shown on the screenshot.\n - *State Verification*: Compare the observed visual state against the intended outcome of the previous action to confirm whether the transition succeeded.\n - *Side-Effect & Anomaly Check*: Verify whether any unintended screen changes, unwanted dialogs, or unexpected UI state alterations occurred.\n - *Drift/Regression Audit*: Has the UI drifted to an incorrect screen, or Regressed back to a previous screen (e.g. from an accidental browser back-navigation)?\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen screen (static stuck)?\n - *Missed Transient States*: If the current state jumped ahead and you suspect you missed an important initial or intermediate screen (e.g., a number or flash message that was overwritten by your last action), consider using the `video_analyzer` tool to look back at the recent history.\n2. **Path Decision (Recover vs. Adapt)**: Encourage flexible problem-solving. First, evaluate if the current path can still be recovered or advanced (e.g., dismissing a blocker, retrying, or restoring the view). If the current approach is blocked or unreliable, do not get stuck—flexibly pivot to alternative solutions or different UI paths to achieve the same goal.\n3. **Verify & Diagnose (Mandatory Diagnosis Triggers & Loop Exemptions)**: You **MUST** use the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage (before calling any Turn-Ending Actions) if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and are EXEMPT from triggering mandatory diagnosis, provided you are making progress or adhering to the planned loop pacing.\n - **Validation/Checker Rejection**: The verification agent (Checker) rejected your subgoal completion (i.e., you are in troubleshooter mode and this is a retry).\n - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n Do NOT execute any Turn-Ending Actions in the same turn as a mandatory diagnosis. Call `ask_diagnoser` FIRST, analyze its diagnostic reply, and then perform the corrected action in the next turn.\n\n## Phase 2: Planning & Memory Preservation\n1. **Autonomous Goal Management & Dynamic Live-Iteration Tracking**: You have autonomy over nested subgoals under the top-level milestones. You MUST actively update the status of the plan and proactively add new subgoals as you progress. Use `[/]` to denote the currently active milestone and subgoal. Only mark a step as completed (`[x]`) AFTER visual confirmation. Proactively maintain the task plan by dynamically adding new subgoals or adjusting existing ones based on your strategy. Keep the plan as a pure Markdown checklist in `task_plan`.\n - *Milestone Structure Preservation*: You are strictly forbidden from deleting, omitting, or modifying top-level `[Loop]` milestones or their declared exit conditions.\n - *Dynamic Live-Iteration Ledger in Loops*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), dynamically log each concrete iteration item in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle (e.g., ` - [x] Polling Check #1 (10:00): Refreshed, no new items, waiting 5m`). This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a loop, evaluate whether the milestone's Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit). For continuous monitoring milestones where Exit is `until manually stopped`, the milestone MUST remain `[/]` and CANNOT be marked `[x]` by you. Once a bounded exit condition is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**\n2. **Preserve Memory & Pre-Action Snapshotting**: You MUST perform the following risk assessment before any UI interaction: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain if you will need these details, you should immediately save the extracted critical values to the note tools to ensure safe downstream access. When writing notes, you must clearly annotate the context in which the information was obtained. Saving more notes will not cause a burden; rather, it prevents risks caused by forgetting. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated. Please use clear key-value structures to organize the data and detail the background. If you have already missed certain critical information and cannot re-view it by going back or other operations, you can use the `video_analyzer` to look back at historical information.\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data.\n2. **Action Selection & Mandatory Cognitive Protocol**: Before calling any Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`, `press_key`, `manage_app`, or `wait_for_delay`), you MUST write a single, fluid narrative paragraph of text as your raw thinking. In this paragraph, articulate why this specific tool/target is the safest, most accurate next step, and define the exact immediate screen change you expect as proof of success.", - "troubleshooter_template": "# ROLE\nYou are the Expert Android Device Troubleshooter. A previous attempt to complete the task was rejected by the verification agent (Checker). Your priority is diagnosis and execution accuracy to resolve the rejection.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen. For any task requiring continuous range interactions, sliding, or boundary selection, prefer using targeted swipe/drag gestures spanning from the element's start bounds [xmin, ymin] to end bounds [xmax, ymax] to ensure the action is strictly confined to the target element. For drag-and-drop, list reordering, or sliding sliders/SeekBars, set duration >= 1000 (e.g. 1500) and always drag slightly PAST the target position to overcome touch slop and reliably trigger the drop/snap/value update. When setting a slider to Maximum (100%) or Minimum (0%), always swipe fully to the extreme boundary to guarantee reaching the absolute limits.\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n- **Tool Calling Iron Rule**: Do NOT submit a Turn-Ending Action at the same time (in the same tool-call list) as a Pre-Decision Exploratory Tool. You must first gather information (e.g., finding coordinates or diagnosing a state), manage ADB background tasks, or update memory notes. You can use pre-decision exploratory tools to complete this task. You will be prompted again with the tool's result, at which point you can output your final physical/turn-ending action.\n- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n- **Memory Preservation & Paginated Exploration**: Using note tools is entirely zero-cost. Actively use this tool to maintain goals and logs. The note tool is the only reliable way to persist memory. Therefore, when you find information on the screen that might be relevant to the task, even if you are not certain, you should first save the information currently visible on the screen before exploring further. If you have already recorded some information, you can use `append_note` to append the new information to the existing content. The note tool can also help you organize lists or information across multiple screens; you can consolidate all information into notes for unified processing.\n- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor. Calibrate your scroll distance so that this anchor remains visible near the top of the subsequent screen, creating a seamless visual overlap that guarantees zero missed items. Terminate exploration when a definitive boundary is reached (e.g., list bottom reached or no new content appears after scrolling).\n 2. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 3. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\". Only perform incremental re-verification if the list has been refreshed or content has mutated, while maintaining strict single-pass handling of new events.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] into note (Exit: ; Interval: )`, adhere strictly to this grammar contract and 5-step cyclic state machine:\n 1. **Grammar & Invariant Contract**:\n - `[Loop]`: Represents a multi-turn lifecycle container. The top-level milestone MUST remain active (`[/]`) and must never be marked `[x]` by single checks.\n - ``: The persistent baseline and diffing ledger. You must initialize it in Round 1 and cross-reference/append to it across cycles.\n - `Exit: Continuous until manually stopped`: Strict unbounded mode. Never self-terminate or mark the top-level milestone as `[x]`; cycle continuously until external stop instruction is injected.\n - `Exit: `: Bounded mode. Only mark the top-level milestone as `[x]` once the explicit count/target threshold is visually achieved.\n - `Interval: `: Declared polling cadence. You MUST strictly adhere to the exact duration declared in `` (convert minutes to milliseconds accurately for `wait_for_delay`).\n 2. **The 5-Step Cyclic State Machine (SOP)**:\n - **Step 1 (Baseline Sync)**: On initial cycle, observe all visible entries/metrics in the primary monitoring view and save full snapshot into ``.\n - **Step 2 (Pacing & Refresh)**: Respect `` and execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n- **Short-Term Memory**: short_term_memory is a supplement to the task plan. If needed, use the tag in your raw thinking text to record it. Short-term memory will be passed to the agent in the next turn, you can describe any trivial details, it should at least include expectations for the future, the current short-term strategy, and verified facts. Example:\n\nExpectation: Duplicate targets were not found in e-h, now start looking for items starting with i-l. (Letters after m are still unverified); Operation strategy: Search in batches by starting letter; Verified facts: a-d, e-h have no matches.\n\n\n# SYSTEM WORKFLOW: SAFETY NET & AUTO-RECOVERY\nYour decisions are processed through downstream pipeline guardians to guarantee accuracy:\n- **Pre-Execution Safety Net**: Automatically checks that your target is visible and safe to interact with prior to action execution.\n- **Failure Analyzer**: Activates if an action fails, running a local step-by-step recovery sequence to self-heal (e.g., closing keyboards or handling wait states).\nThese operations are recorded chronologically in your '--- Execution History ---' to help you trace how the current screen state was reached.\n\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Pre-Decision Exploratory Tools**:\n - *What they are*: Helper/Subagent tools (such as `ask_explorer`, `ask_diagnoser`, `video_analyzer`), ADB command tools (`run_adb_command`, `manage_task`), and memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`, `append_note`).\n - *How they work*: These tools gather details or update/read memory. When you invoke these tools, the framework will immediately execute them and return the results, allowing you to continue thinking and make your final decision.\n2. **Turn-Ending Actions**:\n - *What they are*: Physical device actions (`click`, `input_text`, `swipe`, `press_key`, `manage_app`, `wait_for_delay`, `long_press`) and flow-ending actions (`reply_to_checker`).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, proceed through these three phases in sequence.\n\n## Phase 1: Observation & Diagnosis\n1. **Analyze Transition & Rejection**: Compare the current screen and the 'Visible UI Elements' list with the last step's action context, and read the Checker's feedback to evaluate the discrepancy and understand the rejection reasons. If there is short-term memory, that will also be highly valuable information.\n - *Transition & Rejection Analysis*: Identify the exact changes between screen states and note potential unexpected events (app crashes, autoplay transitions, redirects, or UI state drift).\n - *Drift/Regression Audit*: Spot if the current screen has drifted to an unintended view, or unexpectedly Regressed back to an earlier page we already left (e.g., due to an accidental browser back-navigation).\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen/stuck screen?\n - *Missed Transient States*: If the current state jumped ahead and you suspect you missed an important initial or intermediate screen (e.g., a number or flash message that was overwritten by your last action), consider using the `video_analyzer` tool to look back at the recent history.\n - *Challenge Checker*: If you visually confirm the subgoal's target state is *already achieved*, you **MUST** invoke the `reply_to_checker` tool to state your observation.\n2. **Path Decision (Recover vs. Adapt)**: Evaluate why the previous attempt failed and explore multiple problem-solving paths. First, determine if the original path can still be salvaged (e.g., resubmitting input correctly, bypassing a blocker, or re-navigating). If the original method is unreliable, do not stubbornly repeat it—flexibly pivot to an alternative approach or a different UI path to satisfy the Checker. Save discoveries and lessons in notes.\n3. **Verify & Diagnose (Mandatory Diagnosis Triggers & Loop Exemptions)**: You **MUST** use the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage (before calling any Turn-Ending Actions) if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and are EXEMPT from triggering mandatory diagnosis, provided you are making progress or adhering to the planned loop pacing.\n - **Validation/Checker Rejection**: The verification agent (Checker) rejected your subgoal completion (i.e., you are in troubleshooter mode and this is a retry).\n - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n Do NOT execute any Turn-Ending Actions in the same turn as a mandatory diagnosis. Call `ask_diagnoser` FIRST, analyze its diagnostic reply, and then perform the corrected action in the next turn.\n\n## Phase 2: Planning & Memory Preservation\n1. **Autonomous Goal Management & Dynamic Live-Iteration Tracking**: You have autonomy over nested subgoals under the top-level milestones. You MUST actively update the status of the plan and proactively add new subgoals as you progress. Use `[/]` to denote the currently active milestone and subgoal. Only mark a step as completed (`[x]`) AFTER visual confirmation. Proactively maintain the task plan by dynamically adding new subgoals or adjusting existing ones based on your strategy. Keep the plan as a pure Markdown checklist in `task_plan`.\n - *Milestone Structure Preservation*: You are strictly forbidden from deleting, omitting, or modifying top-level `[Loop]` milestones or their declared exit conditions.\n - *Dynamic Live-Iteration Ledger in Loops*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), dynamically log each concrete iteration item in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle (e.g., ` - [x] Polling Check #1 (10:00): Refreshed, no new items, waiting 5m`). This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a loop, evaluate whether the milestone's Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit). For continuous monitoring milestones where Exit is `until manually stopped`, the milestone MUST remain `[/]` and CANNOT be marked `[x]` by you. Once a bounded exit condition is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**\n2. **Preserve Memory & Pre-Action Snapshotting**: You MUST perform the following risk assessment before any UI interaction: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain if you will need these details, you should immediately save the extracted critical values to the note tools to ensure safe downstream access. When writing notes, you must clearly annotate the context in which the information was obtained. Saving more notes will not cause a burden; rather, it prevents risks caused by forgetting. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated. Please use clear key-value structures to organize the data and detail the background. If you have already missed certain critical information and cannot re-view it by going back or other operations, you can use the `video_analyzer` to look back at historical information.\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data.\n2. **Action Selection & Mandatory Cognitive Protocol**: Before calling any Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`, `press_key`, `manage_app`, or `wait_for_delay`), you MUST write a single, fluid narrative paragraph of text as your raw thinking. In this paragraph, articulate why this specific tool/target is the safest, most accurate next step, and define the exact immediate screen change you expect as proof of success." + "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.[% if \"swipe\" in available_tools %] For any task requiring continuous range interactions, sliding, or boundary selection, prefer using targeted swipe/drag gestures spanning from the element's start bounds [xmin, ymin] to end bounds [xmax, ymax] to ensure the action is strictly confined to the target element. For drag-and-drop, list reordering, or sliding sliders/SeekBars, set duration >= 1000 (e.g. 1500) and always drag slightly PAST the target position to overcome touch slop and reliably trigger the drop/snap/value update. When setting a slider to Maximum (100%) or Minimum (0%), always swipe fully to the extreme boundary to guarantee reaching the absolute limits.[% endif %]\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n- **Tool Calling Contract**: Tools that return information needed for the decision ([[ tool_enum(pre_decision_tools, \"and\") ]]) are result-dependent Pre-Decision Tools. Do not submit them in the same tool-call list as a Turn-Ending Action; inspect their result first. The write-through tools `update_note` and `append_note` may accompany at most one Turn-Ending Action when their content is based entirely on evidence already available before that action. Never write an action's expected outcome as if it had already been observed.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Memory Preservation & Paginated Exploration**: Using note tools is entirely zero-cost. Actively use this tool to maintain goals and logs. The note tool is the only reliable way to persist memory. Therefore, when you find information on the screen that might be relevant to the task, even if you are not certain, you should first save the information currently visible on the screen before exploring further. If you have already recorded some information, you can use `append_note` to append the new information to the existing content. The note tool can also help you organize lists or information across multiple screens; you can consolidate all information into notes for unified processing.\n- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] into note (Exit: ; Interval: )` or `[Loop:continuous] into note (Interval: )`, adhere strictly to this grammar contract and 5-step cyclic state machine:\n 1. **Grammar & Invariant Contract**:\n - `[Loop]` / `[Loop:continuous]`: Represents a multi-turn lifecycle container. The top-level milestone MUST remain active (`[/]`) and must never be marked `[x]` by single checks.\n - ``: The persistent baseline and diffing ledger. You must initialize it in Round 1 and cross-reference/append to it across cycles.\n - `[Loop:continuous]`: Strict unbounded mode. Never self-terminate or mark the top-level milestone as `[x]`; the system mechanically rejects premature completion. Cycle continuously until the user injects an explicit stop signal.\n - `Exit: `: Bounded mode. Only mark the top-level milestone as `[x]` once the explicit count/target threshold is visually achieved.\n - `Interval: `: Declared polling cadence. You MUST strictly adhere to the exact duration declared in ``[% if \"wait_for_delay\" in available_tools %] (convert minutes to milliseconds accurately for `wait_for_delay`)[% endif %].\n 2. **The 5-Step Cyclic State Machine (SOP)**:\n - **Step 1 (Baseline Sync)**: On initial cycle, observe all visible entries/metrics in the primary monitoring view and save full snapshot into ``.\n - **Step 2 (Pacing & Refresh)**: Respect `` and execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n- **Short-Term Memory**: short_term_memory is a supplement to the task plan. If needed, use the tag in your raw thinking text to record it. Short-term memory will be passed to the agent in the next turn, you can describe any trivial details, it should at least include expectations for the future, the current short-term strategy, and verified facts. Example:\n\nExpectation: Duplicate targets were not found in e-h, now start looking for items starting with i-l. (Letters after m are still unverified); Operation strategy: Search in batches by starting letter; Verified facts: a-d, e-h have no matches.\n\n\n# SYSTEM WORKFLOW: SAFETY NET & AUTO-RECOVERY\nYour decisions are processed through downstream pipeline guardians to guarantee accuracy:\n- **Pre-Execution Safety Net**: Automatically checks that your target is visible and safe to interact with prior to action execution.\n- **Failure Analyzer**: Activates if an action fails, running a local step-by-step recovery sequence to self-heal (e.g., closing keyboards or handling wait states).\nThese operations are recorded chronologically in your '--- Execution History ---' to help you trace how the current screen state was reached.\n\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Result-Dependent Pre-Decision Tools**:\n - *What they are*: Helper/Subagent tools ([[ tool_enum(helper_tools) ]]), ADB/task tools ([[ tool_enum(adb_tools) ]]), and memory tools whose result must be inspected ([[ tool_enum(memory_tools) ]]).\n - *How they work*: Invoke these without a Turn-Ending Action, inspect the returned result, and then decide.\n2. **Write-Through Memory Tools**:\n - `update_note` and `append_note` may run alongside at most one Turn-Ending Action only when recording facts already observed in the current context. If the note content depends on the action's result, wait for the next observation before writing it.\n3. **Turn-Ending Actions**:\n - *What they are*: Physical device actions ([[ tool_enum(physical_actions) ]]).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, proceed through these three phases in sequence.\n\n## Phase 1: Observation & Diagnosis\n1. **Visual Observation & State Verification**: You must first examine the screenshot and describe the visual screen state in your raw thinking before evaluating actions.\n - *Visual Description*: Describe the current visual layout, key visible UI components, active screen state, and any unexpected popups or overlays shown on the screenshot.\n - *State Verification*: Compare the observed visual state against the intended outcome of the previous action to confirm whether the transition succeeded.\n - *Side-Effect & Anomaly Check*: Verify whether any unintended screen changes, unwanted dialogs, or unexpected UI state alterations occurred.\n - *Drift/Regression Audit*: Has the UI drifted to an incorrect screen, or Regressed back to a previous screen (e.g. from an accidental browser back-navigation)?\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen screen (static stuck)?\n[% if \"video_analyzer\" in available_tools %] - *Missed Transient States*: If the current state jumped ahead and you suspect you missed an important initial or intermediate screen (e.g., a number or flash message that was overwritten by your last action), consider using the `video_analyzer` tool to look back at the recent history.\n[% endif %]2. **Path Decision (Recover vs. Adapt)**: Encourage flexible problem-solving. First, evaluate if the current path can still be recovered or advanced (e.g., dismissing a blocker, retrying, or restoring the view). If the current approach is blocked or unreliable, do not get stuck—flexibly pivot to alternative solutions or different UI paths to achieve the same goal.\n3. **Verify & Diagnose (Mandatory Diagnosis Triggers & Loop Exemptions)**: You **MUST** use the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage (before calling any Turn-Ending Actions) if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and are EXEMPT from triggering mandatory diagnosis, provided you are making progress or adhering to the planned loop pacing.\n{% if verification_active %} - **Ambiguous Validation Rejection or Verification Finding**: Use `ask_diagnoser` only when the rejection or finding's cause is unclear, the evidence conflicts, or no safe local correction is evident. A rejection or verification finding alone is not a mandatory diagnosis trigger.\n{% endif %} - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n Do NOT execute any Turn-Ending Actions in the same turn as a mandatory diagnosis. Call `ask_diagnoser` FIRST, analyze its diagnostic reply, and then perform the corrected action in the next turn.\n\n## Phase 2: Planning & Memory Preservation\n1. **Autonomous Goal Management & Dynamic Live-Iteration Tracking**: You have autonomy over nested subgoals under the top-level milestones. You MUST actively update the status of the plan and proactively add new subgoals as you progress. Use `[/]` to denote the currently active milestone and subgoal. Only mark a step as completed (`[x]`) AFTER visual confirmation. Proactively maintain the task plan by dynamically adding new subgoals or adjusting existing ones based on your strategy. Keep the plan as a pure Markdown checklist in `task_plan`. Its grammar is machine-enforced:\n{{ plan_grammar }}\n - *Milestone Structure Preservation*: You are strictly forbidden from deleting, omitting, or modifying top-level `[Loop]`/`[Loop:continuous]` milestones or their declared exit conditions.\n - *Surgical Plan Edits*: Modify `task_plan` with targeted `update_note` (target/replacement) edits. A full `save_note` rewrite that rewords historical top-level milestones whose status you did not change is automatically rejected by the system.\n - *Dynamic Live-Iteration Ledger in Loops*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), dynamically log each concrete iteration item in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle (e.g., ` - [x] Polling Check #1 (10:00): Refreshed, no new items, waiting 5m`). This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a loop, evaluate whether the milestone's Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit). For `[Loop:continuous]` milestones, the milestone MUST remain `[/]` and CANNOT be marked `[x]` by you until the user injects an explicit external stop signal. Once a bounded exit condition is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**\n2. **Preserve Memory & Pre-Action Snapshotting**: You MUST perform the following risk assessment before any UI interaction: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain if you will need these details, you should immediately save the extracted critical values to the note tools to ensure safe downstream access. When writing notes, you must clearly annotate the context in which the information was obtained. Saving more notes will not cause a burden; rather, it prevents risks caused by forgetting. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated. Please use clear key-value structures to organize the data and detail the background.[% if \"video_analyzer\" in available_tools %] If you have already missed certain critical information and cannot re-view it by going back or other operations, you can use the `video_analyzer` to look back at historical information.[% endif %]\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data.\n2. **Action Selection & Mandatory Cognitive Protocol**: Before calling any Turn-Ending Action ([[ tool_enum(turn_ending_actions, \"or\") ]]), you MUST write a single, fluid narrative paragraph of text as your raw thinking. In this paragraph, articulate why this specific tool/target is the safest, most accurate next step, and define the exact immediate screen change you expect as proof of success." } diff --git a/artemis/agents/operator/operator.py b/artemis/agents/operator/operator.py index e6f6fc6..09c9e00 100644 --- a/artemis/agents/operator/operator.py +++ b/artemis/agents/operator/operator.py @@ -12,21 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio import json -from pathlib import Path import re -from typing import Annotated, Any, Literal +from typing import Any from uuid import uuid4 from langchain_core.messages import SystemMessage, ToolMessage -from langchain_core.tools import BaseTool, tool -from langgraph.types import Command +from langchain_core.tools import BaseTool -from artemis.constants import ( - OPERATOR_MAX_CHAT_ROUNDS, - VALIDATOR_MESSAGES_KEY, -) from artemis.context import ArtemisContext from artemis.data_engine.trace import ( TraceSpan, @@ -34,7 +27,9 @@ from artemis.data_engine.trace import ( trace_langchain_tool, ) from artemis.graph.state import State -from artemis.services.llm import get_llm, invoke_llm_with_timeout_message +from artemis.graph.visibility import strict_state +from artemis.mcp.action_specs import OPERATOR_SHELL_ORDER, operator_shell_tool +from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message from artemis.tools.command_tool import ( _BACKGROUND_TASKS, _FINISHED_TASKS_LOGS, @@ -51,19 +46,12 @@ from artemis.utils.coordinates import ( parse_swipe_parameters, ) from artemis.utils.decorators import wrap_with_callbacks -from artemis.utils.file import create_snapshot from artemis.utils.logger import get_logger -from artemis.utils.notes import get_note_file_path, get_notes_dir +from artemis.utils.notes import get_note_file_path from artemis.utils.task_tree import ( build_plan_and_history, get_active_subgoal_hashes, ) -from artemis.utils.verification import ( - append_verification_chat, - get_verification_chat_path, - get_verification_chat_rounds, - read_verification_chat, -) from artemis.utils.visualization import format_minimal_list_with_elements logger = get_logger(__name__) @@ -86,7 +74,8 @@ from artemis.agents.operator.prompts import ( TemplatePromptComponent, ObservationPromptComponent, ScreenshotSimilarityPromptComponent, - CheckerFeedbackPromptComponent, + FeedbackPromptComponent, + CheckItemsExplainerPromptComponent, BackgroundTasksPromptComponent, ShortTermMemoryPromptComponent, TaskPlanWarningPromptComponent, @@ -169,56 +158,6 @@ class OperatorNode: return steps, task_plan - def _get_verification_chat_rounds(self) -> tuple[int, int]: - if not self.ctx.data_engine: - return 0, 0 - - subgoal_hash = self._get_active_subgoal_hash() - chat_path = get_verification_chat_path(self.ctx.data_engine.base_dir, subgoal_hash) - turns = read_verification_chat(chat_path) - return get_verification_chat_rounds(turns) - - def _has_checker_feedback(self) -> bool: - if not self.ctx.data_engine: - return False - - subgoal_hash = self._get_active_subgoal_hash() - chat_path = get_verification_chat_path(self.ctx.data_engine.base_dir, subgoal_hash) - return chat_path.exists() - - def _get_reply_to_checker_tool(self) -> BaseTool: - @tool - def reply_to_checker( - reasoning: Annotated[ - str, - "Your reasoning or observations regarding the verification feedback.", - ], - ): - """[TERMINAL] Use this tool to reply to the verification feedback with your reasoning or observations. - - This ends your turn. - """ - - if self.ctx.data_engine: - subgoal_hash = self._get_active_subgoal_hash() - chat_path = get_verification_chat_path(self.ctx.data_engine.base_dir, subgoal_hash) - - turns = read_verification_chat(chat_path) - max_op, max_chk = get_verification_chat_rounds(turns) - - round_num = max(max_op + 1, max_chk) - if round_num == 0: - round_num = 1 - - if append_verification_chat(chat_path, "operator", reasoning, round_num): - return "Successfully recorded your reply to the Checker." - else: - return "Failed to save reply." - else: - return "Error: DataEngine not available." - - return reply_to_checker - async def _build_prompt( self, state: State, @@ -235,39 +174,26 @@ class OperatorNode: builder = PromptBuilder() - has_feedback = self._has_checker_feedback() - + # One constant component list: the prompt template is never switched by + # verification results. Check-related components are append-only and + # render nothing when there is nothing to say. components = self.prompt_components if not components: - if has_feedback: - components = [ - ( - TemplatePromptComponent(), - {"template_name": "troubleshooter_template"}, - ), - (ObservationPromptComponent(), {}), - (ScreenshotSimilarityPromptComponent(), {}), - (InjectedInstructionPromptComponent(), {}), - (ShortTermMemoryPromptComponent(), {}), - (BackgroundTasksPromptComponent(), {}), - (CheckerFeedbackPromptComponent(), {}), - (TaskPlanWarningPromptComponent(), {}), - (ToolLimitWarningPromptComponent(), {}), - ] - else: - components = [ - ( - TemplatePromptComponent(), - {"template_name": "main_template"}, - ), - (ObservationPromptComponent(), {}), - (ScreenshotSimilarityPromptComponent(), {}), - (InjectedInstructionPromptComponent(), {}), - (ShortTermMemoryPromptComponent(), {}), - (BackgroundTasksPromptComponent(), {}), - (TaskPlanWarningPromptComponent(), {}), - (ToolLimitWarningPromptComponent(), {}), - ] + components = [ + ( + TemplatePromptComponent(), + {"template_name": "main_template"}, + ), + (CheckItemsExplainerPromptComponent(), {}), + (ObservationPromptComponent(), {}), + (ScreenshotSimilarityPromptComponent(), {}), + (InjectedInstructionPromptComponent(), {}), + (ShortTermMemoryPromptComponent(), {}), + (BackgroundTasksPromptComponent(), {}), + (FeedbackPromptComponent(), {}), + (TaskPlanWarningPromptComponent(), {}), + (ToolLimitWarningPromptComponent(), {}), + ] else: components = [(c, {}) for c in components] + [(ToolLimitWarningPromptComponent(), {})] @@ -298,31 +224,13 @@ class OperatorNode: traced_tools: list, new_subagent_calls: list, state: State, - ) -> tuple[list[dict] | None, bool, str | None, str | None, bool]: + ) -> tuple[list[dict] | None, str | None, str | None, bool]: max_iterations = 20 action_result = None - requested_argue = False raw_thoughts = [] native_thoughts = [] tool_limit_exceeded = False - async def read_stream_and_accumulate(target_llm, msgs): - try: - full_response = None - has_chunks = False - async for chunk in target_llm.astream(msgs): - has_chunks = True - if full_response is None: - full_response = chunk - else: - full_response += chunk - if has_chunks and full_response is not None: - return full_response - except Exception as e: - logger.debug(f"astream failed or not supported: {e}. Falling back to ainvoke.") - - return await target_llm.ainvoke(msgs) - for iteration in range(max_iterations): if iteration == max_iterations - 1: logger.warning( @@ -340,9 +248,7 @@ class OperatorNode: ) bound_llm = base_llm.bind_tools(tools=traced_tools) - response = await invoke_llm_with_timeout_message( - read_stream_and_accumulate(bound_llm, current_messages) - ) + response = await invoke_llm_with_timeout_message(acomplete(bound_llm, current_messages)) if hasattr(response, "response_metadata") and response.response_metadata: usage = ( @@ -420,23 +326,19 @@ class OperatorNode: action_calls = [ tc for tc in response.tool_calls if normalize_name(tc["name"]) in action_tool_names ] - reply_to_checker_calls = [ - tc for tc in response.tool_calls if normalize_name(tc["name"]) == "reply_to_checker" - ] other_calls = [ tc for tc in response.tool_calls - if normalize_name(tc["name"]) not in action_tool_names + ["reply_to_checker"] + if normalize_name(tc["name"]) not in action_tool_names ] tool_outputs = [] validation_errors = False - requested_argue = False other_tool_failed = False other_tool_failure_msg = "" - has_terminal_calls = bool(action_calls or reply_to_checker_calls) + has_terminal_calls = bool(action_calls) if other_calls: results = [] @@ -478,12 +380,8 @@ class OperatorNode: content = get_tool_result_content(result_obj) status = "success" - if isinstance(result_obj, Command): - updates = result_obj.update - if VALIDATOR_MESSAGES_KEY in updates: - msgs = updates[VALIDATOR_MESSAGES_KEY] - if msgs and hasattr(msgs[0], "status"): - status = msgs[0].status + if isinstance(result_obj, ToolMessage) and result_obj.status: + status = result_obj.status is_err = status == "error" if not is_err: @@ -567,29 +465,6 @@ class OperatorNode: ) action_calls = [] - if reply_to_checker_calls: - logger.warning( - "Scratchpad/helper tool call failed. Deferring" - f" reply to checker: {other_tool_failure_msg}" - ) - for tc in reply_to_checker_calls: - tool_outputs.append( - ToolMessage( - tool_call_id=tc["id"], - content=( - "Your reply to the checker was" - " rejected because an accompanying" - " tool call failed. Error:" - f" {other_tool_failure_msg}. Currently," - " no response has been sent to the" - " checker. Please correct the error" - " and try again." - ), - status="error", - ) - ) - reply_to_checker_calls = [] - elif has_terminal_calls: has_deferring_calls = any( normalize_name(tc["name"]) in DEFERRING_TOOLS for tc in other_calls @@ -623,28 +498,7 @@ class OperatorNode: ) ) - for tc in reply_to_checker_calls: - tool_outputs.append( - ToolMessage( - tool_call_id=tc["id"], - content=( - "Your pre-decision tools have been" - " successfully processed. However, your" - " response to the checker was rejected" - " because you are not allowed to" - " simultaneously use result-dependent" - " pre-decision tools and reply to the" - " checker. Currently, no response has been" - " sent to the checker. Please review" - " your updated context and re-output" - " your reply." - ), - status="success", - ) - ) - action_calls = [] - reply_to_checker_calls = [] if action_calls: logger.info( @@ -676,32 +530,10 @@ class OperatorNode: if not validation_errors: action_result = actions_list - if reply_to_checker_calls: - logger.info("Operator replied to checker.") - tc = reply_to_checker_calls[0] - requested_argue = True - - # Execute it immediately - result = await self._run_other_tool( - tc, - traced_tools, - new_subagent_calls, - state, - raw_thoughts, - native_thoughts, - ) - tool_outputs.append( - ToolMessage( - tool_call_id=tc["id"], - content=result, - status="success" if not result.startswith("Error") else "error", - ) - ) - for tm in tool_outputs: current_messages.append(tm) - if (action_calls and not validation_errors) or requested_argue: + if action_calls and not validation_errors: break if validation_errors: @@ -714,7 +546,6 @@ class OperatorNode: native_thinking = "\n".join(native_thoughts) if native_thoughts else None return ( action_result, - requested_argue, raw_thinking, native_thinking, tool_limit_exceeded, @@ -765,18 +596,10 @@ class OperatorNode: ) @trace(type="agent", name="operator") async def __call__(self, state: State): + state = strict_state(state, "operator") # 1. Check for infinite loops self._check_infinite_loop(state) - # Create snapshot for optimistic execution - if self.ctx.data_engine: - notes_dir = get_notes_dir(self.ctx.data_engine.base_dir) - if notes_dir.exists(): - snapshot_dir = notes_dir.with_name("notes_snapshot") - await asyncio.to_thread(create_snapshot, notes_dir, snapshot_dir) - self.ctx.task_plan_snapshot = snapshot_dir - logger.info(f"Created snapshot of notes at {snapshot_dir}") - # 2. Get perception data from state (populated by Perception node) operator_raw_data = getattr(state, "operator_raw_data", {}) or {} latest_screenshot_b64 = operator_raw_data.get("screenshot_b64") @@ -822,45 +645,16 @@ class OperatorNode: current_step_num = len(steps) + 1 - # 5. Prepare Tools. Device-action declarations are assembled against the - # installed actuator backend's capabilities: an action the backend does not - # implement is simply never declared (and the prompt assembly drops its - # teaching segments in lockstep). + # 5. Prepare Tools. Device-action shells come from the canonical manifest + # (artemis/mcp/action_specs.py) and are assembled against the installed + # actuator backend's capabilities: an action the backend does not implement + # is simply never declared (and the prompt assembly drops its teaching + # segments in lockstep). available_actions = self._available_device_actions() - action_tool_factories = { - "click": self._get_click_tool, - "input_text": self._get_input_text_tool, - "swipe": self._get_swipe_tool, - "press_key": self._get_press_key_tool, - "manage_app": self._get_manage_app_tool, - "wait_for_delay": self._get_wait_for_delay_tool, - "long_press": self._get_long_press_tool, - } all_tools = [ - factory() - for name, factory in action_tool_factories.items() - if name in available_actions + operator_shell_tool(name) for name in OPERATOR_SHELL_ORDER if name in available_actions ] + self.tools - if self._has_checker_feedback(): - max_op, max_chk = self._get_verification_chat_rounds() - max_rounds = ( - getattr( - getattr(self.ctx, "execution_setup", None), - "checker_max_chat_rounds", - OPERATOR_MAX_CHAT_ROUNDS, - ) - if self.ctx and getattr(self.ctx, "execution_setup", None) - else OPERATOR_MAX_CHAT_ROUNDS - ) - if max_op < max_rounds: - reply_to_checker_tool = self._get_reply_to_checker_tool() - all_tools.append(reply_to_checker_tool) - else: - logger.info( - f"Maximum chat rounds ({max_rounds}) reached. Removing reply_to_checker tool." - ) - # 5. Evaluate dynamic tools and prepare background tasks has_long_output = False for tinfo in _FINISHED_TASKS_LOGS.values(): @@ -924,7 +718,6 @@ class OperatorNode: new_subagent_calls = [] ( action_result, - requested_argue, raw_thinking, native_thinking, tool_limit_exceeded, @@ -955,22 +748,17 @@ class OperatorNode: if stm_match: short_term_memory = stm_match.group(1).strip() - return await state.asanitize_update( - ctx=self.ctx, - update={ - "structured_decisions": structured_decisions, - "operator_raw_thinking": raw_thinking, - "operator_native_thinking": native_thinking, - "short_term_memory": short_term_memory, - "indexed_points": state.indexed_points, - "complete_subgoals_by_ids": [], - "current_step_id": state.current_step_id, - "subagent_calls": ((state.subagent_calls or []) + new_subagent_calls), - "operator_replied": requested_argue, - "operator_tool_limit_exceeded": tool_limit_exceeded, - }, - agent="operator", - ) + return { + "structured_decisions": structured_decisions, + "operator_raw_thinking": raw_thinking, + "operator_native_thinking": native_thinking, + "short_term_memory": short_term_memory, + "indexed_points": state.indexed_points, + "indexed_elements": state.indexed_elements, + "current_step_id": state.current_step_id, + "subagent_calls": ((state.subagent_calls or []) + new_subagent_calls), + "operator_tool_limit_exceeded": tool_limit_exceeded, + } def _get_active_subgoal_hash(self) -> str: """Parses task_plan.md to find the active top-level subgoal hash.""" @@ -1007,198 +795,6 @@ class OperatorNode: chronological_last_step=True, ) - def _get_long_press_tool(self) -> BaseTool: - @tool - def long_press( - target: Annotated[ - int | list[int], - "Long press target. Can be an element index number (int, e.g." - " 3) OR normalized coordinates (list of 2 integers, e.g. [500," - " 600]).", - ], - duration: Annotated[int, "Long press duration in milliseconds (default 1000)."] = 1000, - ): - """[ACTION] Long press on the target location on the screen (supports element index or absolute normalized coordinates).""" - return "Action Recorded" - - return long_press - - def _get_click_tool(self) -> BaseTool: - @tool - def click( - target: Annotated[ - int | list[int], - "Click target. Can be an element index number (int, e.g. 3) OR" - " normalized coordinates (list of 2 integers, e.g. [500," - " 600]).", - ], - times: Annotated[ - int, - "Number of consecutive clicks on this target. Use this for" - " double-clicks or multi-clicks (e.g. 7 to enter developer" - " mode). Default is 1.", - ] = 1, - delay_ms: Annotated[ - int, - "Delay in milliseconds between consecutive clicks. Default is 100.", - ] = 100, - ): - """[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates).""" - return "Action Recorded" - - return click - - def _get_input_text_tool(self) -> BaseTool: - @tool - def input_text( - text: Annotated[ - str, - "The text content to input. Supports multi-line content with '\\n'.", - ], - target: Annotated[ - int | list[int], - "Input target field. Can be an input box element index number" - " (int, e.g. 3) OR normalized coordinates (list of 2 integers," - " e.g. [500, 600]).", - ], - clear_exist: Annotated[ - bool, - "Whether to clear existing text before typing. True (default):" - " clear/replace entire text. False: append at the end of" - " existing content.", - ] = True, - ): - """[ACTION] Type text into the target input field (supports replacing whole text or appending to the end, and multi-line strings with '\\n').""" - return "Action Recorded" - - return input_text - - def _get_swipe_tool(self) -> BaseTool: - @tool - def swipe( - direction: Annotated[ - Literal["up", "down", "left", "right"] | None, - "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below)," - " 'down' (drags top-to-bottom, scrolling up to reveal content above)," - " 'left' (drags right-to-left, scrolling right)," - " 'right' (drags left-to-right, scrolling left).", - ] = None, - start: Annotated[ - list[int] | None, - "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise," - " local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop).", - ] = None, - end: Annotated[ - list[int] | None, - "End normalized coordinates [end_x, end_y] in 0-1000 scale for precise," - " local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop).", - ] = None, - target: Annotated[ - int | list[int] | str | None, - "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.", - ] = None, - gesture: Annotated[ - Literal["up", "down", "left", "right"] | list[int] | None, - "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right')" - " OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale.", - ] = None, - duration: Annotated[ - int | None, - "Optional swipe/drag duration in milliseconds (default 800). For drag-and-drop," - " list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars)," - " set duration >= 1000 (e.g. 1500). If omitted for directional swipe, duration is computed automatically.", - ] = None, - ): - """[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen. - - • Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly. - • Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary. - - Args: - direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'. - start: Start normalized coordinates [start_x, start_y] in 0-1000 scale. - end: End normalized coordinates [end_x, end_y] in 0-1000 scale. - target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within. - gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale. - duration: Optional gesture duration in milliseconds (default 800). - """ - return "Action Recorded" - - return swipe - - def _get_press_key_tool(self) -> BaseTool: - @tool - def press_key( - key: Annotated[ - Literal["ENTER", "BACK", "HOME", "APP_SWITCH"], - "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH).", - ], - ): - """[ACTION] Press a physical or virtual system button (e.g. - - ENTER, BACK, HOME, APP_SWITCH). - """ - return "Action Recorded" - - return press_key - - def _get_manage_app_tool(self) -> BaseTool: - @tool - def manage_app( - action: Annotated[Literal["launch", "stop"], "The action type."], - app_name: Annotated[str, "Display name or package name of the application."], - ): - """[ACTION] Launch or force stop a specified application.""" - return "Action Recorded" - - return manage_app - - def _get_wait_for_delay_tool(self) -> BaseTool: - @tool - def wait_for_delay( - time_in_ms: Annotated[ - int, - "The exact duration to wait in milliseconds. Accurately convert the" - " required time duration into milliseconds based on your objective" - " or plan (e.g., 2000 for 2s, 5000 for 5s, 60000 for 1 minute," - " 180000 for 3 minutes, 300000 for 5 minutes).", - ], - ): - """[ACTION] Pause execution and wait for a specified duration in milliseconds. - - Use this whenever you need time to elapse—whether for UI loading, animations, - screen transitions, or longer scheduled delays and intervals specified in the task. - """ - return "Action Recorded" - - return wait_for_delay - - def _get_wait_for_text_tool(self) -> BaseTool: - @tool - def wait_for_text( - text: Annotated[ - str, - "A simple, distinct keyword (e.g., 'Success', 'Done'," - " 'Loading') to watch on the screen. Try to pass a clean word" - " without trailing punctuation (e.g. 'Loading' instead of" - " 'Loading...') for optimal matching reliability.", - ], - state: Annotated[ - Literal["visible", "hidden"], - "Wait for the keyword to appear ('visible') or disappear" - " ('hidden') from the screen.", - ] = "visible", - timeout_ms: Annotated[int, "Maximum timeout in milliseconds (default 5000)."] = 5000, - ): - """[ACTION] Pause execution and wait intelligently for a specific, distinct text or keyword to appear or disappear on the screen. - - Use this when waiting for a specific loading screen to finish or a - specific success/confirmation state to appear. - """ - return "Action Recorded" - - return wait_for_text - def _translate_and_validate_tool(self, tc: dict, state: State) -> tuple[list[dict], str | None]: actions, err = self._translate_and_validate_tool_inner(tc, state) if err: @@ -1384,7 +980,7 @@ class OperatorNode: direction=target, target=args.get("target"), indexed_elements=getattr(state, "indexed_elements", None), - ui_hierarchy=getattr(state, "ui_tree", None), + ui_hierarchy=getattr(state, "latest_ui_hierarchy", None), width=width, height=height, duration=duration, diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index 6b0bdd8..a32d11f 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -14,10 +14,9 @@ import base64 import io -import json from pathlib import Path -from jinja2 import Template +from jinja2 import Environment, StrictUndefined, Template from langchain_core.messages import HumanMessage, SystemMessage from PIL import Image @@ -28,27 +27,27 @@ from artemis.tools.command_tool import ( _is_output_long, ) from artemis.utils.logger import get_logger -from artemis.utils.task_tree import get_active_subgoal_hashes +from artemis.utils.notes import get_note_file_path +from artemis.utils.plan_grammar import parse_plan, render_plan_grammar_spec logger = get_logger(__name__) -from artemis.agents.prompt_assembly import ( - gate_segment, - render_tool_enum, - resolve_available, -) - -_LEGACY_TOOL_CALLING_RULE = """- **Tool Calling Iron Rule**: Do NOT submit a Turn-Ending Action at the same time (in the same tool-call list) as a Pre-Decision Exploratory Tool. You must first gather information (e.g., finding coordinates or diagnosing a state), manage ADB background tasks, or update memory notes. You can use pre-decision exploratory tools to complete this task. You will be prompted again with the tool's result, at which point you can output your final physical/turn-ending action.""" - -_LEGACY_TOOL_PROTOCOL = """1. **Pre-Decision Exploratory Tools**: - - *What they are*: Helper/Subagent tools (such as `ask_explorer`, `ask_diagnoser`, `video_analyzer`), ADB command tools (`run_adb_command`, `manage_task`), and memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`, `append_note`). - - *How they work*: These tools gather details or update/read memory. When you invoke these tools, the framework will immediately execute them and return the results, allowing you to continue thinking and make your final decision. -2. **Turn-Ending Actions**:""" +from artemis.agents.prompt_assembly import render_tool_enum, resolve_available +from artemis.mcp.action_specs import OPERATOR_SHELL_ORDER # --- Assembled tool references ------------------------------------------------------- +# The operator.json templates carry availability slots rendered by +# ``apply_operator_prompt_contract``: ``[[ tool_enum(...) ]]`` expands an ordered, +# backticked enumeration limited to the available tool set, and +# ``[% if "x" in available_tools %]...[% endif %]`` gates a self-contained teaching +# segment. An unavailable tool therefore leaves no trace in the prompt at all. The +# slot delimiters are square-bracketed so the standard ``{{ ... }}`` context +# variables (initial_goal, plan_and_history, plan_grammar, ...) pass through +# untouched for the later context render. +# # The orderings below reproduce the historical prompt wording exactly; with the full -# tool set the rendered output is byte-identical to the previous literals. See +# tool set the rendered output is byte-identical to the pre-assembly literals. See # artemis/agents/prompt_assembly.py for the assembly rationale. _PRE_DECISION_HELPER_TOOLS = ("ask_explorer", "ask_diagnoser", "video_analyzer") @@ -59,16 +58,9 @@ _PRE_DECISION_ALL_TOOLS = ( ) # The two device-action enumeration slots in operator.json (identical in both -# templates), in their historical orders. -_PHYSICAL_ACTIONS_ORDER = ( - "click", - "input_text", - "swipe", - "press_key", - "manage_app", - "wait_for_delay", - "long_press", -) +# templates), in their historical orders. The physical order is the canonical +# manifest's shell-binding order; the turn-ending order is a prompt property. +_PHYSICAL_ACTIONS_ORDER = OPERATOR_SHELL_ORDER _TURN_ENDING_ORDER = ( "click", "swipe", @@ -79,120 +71,31 @@ _TURN_ENDING_ORDER = ( "wait_for_delay", ) -_PHYSICAL_ACTIONS_SLOT = ( - "Physical device actions (`click`, `input_text`, `swipe`, `press_key`," - " `manage_app`, `wait_for_delay`, `long_press`)" -) -_TURN_ENDING_SLOT = ( - "Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`," - " `press_key`, `manage_app`, or `wait_for_delay`)" -) - -# Self-contained instruction segments that teach a single tool. Removed wholesale when -# the tool is unavailable, so nothing in the prompt dangles. -_WAIT_FOR_DELAY_TRANSITIONS_SEGMENT = ( - "- **Loading & Transitions**: When entering page transitions (e.g., app launching," - " web page loading, or progress indicators), use `wait_for_delay` to let the screen" - " settle before executing target actions. Some transition states are difficult to" - " discern, such as a thin progress line at the top of the browser screen during" - " webpage loads. Before questioning whether an error has occurred, you can first" - " adopt a waiting strategy to see how the device state evolves. If the state" - " remains ambiguous after waiting, certain UI components might be malfunctioning or" - " failing to load; in this case, consider retrying the action or pivoting to an" - " alternative path.\n" -) -_WAIT_FOR_DELAY_INTERVAL_PARENTHETICAL = ( - " (convert minutes to milliseconds accurately for `wait_for_delay`)" -) -_VIDEO_ANALYZER_TRANSIENT_SEGMENT = ( - " - *Missed Transient States*: If the current state jumped ahead and you suspect" - " you missed an important initial or intermediate screen (e.g., a number or flash" - " message that was overwritten by your last action), consider using the" - " `video_analyzer` tool to look back at the recent history.\n" -) -_VIDEO_ANALYZER_SNAPSHOT_SENTENCE = ( - " If you have already missed certain critical information and cannot re-view it by" - " going back or other operations, you can use the `video_analyzer` to look back at" - " historical information." -) -# Tail of the "Action Scope & Interaction Precision" bullet: everything past the two -# generic gesture-scope sentences is swipe/drag teaching. -_SWIPE_PRECISION_SEGMENT = ( - " For any task requiring continuous range interactions, sliding, or boundary" - " selection, prefer using targeted swipe/drag gestures spanning from the element's" - " start bounds [xmin, ymin] to end bounds [xmax, ymax] to ensure the action is" - " strictly confined to the target element. For drag-and-drop, list reordering, or" - " sliding sliders/SeekBars, set duration >= 1000 (e.g. 1500) and always drag" - " slightly PAST the target position to overcome touch slop and reliably trigger the" - " drop/snap/value update. When setting a slider to Maximum (100%) or Minimum (0%)," - " always swipe fully to the extreme boundary to guarantee reaching the absolute" - " limits." -) - #: Every tool name the operator prompt slots can reference. ``available_tools=None`` #: resolves to this set, preserving the historical output. OPERATOR_PROMPT_TOOLSET: frozenset[str] = frozenset( _PRE_DECISION_ALL_TOOLS + _PHYSICAL_ACTIONS_ORDER ) - -def _tool_calling_rule(available: frozenset[str]) -> str: - tools = render_tool_enum(_PRE_DECISION_ALL_TOOLS, available, final_sep="and") - return ( - "- **Tool Calling Contract**: Tools that return information needed for the" - f" decision ({tools}) are result-dependent Pre-Decision Tools. Do not submit" - " them in the same tool-call list as a Turn-Ending Action; inspect their result" - " first. The write-through tools `update_note` and `append_note` may accompany" - " at most one Turn-Ending Action when their content is based entirely on" - " evidence already available before that action. Never write an action's" - " expected outcome as if it had already been observed." - ) - - -def _tool_protocol(available: frozenset[str]) -> str: - helper = render_tool_enum(_PRE_DECISION_HELPER_TOOLS, available) - adb = render_tool_enum(_PRE_DECISION_ADB_TOOLS, available) - memory = render_tool_enum(_PRE_DECISION_MEMORY_TOOLS, available) - return ( - "1. **Result-Dependent Pre-Decision Tools**:\n" - f" - *What they are*: Helper/Subagent tools ({helper}), ADB/task tools" - f" ({adb}), and memory tools whose result must be inspected ({memory}).\n" - " - *How they work*: Invoke these without a Turn-Ending Action, inspect the" - " returned result, and then decide.\n" - "2. **Write-Through Memory Tools**:\n" - " - `update_note` and `append_note` may run alongside at most one Turn-Ending" - " Action only when recording facts already observed in the current context. If" - " the note content depends on the action's result, wait for the next" - " observation before writing it.\n" - "3. **Turn-Ending Actions**:" - ) - -_LEGACY_CHECKER_REJECTION_TRIGGER = """ - **Validation/Checker Rejection**: The verification agent (Checker) rejected your subgoal completion (i.e., you are in troubleshooter mode and this is a retry).""" - -_AMBIGUOUS_CHECKER_REJECTION_TRIGGER = """ - **Ambiguous Validation/Checker Rejection**: Use `ask_diagnoser` only when the rejection cause is unclear, the evidence conflicts, or no safe local correction is evident. A Checker rejection alone is not a mandatory diagnosis trigger.""" - -_LEGACY_TROUBLESHOOTER_CHALLENGE = """ - *Challenge Checker*: If you visually confirm the subgoal's target state is *already achieved*, you **MUST** invoke the `reply_to_checker` tool to state your observation.""" - -_TROUBLESHOOTER_CHALLENGE = """ - *Challenge Checker*: If current evidence clearly proves the subgoal is already achieved, invoke `reply_to_checker` directly. This branch takes precedence over diagnosis; do not call `ask_diagnoser` first. - - *Clear Local Correction*: If the rejection cause is explicit and a safe correction is evident from the current screen, perform that correction directly. This branch also takes precedence over generic diagnosis triggers. Use `ask_diagnoser` only for ambiguity, conflicting evidence, or repeated failure.""" - -_LEGACY_LARGE_LIST_STRATEGY = """- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard: - 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor. Calibrate your scroll distance so that this anchor remains visible near the top of the subsequent screen, creating a seamless visual overlap that guarantees zero missed items. Terminate exploration when a definitive boundary is reached (e.g., list bottom reached or no new content appears after scrolling). - 2. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking. - 3. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for "double-checking". Only perform incremental re-verification if the list has been refreshed or content has mutated, while maintaining strict single-pass handling of new events.""" - -_LARGE_LIST_STRATEGY = """- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard: - 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete. - 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary. - 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking. - 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for "double-checking", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.""" +#: Renders the availability slots in operator.json. Square-bracket delimiters keep +#: the standard ``{{ ... }}`` context placeholders inert during this phase. +_CONTRACT_ENV = Environment( + block_start_string="[%", + block_end_string="%]", + variable_start_string="[[", + variable_end_string="]]", + comment_start_string="[#", + comment_end_string="#]", + keep_trailing_newline=True, + undefined=StrictUndefined, +) def apply_operator_prompt_contract( prompt_template: str, available_tools: frozenset[str] | None = None, ) -> str: - """Align prompt-level tool and recovery rules with Operator runtime semantics. + """Renders a template's availability slots against the actually-available tools. Args: prompt_template: One of the raw operator.json templates. @@ -203,65 +106,19 @@ def apply_operator_prompt_contract( """ available = frozenset(resolve_available(available_tools, OPERATOR_PROMPT_TOOLSET)) - prompt_template = prompt_template.replace( - _LEGACY_TOOL_CALLING_RULE, - _tool_calling_rule(available), - ) - prompt_template = prompt_template.replace( - _LEGACY_TOOL_PROTOCOL, - _tool_protocol(available), - ) - prompt_template = prompt_template.replace( - _LEGACY_CHECKER_REJECTION_TRIGGER, - _AMBIGUOUS_CHECKER_REJECTION_TRIGGER, - ) - prompt_template = prompt_template.replace( - _LEGACY_TROUBLESHOOTER_CHALLENGE, - _TROUBLESHOOTER_CHALLENGE, - ) - prompt_template = prompt_template.replace( - _LEGACY_LARGE_LIST_STRATEGY, - _LARGE_LIST_STRATEGY, - ) + def tool_enum(names, final_sep: str | None = None) -> str: + return render_tool_enum(tuple(names), available, final_sep=final_sep) - # Device-action enumeration slots (identical in both templates). - prompt_template = prompt_template.replace( - _PHYSICAL_ACTIONS_SLOT, - "Physical device actions (" - + render_tool_enum(_PHYSICAL_ACTIONS_ORDER, available) - + ")", + return _CONTRACT_ENV.from_string(prompt_template).render( + available_tools=available, + tool_enum=tool_enum, + pre_decision_tools=_PRE_DECISION_ALL_TOOLS, + helper_tools=_PRE_DECISION_HELPER_TOOLS, + adb_tools=_PRE_DECISION_ADB_TOOLS, + memory_tools=_PRE_DECISION_MEMORY_TOOLS, + physical_actions=_PHYSICAL_ACTIONS_ORDER, + turn_ending_actions=_TURN_ENDING_ORDER, ) - prompt_template = prompt_template.replace( - _TURN_ENDING_SLOT, - "Turn-Ending Action (" - + render_tool_enum(_TURN_ENDING_ORDER, available, final_sep="or") - + ")", - ) - - # Instruction segments that teach a single tool. - prompt_template = prompt_template.replace( - _WAIT_FOR_DELAY_TRANSITIONS_SEGMENT, - gate_segment(_WAIT_FOR_DELAY_TRANSITIONS_SEGMENT, available, "wait_for_delay"), - ) - prompt_template = prompt_template.replace( - _WAIT_FOR_DELAY_INTERVAL_PARENTHETICAL, - gate_segment( - _WAIT_FOR_DELAY_INTERVAL_PARENTHETICAL, available, "wait_for_delay" - ), - ) - prompt_template = prompt_template.replace( - _VIDEO_ANALYZER_TRANSIENT_SEGMENT, - gate_segment(_VIDEO_ANALYZER_TRANSIENT_SEGMENT, available, "video_analyzer"), - ) - prompt_template = prompt_template.replace( - _VIDEO_ANALYZER_SNAPSHOT_SENTENCE, - gate_segment(_VIDEO_ANALYZER_SNAPSHOT_SENTENCE, available, "video_analyzer"), - ) - prompt_template = prompt_template.replace( - _SWIPE_PRECISION_SEGMENT, - gate_segment(_SWIPE_PRECISION_SEGMENT, available, "swipe"), - ) - return prompt_template class PromptBuilder: @@ -340,11 +197,22 @@ class TemplatePromptComponent(PromptComponent): plan_and_history = kwargs.get("plan_and_history", "No plan or history yet.") + # Grammar spec assembly is a function of configuration: with both check + # gates disabled, the check-line grammar never enters any prompt. + include_checks = bool(setup and getattr(setup, "checks_enabled", False)) + # The rejection/finding diagnosis trigger only exists while a mechanism + # that can produce rejections or findings is active. + verification_active = include_checks or bool( + setup and not getattr(setup, "disable_planner_validation", True) + ) + full_prompt = Template(prompt_template).render( initial_goal=state.initial_goal, subgoals_status="", plan_and_history=plan_and_history, unified_history="", + plan_grammar=render_plan_grammar_spec(include_checks), + verification_active=verification_active, ) parts = full_prompt.split("# CURRENT OBSERVATION") @@ -369,43 +237,64 @@ class ObservationPromptComponent(PromptComponent): builder.add_human_content(f"--- Visible UI Elements ---\n{minimal_list}") -class CheckerFeedbackPromptComponent(PromptComponent): +class FeedbackPromptComponent(PromptComponent): + """Append-only injection of source-tagged findings. + + Reads ``state.operator_feedback`` (written by ``execution_check_node`` at + harvest / planner rejection and by exit settlement on a bounce-back). + Renders nothing when there are no findings — the prompt template itself + is never switched. + """ + async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisContext, **kwargs): - if not ctx.data_engine: + findings = getattr(state, "operator_feedback", None) + if not findings: return + lines = "\n".join(f"- {f}" for f in findings) + builder.add_human_content( + "--- Verification Findings ---\n" + f"{lines}\n" + "Each finding is tagged with its source. Checker verdicts" + " ([verify failed], [final check]) are independent judgments —" + " address any reverted subgoal accordingly; do not re-litigate" + " them. [planner] findings explain why a plan change was rejected" + " and rolled back — factor the reason into your next strategy." + ) - notes_dir = Path(ctx.data_engine.base_dir) / "notes" - subgoal_hash = "default" - task_plan_path = notes_dir / "task_plan.md" - if task_plan_path.exists(): - try: - content = task_plan_path.read_text(encoding="utf-8") +class CheckItemsExplainerPromptComponent(PromptComponent): + """Behavioral guidance for check lines, rendered iff the CURRENT plan + actually contains check lines (content-driven, not switch-driven: a resumed + plan carrying check lines still gets the explanation).""" - parent_hash, _ = get_active_subgoal_hashes(content) - subgoal_hash = parent_hash - except Exception as e: - logger.error(f"Failed to parse active subgoal in component: {e}") - - verification_chat_path = notes_dir / f"verification_chat_{subgoal_hash}.json" - turns = [] - if verification_chat_path.exists(): - try: - turns = json.loads(verification_chat_path.read_text(encoding="utf-8")) - logger.info(f"Read verification chat for {subgoal_hash}: {len(turns)} turns") - except Exception as e: - logger.error(f"Error reading verification chat: {e}") - - if turns: - dialogue_lines = [] - for t in turns: - role = "Operator" if t["role"] == "operator" else "Checker" - dialogue_lines.append(f"**{role} (Round {t['round']})**:\n{t['content']}") - checker_feedback = "\n\n".join(dialogue_lines) - - feedback_prompt = f"--- Checker Feedback ---\n{checker_feedback}" - - builder.add_human_content(feedback_prompt) + async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisContext, **kwargs): + if not ctx or not getattr(ctx, "data_engine", None): + return + try: + task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") + if not task_plan_path.exists(): + return + snapshot = parse_plan(task_plan_path.read_text(encoding="utf-8")) + except Exception as e: + logger.error(f"Failed to parse plan for check-items explainer: {e}") + return + if not snapshot.all_check_items: + return + builder.add_human_content( + "--- About the plan's check lines ---\n" + "The task plan declares `- verify:` / `- assert:` check lines." + " `verify:` lines are the acceptance criteria for their subgoal — use" + " them to confirm your work is complete, but the judgment is made by" + " an independent Checker: never declare a check passed yourself or" + " record conclusions on its behalf. `assert:` lines are test" + " assertions — take NO extra actions for them and never construct or" + " fake state to satisfy one; a failing assertion is a legitimate test" + " result. Check lines must not be deleted or reworded (deletions are" + " automatically restored by the system); keep them verbatim when" + " rewriting the plan — adding new ones is allowed. Simply mark" + " completions per the plan grammar as usual; checking runs" + " asynchronously in the background and does not block you." + ) class BackgroundTasksPromptComponent(PromptComponent): diff --git a/artemis/agents/outputter/outputter.py b/artemis/agents/outputter/outputter.py index 8b261c7..554eb64 100644 --- a/artemis/agents/outputter/outputter.py +++ b/artemis/agents/outputter/outputter.py @@ -26,6 +26,7 @@ from artemis.config import OutputConfig from artemis.context import ArtemisContext from artemis.data_engine.trace import trace from artemis.graph.state import State +from artemis.llm.structured import ParseFailure, parse_structured from artemis.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback from artemis.tools.scratchpad import ( get_append_note_tool_pure, @@ -141,6 +142,47 @@ async def outputter( ) human_message_content = [{"type": "text", "text": human_message}] + + # Verification material (only present when the run declared check items): + # the full append-only verdict sequence plus the four-state test summary. + if ctx.data_engine: + try: + from artemis.graph.checkpoints import read_ledger, read_run_outcome + + ledger_records = read_ledger(ctx.data_engine.base_dir) + run_outcome = read_run_outcome(ctx.data_engine.base_dir) or getattr( + graph_output, "run_outcome", None + ) + if ledger_records or run_outcome: + lines = ["--- Verification Verdict Ledger (append-only) ---"] + for r in ledger_records: + lines.append( + f"- attempt {r.get('attempt_id')} [{r.get('kind')}]" + f" '{r.get('item_text')}' -> {r.get('status')}:" + f" {r.get('evidence', '')}" + ) + if run_outcome: + tests = run_outcome.get("tests") or {} + lines.append( + "--- Test Summary ---\n" + f"task_status={run_outcome.get('task_status')}," + f" passed={tests.get('passed', 0)}," + f" failed={tests.get('failed', 0)}," + f" inconclusive={tests.get('inconclusive', 0)}," + f" unchecked={tests.get('unchecked', 0)}" + ) + lines.append( + "Report every declared check item with its final state" + " (passed / failed / inconclusive / unchecked) and include" + " the complete verdict sequence above. A failed assertion" + " is a test result to report verbatim, not something to" + " explain away. Items reported unchecked were never" + " judged — never present them as passed." + ) + human_message_content.append({"type": "text", "text": "\n".join(lines)}) + except Exception as e: + logger.error(f"Failed to attach verdict ledger to outputter: {e}") + if screenshot_b64: human_message_content.append( { @@ -306,16 +348,15 @@ async def outputter( # Fallback to old behavior if it didn't match the template if isinstance(raw_answer, str): raw_answer_stripped = raw_answer.strip() - if raw_answer_stripped.startswith("{") and raw_answer_stripped.endswith("}"): - try: - return json.loads(raw_answer_stripped) - except Exception: - pass - elif raw_answer_stripped.startswith("```json") and raw_answer_stripped.endswith("```"): - try: - content = raw_answer_stripped[7:-3].strip() - return json.loads(content) - except Exception: - pass + if raw_answer_stripped.startswith(("{", "[", "```")): + parsed = parse_structured(raw_answer_stripped) + if not isinstance(parsed, ParseFailure): + return parsed + # The raw text is a legitimate final answer; the miss is only + # noteworthy because the answer *looked* like JSON. + logger.warning( + "Outputter answer looked like JSON but could not be parsed" + f" ({parsed.error}); returning it as plain text." + ) return raw_answer diff --git a/artemis/agents/planner/planner.json b/artemis/agents/planner/planner.json index 632727a..bbabdb2 100644 --- a/artemis/agents/planner/planner.json +++ b/artemis/agents/planner/planner.json @@ -1,12 +1,14 @@ { "blocks": { "role": "## Role\nYou are a Senior Test Engineer leading a team of mobile execution agents. Your objective is to deeply analyze and understand user goals, anticipate execution pitfalls, and formulate robust, observable, and feasibility-bounded milestone blueprints.", - "thinking_process": "## Systematic Thinking Process & Operating Principles\nBefore calling any tools or formulating your final milestones, you MUST systematically analyze the task following these 5 sequential thinking steps in your reasoning:\n\n### 1. Deep Intent Deconstruction & Task Composition\n**Core Principle**: Natural language requests describe the *substantive intent and desired real-world outcome*, not the *rigid system keywords or database filters* of mobile apps. Never mechanically map user phrasing into strict literal searches or surface filters.\n\n**Thinking Checklist (Ask yourself)**:\n- *Task Composition*: What phases make up this task? Is it a sequence of one-off linear steps (e.g., open app, configure settings), an iterative loop over multiple items (e.g., inspecting a list of search results), or a periodic polling loop (e.g., refreshing to monitor new messages, waiting for battery level)?\n- *Substantive Intent & Semantic Expansion (Connecting User Words to App Reality)*: What information or outcome is the user truly looking for, and how might that actually appear inside the target app? For example, details for an event taking place in November might have been sent months earlier (in August or September), and the email subject might use terms like \"Summit\", \"Invitation\", or \"Confirmation\" rather than the literal words \"November conference\". Proactively brainstorm the synonyms, related terms, project codes, and indirect records the app might use.\n\n**Abstract Behavioral Contrast**:\n- ❌ *Anti-Pattern (Mechanical Overfitting)*: Treating descriptive modifiers as strict literal filters -> Pre-judging items solely by surface list titles -> Prematurely missing valid candidates.\n- ✅ *Pro-Pattern (Intent Expansion & Deep Inspection)*: Decoupling core intent from surface wording -> Considering all plausible forms and timeframes -> Opening items to verify their real contents.\n\n### 2. Visual Grounding & Starting Context\n**Core Principle**: Every plan must be rooted strictly in the current physical reality of the device.\n\n**Thinking Checklist (Ask yourself)**:\n- *Current State Analysis*: What active app, screen layout, and interactive elements are visible in the provided screenshot to serve as the immediate starting point?\n\n### 3. Tiered Strategy, Funnel Approach & Feasibility Safeguards\n**Core Principle**: Formulate an efficient hierarchy of execution strategies, use a broad-to-narrow funnel for finding information, and enforce clear boundaries on loops and exploration.\n\n**Thinking Checklist (Ask yourself)**:\n- *Tiered Paths*: What is the fastest, highest-signal optimal path (e.g., targeted in-app search, direct shortcuts)? If that fails or yields no results, what is the pragmatic fallback path (e.g., browsing categories, chronological scanning)?\n- *The Funnel Approach (Broad Discovery -> Deep Verification)*:\n - **Top of Funnel (Broad Gathering)**: First collect all potentially relevant candidates (via multi-keyword search, category browsing, or scanning). Do not filter out candidates prematurely based only on incomplete snippet text in list views.\n - **Bottom of Funnel (Deep Inspection)**:\n - Systematically open each candidate to thoroughly inspect its details and confirm whether it meets the criteria.\n- *Feasibility Safeguards & Loop Boundaries*:\n - *For Item Iteration & Exploration*: Set explicit upper bounds (e.g., \"cap inspection at top 15 candidates\", \"scan recent 3 months\") to prevent endless scrolling.\n - *For Periodic Monitoring / Polling*: Specify the check action (e.g., pull-to-refresh, reading system telemetry via ADB), the wait interval between checks (e.g., check every 5 minutes to conserve resources), and the exit criteria (e.g., stop when condition is met, or continue until manually stopped).\n\n### 4. Risk Pre-mortem & Defensive Design\n**Core Principle**: Proactively assume that execution will encounter obstacles, and build self-healing and verification checkpoints directly into the milestones.\n\n**Thinking Checklist (Ask yourself)**:\n- *Anticipate Roadblocks*: What common mobile friction points could occur (e.g., login/auth walls, permission dialogs, promotional popups/ads, search ambiguities, empty states, network latency)?\n- *Embedded Defense*: What observable criteria can be embedded into the milestone to help the executor immediately verify success or detect derailment without mistaking normal wait intervals for system stalls?\n\n### 5. Milestone Granularity, Outcome-Orientation & Dynamic Live-Iteration Tracking\n**Core Principle**: Maintain a strict two-tier hierarchy: top-level milestones define strategic macro-phases, while indented sub-tasks encapsulate every individual iteration instance within a loop.\n\n**Thinking Checklist (Ask yourself)**:\n- *Outcome-Oriented Granularity (2-5 Strategic Phases)*: Top-level milestones describe *what macro result to achieve* (e.g., \"Search for candidate emails and extract confirmed itinerary\" or \"[Loop] Periodically monitor for incoming notifications...\") rather than micromanaging atomic UI actions.\n- *Phase vs. Instance Hierarchy Principle (Loop Subsumption Rule)*:\n - **Macro-Phases at Top Level**: Top-level milestones (`- [ ]`) represent distinct lifecycle phases (e.g., Setup/Prerequisites -> Iterative/Polling Loop -> Final Consolidation).\n - **All Instances Belong Inside Loops**: When a milestone represents an iterative, recurring, or monitoring process (tagged `[Loop]`), **EVERY individual iteration round, polling check, candidate inspection, or processing cycle (starting from the initial baseline/candidate #1 to subsequent #N) MUST be an indented sub-task (` - [ ]`) strictly nested underneath that `[Loop]` milestone**.\n - **Anti-Pattern (Flattened Iterations)**: Never extract the initial check/baseline as an independent top-level milestone outside the loop, and never place individual check cycles as parallel top-level milestones. The top-level `[Loop]` encapsulates the entire recurring lifecycle.\n- *Live Iteration Sub-Tasks & Anti-Loop Mechanism*: Concrete sub-tasks provide an explicit cognitive anchor for downstream execution agents. By initializing the plan with a concrete first-instance sub-task (e.g., ` - [ ] Inspect Candidate #1: ...` or ` - [ ] Polling Check #1 (Baseline & Initial Sync): ...`), you inspire downstream agents to dynamically maintain a clear ledger—logging each processed item/round and checking it off.\n - *For Item Iteration / Conditional Loops*: The dynamic checklist counts toward the explicit boundary (e.g., Candidates #1 to #N until all required info is found or max items reached).\n - *For Continuous Monitor-Trigger Tasks*: The top-level milestone declares the overall continuous monitoring goal and exit boundary (`until manually stopped`), while indented dynamic sub-tasks track each discrete polling cycle and event-triggered response at a reasonable inspection interval (e.g., check every 5 minutes).\n- *Scratchpad Designation & Answer Card Blueprint*: If the task involves collecting information, monitoring events, tracking iterations, or generating summaries, designate a clear, descriptive note name (e.g., `conference_summary`, `monitoring_log`, `candidates_list`) directly in the milestone text as the explicit destination for the executor. Proactively design a structured schema/template for these notes so that downstream execution agents have an unambiguous scaffold to record and append data.", + "thinking_process": "## Systematic Thinking Process & Operating Principles\nBefore calling any tools or formulating your final milestones, you MUST systematically analyze the task following these 5 sequential thinking steps in your reasoning:\n\n### 1. Deep Intent Deconstruction & Task Composition\n**Core Principle**: Natural language requests describe the *substantive intent and desired real-world outcome*, not the *rigid system keywords or database filters* of mobile apps. Never mechanically map user phrasing into strict literal searches or surface filters.\n\n**Thinking Checklist (Ask yourself)**:\n- *Task Composition*: What phases make up this task? Is it a sequence of one-off linear steps (e.g., open app, configure settings), an iterative loop over multiple items (e.g., inspecting a list of search results), or a periodic polling loop (e.g., refreshing to monitor new messages, waiting for battery level)?\n- *Substantive Intent & Semantic Expansion (Connecting User Words to App Reality)*: What information or outcome is the user truly looking for, and how might that actually appear inside the target app? For example, details for an event taking place in November might have been sent months earlier (in August or September), and the email subject might use terms like \"Summit\", \"Invitation\", or \"Confirmation\" rather than the literal words \"November conference\". Proactively brainstorm the synonyms, related terms, project codes, and indirect records the app might use.\n\n**Abstract Behavioral Contrast**:\n- ❌ *Anti-Pattern (Mechanical Overfitting)*: Treating descriptive modifiers as strict literal filters -> Pre-judging items solely by surface list titles -> Prematurely missing valid candidates.\n- ✅ *Pro-Pattern (Intent Expansion & Deep Inspection)*: Decoupling core intent from surface wording -> Considering all plausible forms and timeframes -> Opening items to verify their real contents.\n\n### 2. Visual Grounding & Starting Context\n**Core Principle**: Every plan must be rooted strictly in the current physical reality of the device.\n\n**Thinking Checklist (Ask yourself)**:\n- *Current State Analysis*: What active app, screen layout, and interactive elements are visible in the provided screenshot to serve as the immediate starting point?\n\n### 3. Tiered Strategy, Funnel Approach & Feasibility Safeguards\n**Core Principle**: Formulate an efficient hierarchy of execution strategies, use a broad-to-narrow funnel for finding information, and enforce clear boundaries on loops and exploration.\n\n**Thinking Checklist (Ask yourself)**:\n- *Tiered Paths*: What is the fastest, highest-signal optimal path (e.g., targeted in-app search, direct shortcuts)? If that fails or yields no results, what is the pragmatic fallback path (e.g., browsing categories, chronological scanning)?\n- *The Funnel Approach (Broad Discovery -> Deep Verification)*:\n - **Top of Funnel (Broad Gathering)**: First collect all potentially relevant candidates (via multi-keyword search, category browsing, or scanning). Do not filter out candidates prematurely based only on incomplete snippet text in list views.\n - **Bottom of Funnel (Deep Inspection)**:\n - Systematically open each candidate to thoroughly inspect its details and confirm whether it meets the criteria.\n- *Feasibility Safeguards & Loop Boundaries*:\n - *For Item Iteration & Exploration*: Set explicit upper bounds (e.g., \"cap inspection at top 15 candidates\", \"scan recent 3 months\") to prevent endless scrolling.\n - *For Periodic Monitoring / Polling*: Specify the check action (e.g., pull-to-refresh, reading system telemetry via ADB), the wait interval between checks (e.g., check every 5 minutes to conserve resources), and the exit criteria (e.g., stop when condition is met, or continue until manually stopped — tag such unbounded milestones `[Loop:continuous]`).\n\n### 4. Risk Pre-mortem & Defensive Design\n**Core Principle**: Proactively assume that execution will encounter obstacles, and build self-healing and verification checkpoints directly into the milestones.\n\n**Thinking Checklist (Ask yourself)**:\n- *Anticipate Roadblocks*: What common mobile friction points could occur (e.g., login/auth walls, permission dialogs, promotional popups/ads, search ambiguities, empty states, network latency)?\n- *Embedded Defense*: What observable criteria can be embedded into the milestone to help the executor immediately verify success or detect derailment without mistaking normal wait intervals for system stalls?\n\n### 5. Milestone Granularity, Outcome-Orientation & Dynamic Live-Iteration Tracking\n**Core Principle**: Maintain a strict two-tier hierarchy: top-level milestones define strategic macro-phases, while indented sub-tasks encapsulate every individual iteration instance within a loop.\n\n**Thinking Checklist (Ask yourself)**:\n- *Outcome-Oriented Granularity (2-5 Strategic Phases)*: Top-level milestones describe *what macro result to achieve* (e.g., \"Search for candidate emails and extract confirmed itinerary\" or \"[Loop] Periodically monitor for incoming notifications...\") rather than micromanaging atomic UI actions.\n- *Phase vs. Instance Hierarchy Principle (Loop Subsumption Rule)*:\n - **Macro-Phases at Top Level**: Top-level milestones (`- [ ]`) represent distinct lifecycle phases (e.g., Setup/Prerequisites -> Iterative/Polling Loop -> Final Consolidation).\n - **All Instances Belong Inside Loops**: When a milestone represents an iterative, recurring, or monitoring process (tagged `[Loop]`), **EVERY individual iteration round, polling check, candidate inspection, or processing cycle (starting from the initial baseline/candidate #1 to subsequent #N) MUST be an indented sub-task (` - [ ]`) strictly nested underneath that `[Loop]` milestone**.\n - **Anti-Pattern (Flattened Iterations)**: Never extract the initial check/baseline as an independent top-level milestone outside the loop, and never place individual check cycles as parallel top-level milestones. The top-level `[Loop]` encapsulates the entire recurring lifecycle.\n- *Live Iteration Sub-Tasks & Anti-Loop Mechanism*: Concrete sub-tasks provide an explicit cognitive anchor for downstream execution agents. By initializing the plan with a concrete first-instance sub-task (e.g., ` - [ ] Inspect Candidate #1: ...` or ` - [ ] Polling Check #1 (Baseline & Initial Sync): ...`), you inspire downstream agents to dynamically maintain a clear ledger—logging each processed item/round and checking it off.\n - *For Item Iteration / Conditional Loops*: The dynamic checklist counts toward the explicit boundary (e.g., Candidates #1 to #N until all required info is found or max items reached).\n - *For Continuous Monitor-Trigger Tasks*: The top-level milestone is tagged `[Loop:continuous]` and declares the overall continuous monitoring goal, while indented dynamic sub-tasks track each discrete polling cycle and event-triggered response at a reasonable inspection interval (e.g., check every 5 minutes).\n- *Scratchpad Designation & Answer Card Blueprint*: If the task involves collecting information, monitoring events, tracking iterations, or generating summaries, designate a clear, descriptive note name (e.g., `conference_summary`, `monitoring_log`, `candidates_list`) directly in the milestone text as the explicit destination for the executor. Proactively design a structured schema/template for these notes so that downstream execution agents have an unambiguous scaffold to record and append data.", "capabilities_operator": "## Execution Capabilities\n\nThe execution agent primarily interacts with the mobile device and analyzes screen recordings and logs. It can perform the following types of actions:\n- **UI Interaction**: Tapping, swiping, and entering text on the screen.\n- **App Management**: Launching and stopping specific applications.\n- **Information Retrieval & Analysis**: Reading on-screen text and UI hierarchy, searching the web for external information, querying system logs, and analyzing screen recordings.\n- **Terminal & Background Operations**: Executing arbitrary ADB shell commands on the device for exploration. It is capable of running long-running ADB commands asynchronously in the background as tasks.\n\n### Capability Boundaries & Planning Rules\n- **Cross-App Operations**: When a task requires interacting with multiple applications, split the operations into distinct milestones for each app. Avoid mixing actions for different apps in a single milestone.", - "output_protocol_initial": "## Task Plan & Scratchpad Protocol\n\n1. **Articulate Thinking**: Begin by reasoning through the 5-step **Systematic Thinking Process** to establish your strategy, funnel approach, bounds, and pre-mortem checks.\n2. **Designate Notes in Milestones & Pre-generate Answer Cards**: If the task involves collecting findings, logging periodic monitoring events, recording candidates, or analytical summaries, designate clear, descriptive note names directly in the milestone descriptions (e.g., `candidates_list`, `monitoring_log`, `schedule_summary`). You can and should invoke the `save_note` tool to pre-generate these structured template files (answer card / scratchpad skeletons with Markdown headers, sections, or tables) before submitting `task_plan`. This provides downstream executors with a clear, ready-to-fill schema.\n3. **Submit Plan**: Submit your initial plan by invoking the `save_note` tool with `key='task_plan'`. Avoid writing the plan only in your text response, as it will only be recognized by the system if submitted via the tool.\n\n### Plan Formatting Rules\n- Format the plan strictly as a Markdown task list starting with `- [ ]` for uncompleted items.\n- For linear macro-milestones: `- [ ] `\n- For repetitive, iterative, or periodic polling milestones, tag the item with `[Loop]` and declare its Exit Boundary and Interval (e.g., `- [ ] [Loop] (Exit: ; Interval: )`).\n- **Strict Two-Tier Hierarchy & Loop Subtask Contract**:\n 1. **Subtask Nesting Rule**: ALL individual iteration instances, polling checks, candidate inspections, or baseline setups MUST be formatted as 2-space indented sub-tasks (` - [ ]`) directly nested under their parent `[Loop]` milestone.\n 2. **Mandatory First-Instance Initialization**: Every `[Loop]` milestone MUST be initialized with its first concrete child sub-task (e.g., ` - [ ] Inspect Candidate #1: ...` or ` - [ ] Polling Check #1 (Baseline & Initial Sync): ...`).\n 3. **No Flattened Cycles**: NEVER elevate individual checks, recurring polling cycles, or candidate inspections to top-level milestones outside the `[Loop]`. Top-level items are reserved exclusively for strategic macro-phases.\n- Ensure the content of the `task_plan` note contains only the task list, excluding introductions, explanations, conversational text, or headers.", + "output_protocol_initial": "## Task Plan & Scratchpad Protocol\n\n1. **Articulate Thinking**: Begin by reasoning through the 5-step **Systematic Thinking Process** to establish your strategy, funnel approach, bounds, and pre-mortem checks.\n2. **Designate Notes in Milestones & Pre-generate Answer Cards**: If the task involves collecting findings, logging periodic monitoring events, recording candidates, or analytical summaries, designate clear, descriptive note names directly in the milestone descriptions (e.g., `candidates_list`, `monitoring_log`, `schedule_summary`). You can and should invoke the `save_note` tool to pre-generate these structured template files (answer card / scratchpad skeletons with Markdown headers, sections, or tables) before submitting `task_plan`. This provides downstream executors with a clear, ready-to-fill schema.\n3. **Submit Plan**: Submit your initial plan by invoking the `save_note` tool with `key='task_plan'`. Avoid writing the plan only in your text response, as it will only be recognized by the system if submitted via the tool.\n\n### Plan Formatting Rules\n- Format the plan strictly as a Markdown task list starting with `- [ ]` for uncompleted items.\n{{ plan_grammar }}\n- For linear macro-milestones: `- [ ] `\n- For bounded repetitive, iterative, or periodic polling milestones, tag the item with `[Loop]` and declare its Exit Boundary and Interval (e.g., `- [ ] [Loop] (Exit: ; Interval: )`).\n- For unbounded continuous monitoring that only the user can stop, tag the item with `[Loop:continuous]` instead (e.g., `- [ ] [Loop:continuous] (Interval: )`); never declare a self-serve exit condition for it.\n- **Strict Two-Tier Hierarchy & Loop Subtask Contract**:\n 1. **Subtask Nesting Rule**: ALL individual iteration instances, polling checks, candidate inspections, or baseline setups MUST be formatted as 2-space indented sub-tasks (` - [ ]`) directly nested under their parent `[Loop]` milestone.\n 2. **Mandatory First-Instance Initialization**: Every `[Loop]` milestone MUST be initialized with its first concrete child sub-task (e.g., ` - [ ] Inspect Candidate #1: ...` or ` - [ ] Polling Check #1 (Baseline & Initial Sync): ...`).\n 3. **No Flattened Cycles**: NEVER elevate individual checks, recurring polling cycles, or candidate inspections to top-level milestones outside the `[Loop]`. Top-level items are reserved exclusively for strategic macro-phases.\n- Ensure the content of the `task_plan` note contains only the task list, excluding introductions, explanations, conversational text, or headers.", "examples_initial": "## Master Example: Demonstrating the Complete Cognitive Paradigm\n\n**Goal**: \"Find and summarize all schedule arrangements and venue details for the upcoming November tech conference from my emails.\"\n\n### Naive / Incorrect Approaches:\n1. *Flawed Semantics & Too Granular*:\n- [ ] Tap search icon in Gmail\n- [ ] Type \"November\"\n- [ ] Open first email and tap back\n*(Fails: Micromanages atomic UI taps; assumes emails for November arrangements must have \"November\" in the title or be received in November)*\n2. *Vague & Unbounded Trap*:\n- [ ] Open email and check all emails for conference details\n*(Fails: Unbounded scan trap; causes infinite scrolling and step budget exhaustion on mobile)*\n\n---\n\n### Correct Approach (Full Cognitive Thinking & Robust Plan):\n\n#### Step-by-Step Thinking:\n1. **Intent & Semantic Expansion**:\n - *Core Objective*: Extract confirmed schedule dates, sessions, and venue info for the November tech conference.\n - *Semantic Insight*: Conference communications are often sent months in advance (e.g., ticket confirmations or registration emails in August/September). The subject line may say \"Global Developer Summit\", \"Tech Expo Agenda\", or \"Registration Confirmed\" rather than literally \"November conference\".\n - *Task Composition*: Linear setup (open email app and search) -> Iterative candidate inspection loop (examine each promising email) -> Final consolidation.\n2. **Visual Grounding**:\n - The device screenshot shows the Home screen with the Gmail app icon visible. The plan must start by opening the email client.\n3. **Tiered Strategy, Funnel Approach & Safeguards**:\n - *Tier 1 (Optimal)*: Use in-app search with rich semantic keywords (\"conference\", \"summit\", \"schedule\", \"agenda\", \"venue\", \"November\").\n - *Tier 2 (Fallback)*: If search is unavailable or yields zero results, fallback to chronological scanning of recent 3 months.\n - *Funnel (Broad Gathering -> Deep Inspection)*: Gather candidate emails from search results; do not discard candidates based solely on list previews. Open each candidate thread to inspect full body details.\n - *Feasibility Boundary*: Cap the candidate email inspection pool at top 15 emails to prevent infinite scrolling.\n4. **Risk Pre-mortem**:\n - *Risks*: Promotional marketing spam; conflicting drafts with preliminary dates; cancelled sessions.\n - *Defensive Check*: Instruct downstream executor to filter out promotional newsletters, verify session timestamps, and confirm final venue address.\n5. **Milestone Granularity & Scratchpad Blueprint**:\n - Focus milestones on business outcomes (search, inspect candidates, consolidate), avoiding micromanaged click-by-click steps.\n - Use indented sub-tasks to guide the loop of inspecting each candidate email, initializing with Item #1 and adding subsequent items as they are identified. This prevents the agent from losing state or re-inspecting the same items.\n - Designate note `november_conference_summary` as the destination for recording findings and pre-generate the structured template note.\n\n#### Tool Invocation:\n1. Pre-generate designated answer card note template:\n`save_note(key='november_conference_summary', content='# November Tech Conference Summary\\n\\n## Key Details\\n- Conference Name: \\n- Dates: \\n- Venue / Address: \\n\\n## Daily Schedule & Sessions\\n(To be populated during email inspection)\\n')`\n\n2. Submit milestone plan:\n`save_note(key='task_plan', content='- [ ] Open email application and search for candidate emails regarding the November tech conference (Search targeted keywords like \"conference\", \"summit\", \"agenda\", \"venue\"; if search is unavailable, fallback to scanning recent 3 months, capped at top 15 emails)\\n- [ ] [Loop] Inspect candidate emails and record verified schedule sessions and venue details into note `november_conference_summary` (Exit: All key conference details identified OR candidate pool exhausted / max 15 inspected)\\n - [ ] Inspect Candidate #1: Open candidate, filter out non-relevant content, check body for scheduling/venue, and log findings if present\\n- [ ] Review and consolidate all extracted information in `november_conference_summary` into a clear, finalized conference itinerary')`\n\n#### Content of `task_plan` Note:\n- [ ] Open email application and search for candidate emails regarding the November tech conference (Search targeted keywords like \"conference\", \"summit\", \"agenda\", \"venue\"; if search is unavailable, fallback to scanning recent 3 months, capped at top 15 emails)\n- [ ] [Loop] Inspect candidate emails and record verified schedule sessions and venue details into note `november_conference_summary` (Exit: All key conference details identified OR candidate pool exhausted / max 15 inspected)\n - [ ] Inspect Candidate #1: Open candidate, filter out non-relevant content, check body for scheduling/venue, and log findings if present\n- [ ] Review and consolidate all extracted information in `november_conference_summary` into a clear, finalized conference itinerary", + "check_generation": "## Declaring Verification Standards (check lines)\nYou may attach check lines to the plan; an independent Checker judges them asynchronously.\n\n- **User-declared expectations become `- assert:` lines.** Whenever the user's goal states an explicit expectation (\"should show X\", \"must not crash\", \"the toast Y should appear\"), declare it as an assert line preserving the user's wording verbatim. Expectations about the whole run or the final state use the `@end` suffix (task-level lines at the end of the plan).\n- **`- verify:` lines only for milestones that genuinely need acceptance checks** — completion points of irreversible operations, or milestones that directly contribute to the final goal. Do NOT attach a verify line to every subgoal: restraint is an explicit requirement (checking cost scales with the number of check lines).\n- **Do not fabricate assertions** for subgoals that have nothing verifiable. Rewrite transient expectations as persistently probeable state wherever possible (e.g., \"alarm exists in the alarm list\" instead of \"a toast flashed\") — evidence for transient moments only exists if the execution history happened to capture it.\n- **Ordering requirements are post-hoc.** If the user demands \"A must be confirmed before B runs\", still generate the assertion preserving their semantics, but annotate in the plan that ordering is verified retroactively from history — the system performs post-hoc audits and CANNOT block B from executing. Never silently promise enforcement that does not exist.", + "check_audit": "- **Check Standard Protection**: Check lines (`- verify:` / `- assert:`) declared in the plan are the task's verification standards. REJECT any proposed plan that deletes, weakens, or rewords existing check lines — softening a declared standard is never an acceptable adaptation. Adding new check lines is allowed.", "role_validator": "## Role\nYou are the Task Plan Validator. The Operator has attempted to modify the top-level task plan. Your job is to ensure the proposed new plan still aligns with the user's original intent and is viable.", - "validator_guidelines": "## Validation Principles\n- **Goal Alignment**: Ensure the new plan ultimately accomplishes the `Initial Goal` requested by the user. If the Operator has abandoned the core objective or fundamentally altered it, you must reject the plan.\n- **Logical Coherence**: Verify that the proposed milestones form a logical, achievable sequence to reach the goal. Support valid ongoing iteration/polling subgoals and trigger branches without misconstruing periodic checks as infinite loops. Reject the plan if it introduces contradictory steps, unhandled deadlocks, or obvious logical fallacies.\n- **Loop & Monitoring Protection**: If the existing plan contains an active `[Loop]` milestone (especially continuous polling / monitoring with exit condition `until manually stopped`), you MUST REJECT any proposed plan that deletes the `[Loop]` milestone, removes the loop tag, or marks a continuous monitoring milestone as completed (`[x]`) without explicit external user instruction.\n- **Adaptive Acceptance**: The Operator operates in a dynamic environment and may encounter unexpected UI states or dynamic event triggers. You should approve alternative strategies, extra intermediate steps, loop status tracking, or necessary workarounds as long as they serve the primary goal effectively.\n- **Constructive Feedback**: If you must reject the plan, provide clear, objective, and actionable feedback explaining the logical flaw or goal deviation, and suggest how the Operator should correct the plan." + "validator_guidelines": "## Validation Principles\n- **Goal Alignment**: Ensure the new plan ultimately accomplishes the `Initial Goal` requested by the user. If the Operator has abandoned the core objective or fundamentally altered it, you must reject the plan.\n- **Logical Coherence**: Verify that the proposed milestones form a logical, achievable sequence to reach the goal. Support valid ongoing iteration/polling subgoals and trigger branches without misconstruing periodic checks as infinite loops. Reject the plan if it introduces contradictory steps, unhandled deadlocks, or obvious logical fallacies.\n{{ plan_grammar }}\n- **Loop & Monitoring Protection**: If the existing plan contains an active `[Loop]` or `[Loop:continuous]` milestone, you MUST REJECT any proposed plan that deletes the milestone or removes its loop tag, as well as any plan that marks a `[Loop:continuous]` continuous monitoring milestone as completed (`[x]`) without an explicit external user stop signal.\n- **Adaptive Acceptance**: The Operator operates in a dynamic environment and may encounter unexpected UI states or dynamic event triggers. You should approve alternative strategies, extra intermediate steps, loop status tracking, or necessary workarounds as long as they serve the primary goal effectively.\n- **Constructive Feedback**: If you must reject the plan, provide clear, objective, and actionable feedback explaining the logical flaw or goal deviation, and suggest how the Operator should correct the plan." }, "modes": { "initial_plan": { diff --git a/artemis/agents/planner/planner.py b/artemis/agents/planner/planner.py index d813c3b..984d6e1 100644 --- a/artemis/agents/planner/planner.py +++ b/artemis/agents/planner/planner.py @@ -25,7 +25,9 @@ from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace from artemis.graph.state import State +from artemis.graph.visibility import strict_state from artemis.services.llm import ( + acomplete, get_llm, invoke_llm_with_timeout_message, with_fallback, @@ -44,11 +46,30 @@ from artemis.tools.tool_wrapper import ( from artemis.utils.decorators import wrap_with_callbacks from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path +from artemis.utils.plan_grammar import render_plan_grammar_spec from artemis.utils.task_tree import build_plan_and_history logger = get_logger(__name__) +def _checks_enabled(ctx: ArtemisContext) -> bool: + """Config-driven prompt assembly gate: with both check gates disabled, no + check-related instruction reaches any planner prompt.""" + setup = getattr(ctx, "execution_setup", None) + return bool(setup and getattr(setup, "checks_enabled", False)) + + +def build_planner_system_blocks(prompts_data: dict, mode: str, include_checks: bool) -> list[str]: + """Assembly point for the planner prompt's component list (§1.3): the + check-related block mounts only while the checking feature is active.""" + blocks = list(prompts_data["modes"][mode]["system"]) + if include_checks: + extra = "check_generation" if mode == "initial_plan" else "check_audit" + if extra in prompts_data.get("blocks", {}): + blocks.append(extra) + return blocks + + class _CyFunctionDetectorMeta(type): def __instancecheck__(self, instance): name = type(instance).__name__ @@ -97,10 +118,13 @@ async def run_async_planner_validation( with open(prompts_path, encoding="utf-8") as f: prompts_data = json.load(f) + include_checks = _checks_enabled(ctx) mode_config = prompts_data["modes"]["validator"] - system_blocks = mode_config["system"] + system_blocks = build_planner_system_blocks(prompts_data, "validator", include_checks) system_content = "\n\n".join(prompts_data["blocks"][block] for block in system_blocks) - system_message = Template(system_content).render() + system_message = Template(system_content).render( + plan_grammar=render_plan_grammar_spec(include_checks) + ) history_str = "No history available." if ctx.data_engine: @@ -163,6 +187,7 @@ class PlannerNode: ) @trace(type="agent", name="planner") async def __call__(self, state: State): + state = strict_state(state, "planner") prompts_path = Path(__file__).parent / "planner.json" with open(prompts_path, encoding="utf-8") as f: prompts_data = json.load(f) @@ -170,10 +195,16 @@ class PlannerNode: mode = "initial_plan" mode_config = prompts_data["modes"][mode] - system_blocks = mode_config["system"] + include_checks = _checks_enabled(self.ctx) + # Assembly point (not in-template conditionals): check-generation + # instructions only mount while the checking feature is active, so a + # fully-disabled configuration yields a byte-identical planner prompt. + system_blocks = build_planner_system_blocks(prompts_data, mode, include_checks) system_content = "\n\n".join(prompts_data["blocks"][block] for block in system_blocks) - system_message = Template(system_content).render() + system_message = Template(system_content).render( + plan_grammar=render_plan_grammar_spec(include_checks) + ) # Get screenshot screenshot_b64 = None @@ -287,22 +318,14 @@ class PlannerNode: for validation_attempt in range(max_validation_attempts): tool_called = False - async def run_stream(llm_model): - full_response = None - trace_id = CURRENT_TRACE_ID.get() - async for chunk in llm_model.astream(current_messages): - if full_response is None: - full_response = chunk - else: - full_response += chunk - - pass - return full_response - for _ in range(max_iterations): response = await with_fallback( - main_call=lambda: invoke_llm_with_timeout_message(run_stream(llm)), - fallback_call=lambda: invoke_llm_with_timeout_message(run_stream(llm_fallback)), + main_call=lambda: invoke_llm_with_timeout_message( + acomplete(llm, current_messages) + ), + fallback_call=lambda: invoke_llm_with_timeout_message( + acomplete(llm_fallback, current_messages) + ), ) if response is None or not response.tool_calls: @@ -415,8 +438,4 @@ class PlannerNode: f"Failed to generate a valid plan after {max_validation_attempts} attempts." ) - return await state.asanitize_update( - ctx=self.ctx, - update={}, - agent="planner", - ) + return {} diff --git a/artemis/agents/summarizer/summarizer.py b/artemis/agents/summarizer/summarizer.py index 42455ba..5b0c313 100644 --- a/artemis/agents/summarizer/summarizer.py +++ b/artemis/agents/summarizer/summarizer.py @@ -25,7 +25,8 @@ from langchain_core.messages import ( from artemis.context import ArtemisContext from artemis.data_engine.trace import CURRENT_TRACE_ID, trace from artemis.graph.state import State -from artemis.services.llm import get_llm +from artemis.graph.visibility import strict_state +from artemis.services.llm import acomplete, get_llm from artemis.utils.coordinates import normalize_any_structure from artemis.utils.logger import get_logger from artemis.utils.task_tree import ( @@ -42,6 +43,7 @@ class SummarizerNode: @trace(type="agent", name="summarizer") async def __call__(self, state: State): + state = strict_state(state, "summarizer") # 1. Trigger background task for summary and data engine update if self.ctx.data_engine and state.current_step_id: step_id = UUID(state.current_step_id) @@ -179,26 +181,16 @@ class SummarizerNode: HumanMessage(content="\n\n".join(content)), ] + aggregated_chunk = await acomplete(llm, messages) + generated_text = "" - aggregated_chunk = None - async for chunk in llm.astream(messages): - if aggregated_chunk is None: - aggregated_chunk = chunk - else: - aggregated_chunk += chunk - - if chunk.content: - text_to_stream = "" - if isinstance(chunk.content, str): - text_to_stream = chunk.content - elif isinstance(chunk.content, list): - for item in chunk.content: - if isinstance(item, dict) and item.get("type") == "text": - text_to_stream += item.get("text", "") - - if text_to_stream: - generated_text += text_to_stream - pass + content = getattr(aggregated_chunk, "content", "") if aggregated_chunk else "" + if isinstance(content, str): + generated_text = content + elif isinstance(content, list): + for item in content: + if isinstance(item, dict) and item.get("type") == "text": + generated_text += item.get("text", "") summary = generated_text.strip() logger.info(f"Summary generated: {summary}") diff --git a/artemis/agents/validator/tool_declarations.py b/artemis/agents/validator/tool_declarations.py index 3375077..4fca084 100644 --- a/artemis/agents/validator/tool_declarations.py +++ b/artemis/agents/validator/tool_declarations.py @@ -30,6 +30,7 @@ from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController from artemis.core.tool_declaration import ToolDeclaration from artemis.data_engine.trace import trace +from artemis.mcp.action_specs import tool_declaration from artemis.mcp.action_types import ActionCode from artemis.mcp.actuators.adb import AdbActuator from artemis.mcp.observation import observe @@ -187,9 +188,7 @@ async def capture_screenshot_and_parse_ui( Thin adapter over :func:`artemis.mcp.observation.observe` that adds the LangGraph ``State`` write-back (indexed points/elements) the agents rely on. """ - obs, screenshot_bytes = await observe( - ctx, controller, settle_ms=0 if skip_settling else 400 - ) + obs, screenshot_bytes = await observe(ctx, controller, settle_ms=0 if skip_settling else 400) if not obs.ok: return None, None, None @@ -473,7 +472,7 @@ class MobileActionExecutor: direction=target, target=swipe_input.get("target"), indexed_elements=getattr(state, "indexed_elements", None) if state else None, - ui_hierarchy=getattr(state, "ui_tree", None) if state else None, + ui_hierarchy=getattr(state, "latest_ui_hierarchy", None) if state else None, width=width, height=height, duration=final_duration, @@ -753,189 +752,19 @@ class MobileActionExecutor: ) -CLICK_TOOL = ToolDeclaration( - name="click", - description=( - "[ACTION] Tap/click on the target coordinate on the screen. The screen" - " after click will be returned automatically. For buttons, checkboxes," - " tabs, icons, items in a list, tap ON the element. For text fields /" - " search bars / input boxes, tap INSIDE the input box to focus it." - ), - parameters={ - "type": "object", - "properties": { - "target": { - "type": "array", - "items": {"type": "integer"}, - "description": "Normalized coordinates [x, y] in 0-1000 scale.", - }, - "times": { - "type": "integer", - "description": "Number of taps to perform (default 1). Set to 2 for double click.", - }, - "delay_ms": { - "type": "integer", - "description": "Delay between taps in milliseconds (default 100).", - }, - }, - "required": ["target"], - }, -) +# Device-action declarations are generated from the canonical manifest +# (artemis/mcp/action_specs.py); the historical constant names remain the public API. +CLICK_TOOL = tool_declaration("click") -CLICK_SEQUENCE_TOOL = ToolDeclaration( - name="click_sequence", - description=( - "[ACTION] Executes a sequence of taps one by one in order on the" - " specified targets (e.g. [[500, 280], [885, 362]]). The screen will be returned ONLY after all clicks" - " in the sequence have completed." - ), - parameters={ - "type": "object", - "properties": { - "sequence": { - "type": "array", - "items": { - "type": "array", - "items": {"type": "integer"}, - "description": ( - "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280])," - " or a single integer element index." - ), - }, - "description": "List of targets to tap in sequence, e.g. [[500, 280], [885, 362]].", - }, - "delay_ms": { - "type": "integer", - "description": "Delay between consecutive taps in milliseconds (default 50ms).", - }, - }, - "required": ["sequence"], - }, -) +CLICK_SEQUENCE_TOOL = tool_declaration("click_sequence") -LONG_PRESS_TOOL = ToolDeclaration( - name="long_press", - description=( - "[ACTION] Long press on a target coordinate on the screen. The screen" - " after long pressing will be returned automatically." - ), - parameters={ - "type": "object", - "properties": { - "target": { - "type": "array", - "items": {"type": "integer"}, - "description": "Normalized coordinates [x, y] in 0-1000 scale.", - }, - "duration_ms": { - "type": "integer", - "description": "Duration of press in milliseconds (default 1000ms).", - }, - }, - "required": ["target"], - }, -) +LONG_PRESS_TOOL = tool_declaration("long_press") -INPUT_TEXT_TOOL = ToolDeclaration( - name="input_text", - description=( - "[ACTION] Type text into an input field on the screen. The screen after" - " typing will be returned automatically. Automatically taps inside the" - " input box at target [x, y] to focus before typing." - ), - parameters={ - "type": "object", - "properties": { - "text": { - "type": "string", - "description": "Text to type into the focused input field.", - }, - "target": { - "type": "array", - "items": {"type": "integer"}, - "description": "Coordinates [x, y] of the input box in 0-1000 scale.", - }, - "clear_exist": { - "type": "boolean", - "description": ( - "Whether to clear existing text in the input box before typing (default True)." - ), - }, - }, - "required": ["text", "target"], - }, -) +INPUT_TEXT_TOOL = tool_declaration("input_text") -SWIPE_TOOL = ToolDeclaration( - name="swipe", - description=( - "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen. The screen and UI hierarchy after swipe will be returned automatically.\n\n" - "• Directional Scrolling ('direction' or 'action'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n" - "• Precise Coordinate Gestures ('start', 'end' or coordinates list): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary." - ), - parameters={ - "type": "object", - "properties": { - "direction": { - "type": "string", - "enum": ["up", "down", "left", "right"], - "description": ( - "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below)," - " 'down' (drags top-to-bottom, scrolling up to reveal content above)," - " 'left' (drags right-to-left, scrolling right)," - " 'right' (drags left-to-right, scrolling left)." - ), - }, - "start": { - "type": "array", - "items": {"type": "integer"}, - "description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale.", - }, - "end": { - "type": "array", - "items": {"type": "integer"}, - "description": "End normalized coordinates [end_x, end_y] in 0-1000 scale.", - }, - "target": { - "description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.", - }, - "action": { - "description": ( - "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right')" - " OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." - ), - }, - "duration": { - "type": "integer", - "description": ( - "Optional swipe/drag duration in milliseconds (default 800). For drag-and-drop," - " list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars)," - " set duration >= 1000 (e.g. 1500). If omitted for directional swipe, duration is computed automatically." - ), - }, - }, - }, -) +SWIPE_TOOL = tool_declaration("swipe") -PRESS_KEY_TOOL = ToolDeclaration( - name="press_key", - description=( - "[ACTION] Press a physical or virtual system button. The screen after" - " pressing will be returned automatically." - ), - parameters={ - "type": "object", - "properties": { - "key": { - "type": "string", - "description": ( - "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH)." - ), - } - }, - "required": ["key"], - }, -) +PRESS_KEY_TOOL = tool_declaration("press_key") READ_NOTE_TOOL = ToolDeclaration( name="read_note", @@ -966,49 +795,9 @@ LIST_NOTES_TOOL = ToolDeclaration( parameters={"type": "object", "properties": {}}, ) -MANAGE_APP_TOOL = ToolDeclaration( - name="manage_app", - description=( - "[ACTION] Launch or force stop a specified application. The screen" - " after launch/stop will be returned automatically." - ), - parameters={ - "type": "object", - "properties": { - "action": { - "type": "string", - "description": "The action type ('launch' or 'stop').", - }, - "app_name": { - "type": "string", - "description": "Display name or package name of the application.", - }, - }, - "required": ["action", "app_name"], - }, -) +MANAGE_APP_TOOL = tool_declaration("manage_app") -WAIT_FOR_DELAY_TOOL = ToolDeclaration( - name="wait_for_delay", - description=( - "[ACTION] Pause execution and wait for a specified duration in milliseconds." - " The screen after pause will be returned automatically." - ), - parameters={ - "type": "object", - "properties": { - "time_in_ms": { - "type": "integer", - "description": ( - "The exact duration to pause in milliseconds (e.g., 2000 for 2s," - " 60000 for 1 min, 180000 for 3 mins, 300000 for 5 mins)." - " Convert any required waiting duration into milliseconds." - ), - } - }, - "required": ["time_in_ms"], - }, -) +WAIT_FOR_DELAY_TOOL = tool_declaration("wait_for_delay") REPORT_FAILURE_ANALYSIS_TOOL = ToolDeclaration( name="report_failure_analysis", diff --git a/artemis/agents/validator/validator.py b/artemis/agents/validator/validator.py index 09d1978..fb2de63 100644 --- a/artemis/agents/validator/validator.py +++ b/artemis/agents/validator/validator.py @@ -38,9 +38,10 @@ from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController from artemis.data_engine.trace import CURRENT_TRACE_ID, trace from artemis.graph.state import State +from artemis.graph.visibility import strict_state from artemis.mcp.action_names import to_canonical_call from artemis.mcp.action_session import ActionSession, get_action_session -from artemis.services.llm import get_llm +from artemis.services.llm import acomplete_structured, get_llm from artemis.utils import image_diff, visualization from artemis.utils.decorators import wrap_with_callbacks from artemis.utils.logger import get_logger @@ -154,11 +155,7 @@ class ValidatorNode: async def _execute_validation_loop(self, state: State): actions, error_msg = self._parse_decisions(state.structured_decisions) if error_msg: - return await state.asanitize_update( - ctx=self.ctx, - update={}, - agent="validator", - ) + return {} execution = [] failed_action = None @@ -176,11 +173,7 @@ class ValidatorNode: decision_screenshot_b64 = last_screenshot_b64 decision_screenshot_name = last_screenshot_name except Exception: - return await state.asanitize_update( - ctx=self.ctx, - update={}, - agent="validator", - ) + return {} step_id = None if state.current_step_id: @@ -355,7 +348,9 @@ class ValidatorNode: if post_screenshot_b64: decoded_bytes = base64.b64decode(post_screenshot_b64) last_screenshot_b64 = post_screenshot_b64 - logger.info(f"Successfully decoded post_screenshot_b64 ({len(decoded_bytes)} bytes)") + logger.info( + f"Successfully decoded post_screenshot_b64 ({len(decoded_bytes)} bytes)" + ) if self.ctx.data_engine: post_image_name = self.ctx.data_engine.get_or_create_image(decoded_bytes) last_screenshot_name = post_image_name @@ -466,11 +461,7 @@ class ValidatorNode: step_id, report, post_image_name=distinct_post_image_name ) - return await state.asanitize_update( - ctx=self.ctx, - update={"last_execution_result": report}, - agent="validator", - ) + return {"last_execution_result": report} @wrap_with_callbacks( before=lambda: logger.info("Starting Validator Agent..."), @@ -479,6 +470,7 @@ class ValidatorNode: ) @trace(type="agent", name="validator") async def __call__(self, state: State): + state = strict_state(state, "validator") step_id = None if state.current_step_id: @@ -1196,22 +1188,16 @@ class ValidatorNode: HumanMessage(content=user_content), ] - # D. Invoke Universal VLM - response = await llm.ainvoke(messages) - output = response.content if isinstance(response.content, str) else "" - if isinstance(response.content, list): - output = "".join( - b.get("text", "") - for b in response.content - if isinstance(b, dict) and "text" in b + # D. Invoke Universal VLM. Parsing (fences, repair) and one + # corrective re-ask on unparseable JSON are handled by the + # structured-output layer; a StructuredOutputError lands in + # this attempt loop's except like any other attempt failure. + res_json = await acomplete_structured(llm, messages) + if not isinstance(res_json, dict): + raise ValueError( + "Pixel validation response parsed to" + f" {type(res_json).__name__}, expected a JSON object." ) - - if output.startswith("```"): - code_match = re.search(r"```(?:json)?\s*([\s\S]*?)\s*```", output) - if code_match: - output = code_match.group(1).strip() - - res_json = json.loads(output) reasoning = res_json.get("reasoning", "") is_present = res_json.get("is_present", True) confidence = res_json.get("confidence", 1.0) diff --git a/artemis/agents/video_analyzer/reliability.py b/artemis/agents/video_analyzer/reliability.py index 47393c1..e02f72f 100644 --- a/artemis/agents/video_analyzer/reliability.py +++ b/artemis/agents/video_analyzer/reliability.py @@ -12,15 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Failure classification and circuit breaking for video model calls.""" +"""Failure classification and circuit breaking for video model calls. + +Thin media-aware extension of :mod:`artemis.llm.reliability`, which owns the +generic taxonomy and circuit breaker. This module adds the media-processing +category and the ``should_split`` recovery decision (bisecting long clips). +""" from __future__ import annotations from dataclasses import dataclass from enum import StrEnum -import threading -import time -from typing import Any + +from artemis.llm.reliability import ( + CircuitBreaker, + FailureCategory, + classify_failure, + extract_status_code, +) class VideoFailureCategory(StrEnum): @@ -43,115 +52,37 @@ class VideoFailure: should_fallback: bool +_MEDIA_MARKERS = ("ffmpeg", "codec", "corrupt", "invalid video", "invalid audio", "media") + + def classify_video_failure(error: BaseException) -> VideoFailure: """Convert provider/transport/media exceptions into recovery decisions.""" - - if isinstance(error, KeyboardInterrupt) or type(error).__name__ == "CancelledError": - return VideoFailure(VideoFailureCategory.CANCELLED, False, False, False) - - code = getattr(error, "code", None) or getattr(error, "status_code", None) - try: - code = int(code) if code is not None else None - except (TypeError, ValueError): - code = None - + generic = classify_failure(error) message = str(error).lower() - if code == 429 or any(marker in message for marker in ("rate limit", "quota", "resource_exhausted")): - return VideoFailure(VideoFailureCategory.RATE_LIMIT, True, False, True) - if code in {500, 502, 503, 504} or any( - marker in message - for marker in ("unavailable", "overloaded", "high demand", "service unavailable") - ): - return VideoFailure(VideoFailureCategory.PROVIDER_UNAVAILABLE, True, False, True) - if isinstance(error, TimeoutError) or "timed out" in message or "timeout" in message: - return VideoFailure(VideoFailureCategory.TIMEOUT, True, True, True) - if isinstance(error, (ConnectionError, OSError)) and not any( - marker in message for marker in ("ffmpeg", "video", "audio", "codec", "media") - ): - return VideoFailure(VideoFailureCategory.CONNECTION, True, False, True) - if code in {401, 403} or any( - marker in message for marker in ("unauthorized", "forbidden", "api key", "credential") - ): - return VideoFailure(VideoFailureCategory.AUTHENTICATION, False, False, True) - if any( - marker in message - for marker in ("ffmpeg", "codec", "corrupt", "invalid video", "invalid audio", "media") - ): + has_media_marker = any(marker in message for marker in _MEDIA_MARKERS) + + # Media markers take precedence over generic connection / bad-request / + # unknown classification (rate-limit, unavailable, timeout, and auth + # signals still win, matching the original ordering of checks). + if has_media_marker and generic.category in { + FailureCategory.CONNECTION, + FailureCategory.BAD_REQUEST, + FailureCategory.UNKNOWN, + }: return VideoFailure(VideoFailureCategory.MEDIA_PROCESSING, False, True, True) - if code in {400, 404, 413, 415, 422}: - return VideoFailure(VideoFailureCategory.BAD_REQUEST, False, code == 413, True) - return VideoFailure(VideoFailureCategory.UNKNOWN, True, True, True) + + category = VideoFailureCategory(generic.category.value) + should_split = category in {VideoFailureCategory.TIMEOUT, VideoFailureCategory.UNKNOWN} or ( + category is VideoFailureCategory.BAD_REQUEST and extract_status_code(error) == 413 + ) + # The video pipeline keeps its historical behavior of falling back to the + # universal analyzer even for bad requests (a different provider may accept + # the same media payload). + should_fallback = generic.should_fallback or category is VideoFailureCategory.BAD_REQUEST + if category is VideoFailureCategory.CANCELLED: + should_fallback = False + return VideoFailure(category, generic.retryable, should_split, should_fallback) -@dataclass -class _CircuitState: - consecutive_failures: int = 0 - open_until: float = 0.0 - half_open_in_flight: bool = False - - -class VideoCircuitBreaker: - """Small context-scoped circuit breaker keyed by provider/model identity.""" - - def __init__(self, threshold: int = 3, cooldown_seconds: float = 60.0) -> None: - self.threshold = max(1, int(threshold)) - self.cooldown_seconds = max(0.1, float(cooldown_seconds)) - self._states: dict[str, _CircuitState] = {} - self._lock = threading.RLock() - - def allow(self, key: str, *, now: float | None = None) -> bool: - current = time.time() if now is None else now - with self._lock: - state = self._states.get(key) - if state is None: - return True - if state.open_until: - if current < state.open_until or state.half_open_in_flight: - return False - # Half-open: permit exactly one trial. Other concurrent calls - # continue to bypass the primary until it succeeds or fails. - state.half_open_in_flight = True - state.consecutive_failures = max(0, self.threshold - 1) - return True - return not state.half_open_in_flight - - def record_success(self, key: str) -> None: - with self._lock: - self._states.pop(key, None) - - def record_failure( - self, - key: str, - failure: VideoFailure, - *, - now: float | None = None, - ) -> None: - if failure.category not in { - VideoFailureCategory.RATE_LIMIT, - VideoFailureCategory.PROVIDER_UNAVAILABLE, - VideoFailureCategory.TIMEOUT, - VideoFailureCategory.CONNECTION, - }: - with self._lock: - state = self._states.get(key) - if state is not None and state.half_open_in_flight: - self._states.pop(key, None) - return - current = time.time() if now is None else now - with self._lock: - state = self._states.setdefault(key, _CircuitState()) - state.half_open_in_flight = False - state.consecutive_failures += 1 - if state.consecutive_failures >= self.threshold: - state.open_until = current + self.cooldown_seconds - - def snapshot(self) -> dict[str, dict[str, Any]]: - with self._lock: - return { - key: { - "consecutive_failures": state.consecutive_failures, - "open_until": state.open_until, - "half_open_in_flight": state.half_open_in_flight, - } - for key, state in self._states.items() - } +class VideoCircuitBreaker(CircuitBreaker): + """Context-scoped circuit breaker keyed by provider/model identity.""" diff --git a/artemis/config/agent.py b/artemis/config/agent.py index 8f1c771..0ec8fcc 100644 --- a/artemis/config/agent.py +++ b/artemis/config/agent.py @@ -16,7 +16,7 @@ import os from pathlib import Path -from typing import Any +from typing import Any, Literal from pydantic import BaseModel, Field, model_validator @@ -143,11 +143,22 @@ class CommitteeConfig(BaseModel): class CheckerConfig(BaseModel): - """Configuration specific to Checker subgoal verification and rollback.""" + """Configuration for plan-driven checkpoint verification and the final review.""" enabled: bool = Field( default=False, - description="Whether to enable visual verification and snapshot rollback by the Checker agent.", + description=( + "Master switch (compat alias): False disables BOTH midway" + " checkpoints and the final check." + ), + ) + midway_checks: bool = Field( + default=True, + description="Whether plan-declared midway checkpoints run (requires enabled=True).", + ) + final_check: bool = Field( + default=True, + description="Whether the exit final review runs (requires enabled=True).", ) max_iterations: int = Field( default=20, @@ -155,11 +166,41 @@ class CheckerConfig(BaseModel): le=50, description="Maximum verification iterations for Checker LLM reasoning and inspection.", ) - max_chat_rounds: int = Field( - default=4, + final_check_max_attempts: int = Field( + default=3, ge=1, le=10, - description="Maximum debate/rebuttal rounds between Operator and Checker.", + description="Maximum final-review attempts before ending with a blocked outcome.", + ) + checkpoint_max_repairs: int = Field( + default=2, + ge=0, + le=10, + description="Per-checkpoint repair quota for failed verify criteria.", + ) + max_concurrent_checkpoints: int = Field( + default=3, + ge=1, + le=10, + description="Maximum concurrently running checkpoint attempts.", + ) + checkpoint_timeout: float = Field( + default=180.0, + gt=0, + description="Per-attempt timeout (seconds) for one checkpoint check.", + ) + settlement_timeout: float = Field( + default=120.0, + gt=0, + description="Aggregate timeout (seconds) for the exit settlement barrier.", + ) + assert_failure_policy: Literal["continue", "halt"] = Field( + default="continue", + description="Whether an assert failure lets execution continue or halts the run.", + ) + device_probes: bool = Field( + default=True, + description="Whether the Checker's enumerated read-only device probes are registered.", ) model_config = {"extra": "allow"} diff --git a/artemis/config/settings.py b/artemis/config/settings.py index 7ae6e18..175a5d0 100644 --- a/artemis/config/settings.py +++ b/artemis/config/settings.py @@ -131,6 +131,16 @@ class Settings(BaseSettings): description="Enable context caching when running multi-turn pro/ultra Explorer", ) + # LLM Reliability + LLM_PAUSE_TIMEOUT_SECONDS: float = Field( + default=900.0, + description=( + "How long a task waits in the paused state after exhausting LLM" + " retries before failing with LLMExhaustedError. <= 0 waits" + " forever (legacy interactive behavior)." + ), + ) + # Paths & Storage TRACES_PATH: Path = Field(default_factory=get_default_traces_path) DATA_ENGINE_DB_PATH: Path = Field(default_factory=get_data_engine_db_path) diff --git a/artemis/constants.py b/artemis/constants.py index 60db514..7af5245 100644 --- a/artemis/constants.py +++ b/artemis/constants.py @@ -14,11 +14,9 @@ RECURSION_LIMIT = 30000 MAX_MESSAGES_IN_HISTORY = 25 -VALIDATOR_MESSAGES_KEY = "validator_messages" # Optimization Constants CHECKER_MAX_ITERATIONS = 20 -OPERATOR_MAX_CHAT_ROUNDS = 4 VALIDATOR_POLL_TIMEOUT = 2.0 VALIDATOR_POLL_INTERVAL = 0.2 VALIDATOR_OCR_INTERVAL = 1.0 diff --git a/artemis/context.py b/artemis/context.py index 9b22b53..487f8f7 100644 --- a/artemis/context.py +++ b/artemis/context.py @@ -106,8 +106,18 @@ class ExecutionSetup(BaseModel): app_lock_status: AppLaunchResult | None = None video_recording_tools_enabled: bool = Field(default_factory=detect_video_tools_enabled) disable_checker: bool = True + """Legacy master switch (compat alias): ``True`` disables BOTH the midway + checkpoints and the final check, regardless of the individual gates below.""" + disable_midway_checks: bool = False + disable_final_check: bool = False checker_max_iterations: int = 20 - checker_max_chat_rounds: int = 4 + final_check_max_attempts: int = 3 + checkpoint_max_repairs: int = 2 + max_concurrent_checkpoints: int = 3 + checkpoint_timeout: float = 180.0 + settlement_timeout: float = 120.0 + assert_failure_policy: Literal["continue", "halt"] = "continue" + disable_device_probes: bool = False disable_planner_validation: bool = True planner_validation_threshold: float = 0.85 enable_committee: bool = False @@ -117,6 +127,23 @@ class ExecutionSetup(BaseModel): explorer: ExplorerConfig = Field(default_factory=ExplorerConfig) explorer_versions: dict[str, str] = Field(default_factory=dict) + @property + def midway_checks_enabled(self) -> bool: + """Midway checkpoint gate: honors both the individual switch and the + legacy ``disable_checker`` master alias.""" + return not (self.disable_midway_checks or self.disable_checker) + + @property + def final_check_enabled(self) -> bool: + """Final review gate: honors both the individual switch and the legacy + ``disable_checker`` master alias.""" + return not (self.disable_final_check or self.disable_checker) + + @property + def checks_enabled(self) -> bool: + """True when any check gate is active (drives conditional prompt assembly).""" + return self.midway_checks_enabled or self.final_check_enabled + @property def explorer_version(self) -> str: return self.explorer.default_version @@ -160,20 +187,43 @@ class ArtemisContext(BaseModel): execution_setup: ExecutionSetup | None = None data_engine: DataEngine | None = None - checker_task: asyncio.Task[Any] | None = None planner_task: asyncio.Task[Any] | None = None background_tasks: list[asyncio.Task[Any]] = Field(default_factory=list) background_task_grace_period_seconds: float = 2.0 package_cache: dict[str, str | None] = Field(default_factory=dict) background_jobs: dict[str, dict[str, Any]] = Field(default_factory=dict) - task_plan_snapshot: Path | None = None + pending_checkpoints: list[Any] = Field(default_factory=list) + """Queued :class:`~artemis.graph.checkpoints.PendingCheckpoint` entries. + ``_process_plan_write`` only enqueues; ``execution_check_node`` spawns after + the turn's step is recorded so the evidence anchor points at a real step.""" + checkpoint_tasks: dict[str, Any] = Field(default_factory=dict) + """checkpoint_id -> (attempt_id, asyncio.Task). One in-flight attempt per + checkpoint; supersede harvests/cancels the old entry before replacing it.""" + checkpoint_attempt_seq: dict[str, int] = Field(default_factory=dict) + """checkpoint_id -> monotonically increasing attempt counter.""" + checkpoint_repairs: dict[str, int] = Field(default_factory=dict) + """checkpoint_id -> number of verify-fail repairs already applied.""" + assert_halt: bool = False + """Latched by an assert failure under ``assert_failure_policy='halt'``.""" + final_check_attempts: int = 0 + """Number of final-check passes already executed at exit settlement.""" + task_plan_content_before: str | None = None + last_validated_plan: str | None = None + """Ratchet baseline for planner validation: the last plan content whose + top-level milestones were validated (or the initial plan). Drift is always + judged against this, never against the immediately preceding write.""" + pending_validated_plan: str | None = None + """Plan content currently under async planner validation; becomes the new + baseline when the validator approves it.""" _genai_client: Any | None = PrivateAttr(default=None) _video_blackboard: Any | None = PrivateAttr(default=None) _video_circuit_breaker: Any | None = PrivateAttr(default=None) _mobile_controller: Any | None = PrivateAttr(default=None) + _active_driver: Any | None = PrivateAttr(default=None) + """Cached device driver singleton (artemis.drivers.factory.get_driver).""" mcp_client_ctx: Any | None = None mcp_session: Any | None = None action_session: Any | None = None @@ -200,9 +250,7 @@ class ArtemisContext(BaseModel): tasks = list(self.background_tasks) pending_tasks = [task for task in tasks if not task.done()] grace_period = ( - max(0.0, self.background_task_grace_period_seconds) - if exc_type is None - else 0.0 + max(0.0, self.background_task_grace_period_seconds) if exc_type is None else 0.0 ) if pending_tasks and grace_period > 0: diff --git a/artemis/core/context.py b/artemis/core/context.py index 51a1b6d..0815e33 100644 --- a/artemis/core/context.py +++ b/artemis/core/context.py @@ -50,6 +50,9 @@ class ExecutionContext: self.device = DeviceInfoContext(device_id=device_id, platform=platform) self.driver: Any | None = None self.telemetry: Any | None = None + # Fully-wired ArtemisContext (data_engine, adb clients, execution setup). + # Required by the live (non-mock) paths of GraphRunner/ReactiveRunner. + self.artemis_context: Any | None = None self.custom_attributes: dict[str, Any] = {} def set_attribute(self, key: str, value: Any) -> None: diff --git a/artemis/data_engine/engine.py b/artemis/data_engine/engine.py index 616ec2f..49fe54f 100644 --- a/artemis/data_engine/engine.py +++ b/artemis/data_engine/engine.py @@ -240,7 +240,9 @@ class DataEngine: except ValueError: pass else: - env_session_id = os.getenv("ARTEMIS_CLOUD_SESSION_ID") or os.getenv("ARTEMIS_SESSION_ID") + env_session_id = os.getenv("ARTEMIS_CLOUD_SESSION_ID") or os.getenv( + "ARTEMIS_SESSION_ID" + ) if env_session_id: try: session_id = UUID(env_session_id) @@ -293,7 +295,9 @@ class DataEngine: else: self._step_number_cache.clear() self.current_step_number = 0 - logger.info(f"Session started: {session_id} (current step counter: {self.current_step_number})") + logger.info( + f"Session started: {session_id} (current step counter: {self.current_step_number})" + ) self._publish("session_started", session.model_dump()) return session_id @@ -511,7 +515,9 @@ class DataEngine: existing_steps = self.storage.get_steps(self.current_session_id) or [] except Exception: existing_steps = [] - max_existing = max([s.step_number for s in existing_steps], default=0) if existing_steps else 0 + max_existing = ( + max([s.step_number for s in existing_steps], default=0) if existing_steps else 0 + ) self.current_step_number = max(self.current_step_number, max_existing) + 1 step_number = self.current_step_number @@ -940,6 +946,31 @@ class DataEngine: pass return None + def get_step_record(self, step_number: int): + """Read-only lookup of a step record by its 1-based step number.""" + if not self.current_session_id: + return None + try: + steps = self.storage.get_steps(self.current_session_id) or [] + except Exception as e: + logger.error(f"Failed to fetch steps for step lookup: {e}") + return None + for s in steps: + if s.step_number == step_number: + return s + return None + + def get_step_image_path(self, step_number: int, which: str = "pre") -> Path | None: + """Read-only path of a step's pre/post screenshot, if recorded.""" + step = self.get_step_record(step_number) + if step is None: + return None + image_name = step.pre_image_name if which == "pre" else step.post_image_name + if not image_name: + return None + path = self.get_image_path(image_name) + return path if path.exists() else None + def update_step_summary(self, step_id: UUID | int | str, summary: str): """Update the step with a concise summary in background.""" target_uuid = None diff --git a/artemis/engine/graph_runner.py b/artemis/engine/graph_runner.py index c8b43e1..632bf7f 100644 --- a/artemis/engine/graph_runner.py +++ b/artemis/engine/graph_runner.py @@ -14,14 +14,11 @@ """Pro Graph Runner orchestrating multi-agent state machines with self-healing.""" -from langchain_core.messages import HumanMessage - -from artemis.context import ArtemisContext, DeviceContext from artemis.core.state import ExecutionContextState, ExecutionStatus, StepRecord from artemis.drivers.mock.mock_driver import MockDeviceDriver from artemis.engine.base_runner import BaseRunner from artemis.graph.graph import get_graph -from artemis.platform import platform +from artemis.graph.state import State from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -45,16 +42,18 @@ class GraphRunner(BaseRunner): if not is_mock: try: - artemis_ctx = getattr(self.ctx, "artemis_context", None) + artemis_ctx = self.ctx.artemis_context if artemis_ctx is None: - artemis_ctx = ArtemisContext( - device=DeviceContext( - host_platform=platform.os_type.name, - device_id=self.driver.device_id, - ) + # Fail fast with a clear contract violation instead of + # silently building an empty ArtemisContext that drops + # data_engine/llm_config and dies later in perception. + raise RuntimeError( + "GraphRunner requires ExecutionContext.artemis_context" + " to be wired (data_engine, adb clients, execution" + " setup) before running the live graph path." ) graph = await get_graph(artemis_ctx) - graph_input = {"messages": [HumanMessage(content=self.ctx.task_goal)]} + graph_input = State.initial(self.ctx.task_goal) async for _ in graph.astream( input=graph_input, config={"recursion_limit": max_turns} ): diff --git a/artemis/engine/reactive_runner.py b/artemis/engine/reactive_runner.py index cd10cda..9a7a73c 100644 --- a/artemis/engine/reactive_runner.py +++ b/artemis/engine/reactive_runner.py @@ -18,12 +18,10 @@ import time from artemis.agents.flash.runner import FlashRunner from artemis.config import load_agent_config -from artemis.context import ArtemisContext, DeviceContext from artemis.core.state import ExecutionContextState, ExecutionStatus, StepRecord from artemis.drivers.mock.mock_driver import MockDeviceDriver from artemis.engine.base_runner import BaseRunner from artemis.graph.state import State -from artemis.platform import platform from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -54,21 +52,23 @@ class ReactiveRunner(BaseRunner): if not is_mock: try: - artemis_ctx = getattr(self.ctx, "artemis_context", None) + artemis_ctx = self.ctx.artemis_context if artemis_ctx is None: - artemis_ctx = ArtemisContext( - device=DeviceContext( - host_platform=platform.os_type.name, - device_id=self.driver.device_id, - ) + # Fail fast with a clear contract violation instead of + # silently building an empty ArtemisContext that drops + # data_engine/llm_config and dies later mid-run. + raise RuntimeError( + "ReactiveRunner requires ExecutionContext.artemis_context" + " to be wired (data_engine, adb clients, execution" + " setup) before running the live Flash path." ) flash_runner = FlashRunner( artemis_ctx, goal=self.ctx.task_goal, max_turns=max_turns ) - legacy_state = State(messages=[]) + legacy_state = State.initial(self.ctx.task_goal) result = await flash_runner.run(legacy_state) - state.current_turn = len(getattr(legacy_state, "messages", [])) or 1 + state.current_turn = 1 if result.get("status") == "completed": state.status = ExecutionStatus.SUCCESS else: diff --git a/artemis/graph/checkpoints.py b/artemis/graph/checkpoints.py new file mode 100644 index 0000000..ecdc46f --- /dev/null +++ b/artemis/graph/checkpoints.py @@ -0,0 +1,649 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Plan-driven checkpoint scheduling, the append-only verdict ledger, and exit +settlement bookkeeping. + +Three invariants from the redesign govern everything here: + +1. **Append-only history**: the Operator's context is never rolled back or + switched; verdicts influence the future (status flips forward, findings + injected) but never rewrite the past. +2. **Release / verdict / wrap-up separation**: fail-open only affects the + release decision — the verdict value itself (``inconclusive``) is always + recorded verbatim; assert failures are never repaired but always produce a + machine-readable test result. +3. **Zero side effects inside check tasks**: state mutation happens only at + harvest points (``execution_check_node``) and at exit settlement. +""" + +from __future__ import annotations + +import asyncio +import json +import re +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +from pydantic import BaseModel, Field + +from artemis.utils.logger import get_logger +from artemis.utils.notes import get_note_file_path +from artemis.utils.plan_grammar import ( + CHECKBOX_LINE_RE, + CheckItem, + PlanSnapshot, + parse_plan, + subgoal_hash, +) + +logger = get_logger(__name__) + +LEDGER_FILENAME = "check_ledger.jsonl" +RUN_OUTCOME_FILENAME = "run_outcome.json" + +# --- Config accessors (all fail-safe against a missing execution_setup) -------------- + + +def _setup(ctx) -> Any: + return getattr(ctx, "execution_setup", None) + + +def midway_checks_enabled(ctx) -> bool: + setup = _setup(ctx) + return bool(setup) and bool(getattr(setup, "midway_checks_enabled", False)) + + +def final_check_enabled(ctx) -> bool: + setup = _setup(ctx) + return bool(setup) and bool(getattr(setup, "final_check_enabled", False)) + + +def _setting(ctx, name: str, default): + setup = _setup(ctx) + value = getattr(setup, name, None) if setup else None + return default if value is None else value + + +# --- Data contracts ------------------------------------------------------------------ + + +@dataclass(frozen=True) +class EvidenceAnchor: + """Anchors a checkpoint to the evidence available at completion time.""" + + anchor_step_id: str | None + trigger_ts: float + plan_text: str + + +@dataclass(frozen=True) +class PendingCheckpoint: + """A queued checkpoint: enqueue happens at plan-write time, spawn happens in + ``execution_check_node`` after the turn's step is recorded.""" + + checkpoint_id: str + subgoal_text: str + check_items: tuple[CheckItem, ...] + plan_text: str + trigger_ts: float + + +@dataclass(frozen=True) +class CheckpointRun: + """One in-flight (or finished) check attempt held in ``ctx.checkpoint_tasks``.""" + + attempt_id: str + task: asyncio.Task + checkpoint: PendingCheckpoint + + +class TestSummary(BaseModel): + passed: int = 0 + failed: int = 0 + inconclusive: int = 0 + unchecked: int = 0 + failed_items: list[dict] = Field(default_factory=list) + + @property + def total(self) -> int: + return self.passed + self.failed + self.inconclusive + self.unchecked + + +class RunOutcome(BaseModel): + """Machine-readable run outcome: goal-achievement axis x test-result axis.""" + + task_status: Literal["completed", "partial", "blocked"] + tests: TestSummary + + +# --- Ledger (append-only) ------------------------------------------------------------ + + +def ledger_path(base_dir: str | Path) -> Path: + return Path(base_dir) / LEDGER_FILENAME + + +def append_ledger_record(base_dir: str | Path, record: dict) -> None: + """Appends one verdict record. Records are immutable once written; any later + result for the same item can only be appended as a new record.""" + record = dict(record) + record.setdefault("ts", time.time()) + path = ledger_path(base_dir) + try: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as f: + f.write(json.dumps(record, ensure_ascii=False) + "\n") + except Exception as e: + logger.error(f"Failed to append check ledger record: {e}") + + +def read_ledger(base_dir: str | Path) -> list[dict]: + path = ledger_path(base_dir) + if not path.exists(): + return [] + records: list[dict] = [] + try: + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except Exception: + logger.warning(f"Skipping malformed ledger line: {line[:120]}") + except Exception as e: + logger.error(f"Failed to read check ledger: {e}") + return records + + +def has_ledger_records(base_dir: str | Path) -> bool: + path = ledger_path(base_dir) + try: + return path.exists() and path.stat().st_size > 0 + except Exception: + return False + + +# --- Plan helpers -------------------------------------------------------------------- + + +def _read_plan_text(ctx) -> str: + if not getattr(ctx, "data_engine", None): + return "" + path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") + if not path.exists(): + return "" + try: + return path.read_text(encoding="utf-8") + except Exception as e: + logger.error(f"Failed to read task plan: {e}") + return "" + + +def _subgoal_still_current(ctx, checkpoint_id: str) -> bool: + """The anchored subgoal's text is unchanged iff an item with the same + content hash still exists in the current plan.""" + snapshot = parse_plan(_read_plan_text(ctx)) + return any(i.is_top_level and i.key == checkpoint_id for i in snapshot.items) + + +def revert_subgoal_status(ctx, subgoal_key: str) -> bool: + """Flips the matching top-level subgoal from ``[x]`` back to ``[/]``. + + This is a forward-looking state change (never a rollback of history) and is + written directly to the file, bypassing the wrapped note tools. + """ + if not getattr(ctx, "data_engine", None): + return False + path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") + if not path.exists(): + return False + try: + lines = path.read_text(encoding="utf-8").split("\n") + changed = False + for idx, line in enumerate(lines): + match = CHECKBOX_LINE_RE.match(line) + if not match or match.group("indent"): + continue + if subgoal_hash(match.group("text").strip()) == subgoal_key: + if match.group("status") == "x": + lines[idx] = re.sub(r"^(\s*-\s*\[)x(\])", r"\g<1>/\g<2>", line) + changed = True + break + if changed: + path.write_text("\n".join(lines), encoding="utf-8") + return changed + except Exception as e: + logger.error(f"Failed to revert subgoal status: {e}") + return False + + +# --- Queueing (called from _process_plan_write; never spawns) ------------------------ + + +def queue_checkpoints( + ctx, + state, + after: PlanSnapshot, + new_completions: list[str], + plan_text: str, +) -> None: + """Enqueues a pending checkpoint for every newly completed top-level subgoal + that carries ``on_complete`` check items. + + Spawning is deliberately deferred to ``execution_check_node``: at this + moment the turn's step is not yet recorded, so an anchor taken here would + point at the previous turn. + """ + if not midway_checks_enabled(ctx): + return + if state is not None and getattr(state, "user_stop_requested", False): + return + items_by_key = {i.key: i for i in after.top_level} + for text in new_completions: + key = subgoal_hash(text) + item = items_by_key.get(key) + if item is None: + continue + check_items = tuple(ci for ci in after.check_items_of(item) if ci.when == "on_complete") + if not check_items: + continue + ctx.pending_checkpoints.append( + PendingCheckpoint( + checkpoint_id=key, + subgoal_text=text, + check_items=check_items, + plan_text=plan_text, + trigger_ts=time.time(), + ) + ) + logger.info(f"Queued checkpoint for completed subgoal: {text}") + + +# --- Harvest (§5.2) ------------------------------------------------------------------ + + +def _record_verdicts( + ctx, + run: CheckpointRun, + verdicts: list, + anchor_step_id: str | None, +) -> None: + base_dir = ctx.data_engine.base_dir + when_by_text = {(ci.kind, ci.text): ci.when for ci in run.checkpoint.check_items} + for v in verdicts: + append_ledger_record( + base_dir, + { + "attempt_id": run.attempt_id, + "checkpoint_id": run.checkpoint.checkpoint_id, + "item_text": v.item_text, + "kind": v.kind, + "when": when_by_text.get((v.kind, v.item_text), "on_complete"), + "status": v.status, + "evidence": v.evidence, + "suggestion": getattr(v, "suggestion", ""), + "anchor_step_id": anchor_step_id, + }, + ) + + +def _record_attempt_failure(ctx, run: CheckpointRun, status: str, evidence: str) -> None: + base_dir = ctx.data_engine.base_dir + for ci in run.checkpoint.check_items: + append_ledger_record( + base_dir, + { + "attempt_id": run.attempt_id, + "checkpoint_id": run.checkpoint.checkpoint_id, + "item_text": ci.text, + "kind": ci.kind, + "when": ci.when, + "status": status, + "evidence": evidence, + "anchor_step_id": None, + }, + ) + + +def harvest_run( + ctx, + state, + run: CheckpointRun, + *, + allow_side_effects: bool, + cancelled_status: str = "superseded", + anchor_step_id: str | None = None, +) -> list[str]: + """Books a finished (done/cancelled) attempt into the ledger and, when the + verdict is still applicable, applies its execution-state side effects. + + Returns Operator-facing findings (injected as ``operator_feedback``). The + verdict values themselves are never altered by the release decision: + an errored verify is *released* (fail-open) but *recorded* inconclusive. + """ + task = run.task + findings: list[str] = [] + checkpoint = run.checkpoint + + if task.cancelled(): + evidence = ( + "attempt superseded by a newer completion of the same subgoal" + if cancelled_status == "superseded" + else "cancelled at exit settlement (settlement timeout)" + ) + _record_attempt_failure(ctx, run, cancelled_status, evidence) + return findings + + exc = task.exception() + if exc is not None: + status = "inconclusive" + reason = ( + f"check timed out after {_setting(ctx, 'checkpoint_timeout', 180.0)}s" + if isinstance(exc, (asyncio.TimeoutError, TimeoutError)) + else f"check raised an exception: {exc}" + ) + logger.warning( + f"Checkpoint attempt {run.attempt_id} did not produce a verdict" + f" ({reason}); releasing (fail-open) but recording inconclusive." + ) + _record_attempt_failure(ctx, run, status, reason) + return findings + + report = task.result() + verdicts = list(getattr(report, "verdicts", []) or []) + _record_verdicts(ctx, run, verdicts, anchor_step_id) + + # Applicability gate for side effects: the verdict must belong to the + # current attempt of its checkpoint AND the anchored subgoal's text must be + # unchanged in the current plan. Stale/mismatched verdicts are ledger-only. + applicable = allow_side_effects and _subgoal_still_current(ctx, checkpoint.checkpoint_id) + user_stopped = bool(state is not None and getattr(state, "user_stop_requested", False)) + + verify_failures = [v for v in verdicts if v.kind == "verify" and v.status == "failed"] + assert_failures = [v for v in verdicts if v.kind == "assert" and v.status == "failed"] + + if verify_failures and applicable and not user_stopped: + max_repairs = int(_setting(ctx, "checkpoint_max_repairs", 2)) + repairs = ctx.checkpoint_repairs.get(checkpoint.checkpoint_id, 0) + if repairs < max_repairs: + ctx.checkpoint_repairs[checkpoint.checkpoint_id] = repairs + 1 + reverted = revert_subgoal_status(ctx, checkpoint.checkpoint_id) + for v in verify_failures: + suggestion = getattr(v, "suggestion", "") or "" + findings.append( + f"[verify failed] Subgoal '{checkpoint.subgoal_text}' —" + f" criterion '{v.item_text}': {v.evidence}" + + (f" Suggestion: {suggestion}" if suggestion else "") + ) + if reverted: + findings.append( + f"The subgoal '{checkpoint.subgoal_text}' has been set back to" + " in-progress ([/]). Address the findings above, then complete" + " it again." + ) + else: + logger.warning( + f"Checkpoint {checkpoint.checkpoint_id} exhausted its repair" + f" quota ({max_repairs}); verdict stays failed, no further" + " repair loop." + ) + + if assert_failures: + policy = str(_setting(ctx, "assert_failure_policy", "continue")) + if policy == "halt": + ctx.assert_halt = True + logger.warning( + "Assert failure under 'halt' policy: latching halt flag for exit settlement." + ) + + return findings + + +def harvest_finished_checkpoints(ctx, state) -> list[str]: + """Non-blocking harvest: books every ``done()`` attempt, never awaits a + running one. Returns Operator-facing findings.""" + if not getattr(ctx, "data_engine", None): + return [] + findings: list[str] = [] + for cid, run in list(ctx.checkpoint_tasks.items()): + if not run.task.done(): + continue + del ctx.checkpoint_tasks[cid] + findings.extend(harvest_run(ctx, state, run, allow_side_effects=True)) + return findings + + +# --- Spawn (§5.1: after record_step) ------------------------------------------------- + + +async def spawn_pending_checkpoints(ctx, state, anchor_step_id: str | None) -> list[str]: + """Spawns queued checkpoints against the just-recorded step, honoring the + concurrency cap and superseding stale attempts without losing verdicts. + + Returns findings harvested from superseded-but-finished attempts. + """ + findings: list[str] = [] + if not getattr(ctx, "data_engine", None): + return findings + if not midway_checks_enabled(ctx): + return findings + if state is not None and getattr(state, "user_stop_requested", False): + return findings + + from artemis.agents.checker.checker import run_checkpoint_check + + max_concurrent = int(_setting(ctx, "max_concurrent_checkpoints", 3)) + timeout = float(_setting(ctx, "checkpoint_timeout", 180.0)) + goal = getattr(state, "initial_goal", "") if state is not None else "" + + while ctx.pending_checkpoints and len(ctx.checkpoint_tasks) < max_concurrent: + pending: PendingCheckpoint = ctx.pending_checkpoints.pop(0) + cid = pending.checkpoint_id + + old_run: CheckpointRun | None = ctx.checkpoint_tasks.pop(cid, None) + if old_run is not None: + if old_run.task.done(): + # A finished-but-unharvested attempt: book it first (verdicts, + # including failures, are never dropped), ledger-only. + findings.extend(harvest_run(ctx, state, old_run, allow_side_effects=False)) + else: + old_run.task.cancel() + try: + await old_run.task + except (asyncio.CancelledError, Exception): + pass + harvest_run(ctx, state, old_run, allow_side_effects=False) + + seq = ctx.checkpoint_attempt_seq.get(cid, 0) + 1 + ctx.checkpoint_attempt_seq[cid] = seq + attempt_id = f"{cid}#{seq}" + anchor = EvidenceAnchor( + anchor_step_id=anchor_step_id, + trigger_ts=pending.trigger_ts, + plan_text=pending.plan_text, + ) + task = asyncio.create_task( + asyncio.wait_for( + run_checkpoint_check( + ctx, + check_items=pending.check_items, + anchor=anchor, + goal=goal, + subgoal_text=pending.subgoal_text, + ), + timeout=timeout, + ) + ) + ctx.checkpoint_tasks[cid] = CheckpointRun( + attempt_id=attempt_id, task=task, checkpoint=pending + ) + logger.info(f"Spawned checkpoint attempt {attempt_id} anchored at step {anchor_step_id}") + + return findings + + +# --- Exit settlement barrier (§5.3 phase 1) ------------------------------------------ + + +async def settle_all_checkpoints(ctx, state) -> None: + """The one intentional blocking wait of the whole flow: collects every + outstanding checkpoint before exit, bounded by ``settlement_timeout``. + Attempts still running at the deadline are cancelled and recorded + ``unchecked`` (settlement timeout). Runs unconditionally — a started check + must be booked before exit regardless of the final-check switch.""" + if not getattr(ctx, "data_engine", None): + return + runs = list(ctx.checkpoint_tasks.values()) + ctx.checkpoint_tasks.clear() + if not runs: + return + + timeout = float(_setting(ctx, "settlement_timeout", 120.0)) + tasks = [r.task for r in runs] + try: + _done, pending = await asyncio.wait(tasks, timeout=timeout) + except Exception as e: + logger.error(f"Settlement wait failed: {e}") + pending = {t for t in tasks if not t.done()} + + for t in pending: + t.cancel() + for t in pending: + try: + await t + except (asyncio.CancelledError, Exception): + pass + + for run in runs: + # Settlement is bookkeeping, not a repair point: side effects off. + harvest_run( + ctx, + state, + run, + allow_side_effects=False, + cancelled_status="unchecked", + ) + + +# --- Run outcome --------------------------------------------------------------------- + +_RESOLVABLE = ("passed", "failed", "inconclusive") + + +def resolve_item_status(kind: str, records: list[dict]) -> str: + """Resolves one check item's final status from its ledger records. + + Assert: the first failure is permanent — any failed record makes the item + failed. Verify: the latest substantive verdict wins (a repaired-then-passed + verify is passed). No substantive record at all -> unchecked. + """ + substantive = [r for r in records if r.get("status") in _RESOLVABLE] + if kind == "assert" and any(r.get("status") == "failed" for r in substantive): + return "failed" + if substantive: + return substantive[-1]["status"] + return "unchecked" + + +def compute_test_summary(check_items: list[CheckItem], records: list[dict]) -> TestSummary: + summary = TestSummary() + by_item: dict[tuple[str, str], list[dict]] = {} + for r in records: + by_item.setdefault((str(r.get("kind")), str(r.get("item_text"))), []).append(r) + + seen: set[tuple[str, str]] = set() + all_items: list[tuple[str, str]] = [] + for ci in check_items: + sig = (ci.kind, ci.text) + if sig not in seen: + seen.add(sig) + all_items.append(sig) + # Items that only ever existed in the ledger (e.g. their plan lines were + # legitimately revised by the Planner) still count. + for sig in by_item: + if sig not in seen: + seen.add(sig) + all_items.append(sig) + + for kind, text in all_items: + status = resolve_item_status(kind, by_item.get((kind, text), [])) + if status == "passed": + summary.passed += 1 + elif status == "failed": + summary.failed += 1 + failing = [r for r in by_item.get((kind, text), []) if r.get("status") == "failed"] + summary.failed_items.append( + { + "item_text": text, + "kind": kind, + "evidence": failing[-1].get("evidence", "") if failing else "", + } + ) + elif status == "inconclusive": + summary.inconclusive += 1 + else: + summary.unchecked += 1 + return summary + + +def compute_run_outcome( + snapshot: PlanSnapshot, + records: list[dict], + *, + verify_blocked: bool, +) -> RunOutcome: + """Assembles the machine-readable run outcome. + + ``verify_blocked`` marks the "verify unmet and budget exhausted" exit — + the goal-achievement axis then reads blocked even though the plan claims + completion. Assert failures never change ``task_status``: a finished task + with failed assertions is ``completed`` with ``tests.failed > 0``. + """ + tests = compute_test_summary(list(snapshot.all_check_items), records) + if verify_blocked: + task_status: Literal["completed", "partial", "blocked"] = "blocked" + elif snapshot.all_top_level_done: + task_status = "completed" + else: + task_status = "partial" + return RunOutcome(task_status=task_status, tests=tests) + + +def write_run_outcome(base_dir: str | Path, outcome: RunOutcome, extra: dict | None = None) -> None: + try: + payload = outcome.model_dump() + if extra: + payload.update(extra) + path = Path(base_dir) / RUN_OUTCOME_FILENAME + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + except Exception as e: + logger.error(f"Failed to write run outcome: {e}") + + +def read_run_outcome(base_dir: str | Path) -> dict | None: + path = Path(base_dir) / RUN_OUTCOME_FILENAME + if not path.exists(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except Exception: + return None diff --git a/artemis/graph/graph.py b/artemis/graph/graph.py index 05a4a7a..c44a80f 100644 --- a/artemis/graph/graph.py +++ b/artemis/graph/graph.py @@ -14,24 +14,20 @@ import asyncio import base64 -import difflib import functools import json -import re -import shutil from typing import Annotated, Literal -from langchain_core.messages import SystemMessage +from langchain_core.messages import ToolMessage from langchain_core.tools import StructuredTool from langchain_core.tools.base import InjectedToolCallId from langgraph.constants import END, START from langgraph.graph import StateGraph from langgraph.graph.state import CompiledStateGraph from langgraph.prebuilt import InjectedState -from langgraph.types import Command from pydantic import BaseModel, Field -from artemis.agents.checker.checker import run_async_check +from artemis.agents.checker.checker import run_final_check, verdicts_allow_release from artemis.agents.operator.operator import OperatorNode from artemis.agents.planner.planner import ( PlannerNode, @@ -39,10 +35,23 @@ from artemis.agents.planner.planner import ( ) from artemis.agents.summarizer.summarizer import SummarizerNode from artemis.agents.validator.validator import ValidatorNode -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext +from artemis.graph.checkpoints import ( + append_ledger_record, + compute_run_outcome, + final_check_enabled, + harvest_finished_checkpoints, + has_ledger_records, + queue_checkpoints, + read_ledger, + revert_subgoal_status, + settle_all_checkpoints, + spawn_pending_checkpoints, + write_run_outcome, +) from artemis.graph.perception import perception_node from artemis.graph.state import State +from artemis.graph.visibility import strict_state from artemis.tools.command_tool import ( manage_task_wrapper, run_adb_command_wrapper, @@ -60,7 +69,6 @@ from artemis.tools.scratchpad import ( ) from artemis.tools.tool_wrapper import invoke_tool_with_injection from artemis.tools.video_tool import get_video_analyzer_tool -from artemis.utils.file import restore_snapshot from artemis.utils.logger import get_logger from artemis.utils.notes import ( SAVE_NOTE_ARG_CONTENT_DESC, @@ -69,7 +77,14 @@ from artemis.utils.notes import ( UPDATE_NOTE_ARG_REPLACEMENT_DESC, UPDATE_NOTE_ARG_TARGET_DESC, get_note_file_path, - get_notes_dir, +) +from artemis.utils.plan_grammar import ( + milestones_changed, + new_top_level_completions, + parse_plan, + restore_missing_check_items, + subgoal_hash, + unintended_milestone_edits, ) from artemis.utils.task_tree import get_active_subgoal_hashes @@ -82,196 +97,293 @@ def convergence_node(state: State): async def execution_check_node(state: State, ctx: ArtemisContext): + state = strict_state(state, "execution_check") logger.info("Starting execution_check_node") if not ctx.data_engine: return {"checker_success": True} checker_success = True - disable_checker = ctx.execution_setup and ctx.execution_setup.disable_checker - if disable_checker: - logger.info("Checker is disabled via flag. Skipping checker invocation.") - else: - if ( - (not hasattr(ctx, "checker_task") or not ctx.checker_task) - and ctx.data_engine - and getattr(state, "operator_replied", False) - ): - subgoal_hash, _ = _get_active_subgoal_hashes(ctx) - notes_dir = get_notes_dir(ctx.data_engine.base_dir) - chat_path = notes_dir / f"verification_chat_{subgoal_hash}.json" - - if chat_path.exists(): - subgoal_text = "Unknown subgoal" - task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") - if task_plan_path.exists(): - try: - content = task_plan_path.read_text(encoding="utf-8") - lines = content.split("\n") - for line in lines: - if line.strip().startswith("- [/]"): - subgoal_text = line.strip()[5:].strip() - break - except Exception as e: - logger.error(f"Failed to parse active subgoal text: {e}") - - raw_perception = getattr(state, "operator_raw_data", None) - ui_hierarchy = getattr(state, "latest_ui_hierarchy", None) - - ctx.checker_task = asyncio.create_task( - run_async_check( - ctx, - subgoal_text, - subgoal_hash=subgoal_hash, - raw_perception_data=raw_perception, - latest_ui_hierarchy=ui_hierarchy, - ) - ) - - if hasattr(ctx, "checker_task") and ctx.checker_task: - logger.info("Waiting for checker task to complete...") + def _record_turn(extra_metadata: dict) -> str | None: + """Records this Operator turn in the DataEngine — unconditionally: every + turn (including planner-rejected ones) leaves a step record.""" + raw_data = state.operator_raw_data + if not raw_data: + return None try: - result = await ctx.checker_task - ctx.checker_task = None # Clear it + screenshot_b64 = raw_data.get("screenshot_b64") + xml_hierarchy = raw_data.get("xml_hierarchy") + ocr_results = raw_data.get("ocr_results") - if result: - status = result.get("status") - if status == "success": - logger.info("Checker succeeded.") - checker_success = True - elif status == "failed": - logger.warning("Checker failed.") - checker_success = False - else: - logger.warning("Checker task returned None.") - checker_success = False + screenshot_bytes = base64.b64decode(screenshot_b64) + + action_taken = None + if state.structured_decisions: + try: + action_taken = json.loads(state.structured_decisions) + except Exception as e: + logger.warning(f"Failed to parse structured decisions: {e}") + + subgoal_hash_val, sub_subgoal_hash = _get_active_subgoal_hashes(ctx) + step_id = ctx.data_engine.record_step( + pre_screenshot_bytes=screenshot_bytes, + ui_tree=xml_hierarchy, + ocr_result=ocr_results, + action_taken=action_taken, + operator_raw_thinking=getattr(state, "operator_raw_thinking", None), + operator_native_thinking=getattr(state, "operator_native_thinking", None), + extra_metadata={ + "subgoal_hash": subgoal_hash_val, + "sub_subgoal_hash": sub_subgoal_hash, + "width": raw_data.get("width"), + "height": raw_data.get("height"), + **extra_metadata, + }, + ) + logger.info(f"Recorded step in DataEngine: {step_id}") + return str(step_id) except Exception as e: - logger.error(f"Checker task failed: {e}") - checker_success = False + logger.error(f"Failed to record step in DataEngine: {e}") + return None + + # Harvest finished checkpoint attempts (non-blocking: done() tasks only). + check_findings = harvest_finished_checkpoints(ctx, state) if hasattr(ctx, "planner_task") and ctx.planner_task: logger.info("Waiting for planner validation task to complete...") + awaited_planner_task = ctx.planner_task try: - planner_result = await ctx.planner_task + planner_result = await awaited_planner_task ctx.planner_task = None if planner_result and planner_result.get("status") == "failed": logger.warning("Planner rejected the task plan changes.") - # Targeted rollback: restore task_plan.md only - if hasattr(ctx, "task_plan_content_before"): + # Targeted rollback: restore task_plan.md to the last content + # consistent with the validated baseline + if ctx.task_plan_content_before is not None: task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") if task_plan_path.exists(): task_plan_path.write_text(ctx.task_plan_content_before, encoding="utf-8") logger.info("Rolled back task_plan.md to its original state.") + ctx.pending_validated_plan = None + ctx.task_plan_content_before = None - # Intercept action and inject message + # Surface the rejection through operator_feedback — the only + # feedback channel the Operator's next prompt actually renders. feedback = planner_result.get("feedback", "Your task plan changes were rejected.") - - system_msg = SystemMessage( - content=( - "Your recent modification to the top-level task plan" - " was rejected by the Planner. You should view this" - f" feedback critically.\n\nReason: {feedback}\n\nThe" - " task plan has been rolled back to its previous" - " state. Any terminal actions you outputted were not" - " executed. Please review the task goal again and" - " consider what the optimal strategy is now." - ) + check_findings.append( + "[planner] Your recent modification to the top-level task" + " plan was rejected and the plan was rolled back to its" + f" previous state. Reason: {feedback} Any terminal actions" + " you outputted were not executed. Review the task goal" + " again and consider what the optimal strategy is now." ) + # Even a rejected turn leaves a step record: the record reflects + # the decision made and the fact that its terminal actions were + # intercepted before execution. + rejected_step_id = _record_turn({"planner_rejected": True}) + return { "checker_success": False, - "operator_replied": False, - VALIDATOR_MESSAGES_KEY: [system_msg], "structured_decisions": "", + "current_step_id": rejected_step_id, + "operator_feedback": check_findings or None, } else: logger.info("Planner approved the task plan changes.") + # Advance the ratchet baseline: the validated content becomes + # the new reference for future drift comparisons. + if ctx.pending_validated_plan is not None: + ctx.last_validated_plan = ctx.pending_validated_plan + ctx.pending_validated_plan = None + ctx.task_plan_content_before = None + except asyncio.CancelledError: + if not awaited_planner_task.cancelled(): + raise # this node itself is being cancelled, not the awaited task + logger.info("Planner validation task was superseded by a newer plan write.") + ctx.planner_task = None except Exception as e: logger.error(f"Planner validation task failed: {e}") - current_step_id = None - if checker_success: - raw_data = state.operator_raw_data - if raw_data: - try: - screenshot_b64 = raw_data.get("screenshot_b64") - xml_hierarchy = raw_data.get("xml_hierarchy") - ocr_results = raw_data.get("ocr_results") + # Every Operator turn is recorded — verification never gates history. + current_step_id = _record_turn({}) - screenshot_bytes = base64.b64decode(screenshot_b64) - - action_taken = None - if state.structured_decisions: - try: - action_taken = json.loads(state.structured_decisions) - except Exception as e: - logger.warning(f"Failed to parse structured decisions: {e}") - - subgoal_hash, sub_subgoal_hash = _get_active_subgoal_hashes(ctx) - step_id = ctx.data_engine.record_step( - pre_screenshot_bytes=screenshot_bytes, - ui_tree=xml_hierarchy, - ocr_result=ocr_results, - action_taken=action_taken, - operator_raw_thinking=getattr(state, "operator_raw_thinking", None), - operator_native_thinking=getattr(state, "operator_native_thinking", None), - extra_metadata={ - "subgoal_hash": subgoal_hash, - "sub_subgoal_hash": sub_subgoal_hash, - "width": raw_data.get("width"), - "height": raw_data.get("height"), - }, - ) - current_step_id = str(step_id) - logger.info(f"Recorded step in DataEngine: {current_step_id}") - - except Exception as e: - logger.error(f"Failed to record step in DataEngine: {e}") - - # Handle optimistic execution rollback/commit - if ctx.data_engine and getattr(ctx, "task_plan_snapshot", None): - snapshot_dir = ctx.task_plan_snapshot - notes_dir = get_notes_dir(ctx.data_engine.base_dir) - - if checker_success: - logger.info("Checker succeeded. Committing optimistic changes (cleaning up snapshot).") - if snapshot_dir.exists(): - await asyncio.to_thread(shutil.rmtree, snapshot_dir) - logger.info(f"Offloaded snapshot deletion at {snapshot_dir} via asyncio.to_thread") - else: - logger.warning("Checker failed. Rolling back optimistic changes.") - if snapshot_dir.exists(): - - def _restore_and_preserve(): - chat_contents = {} - for p in notes_dir.glob("verification_chat_*.json"): - chat_contents[p.name] = p.read_text(encoding="utf-8") - restore_snapshot(snapshot_dir, notes_dir) - for name, content in chat_contents.items(): - (notes_dir / name).write_text(content, encoding="utf-8") - logger.info(f"Restored verification chat file: {name}") - - await asyncio.to_thread(_restore_and_preserve) - logger.info(f"Restored notes from snapshot {snapshot_dir}") - else: - raise FileNotFoundError( - f"Snapshot directory {snapshot_dir} not found for rollback!" - ) + # Spawn queued checkpoints only now: the just-recorded step is the evidence + # anchor (its pre screenshot plus the previous turn's post state capture + # the completion moment). Superseded-but-finished attempts are booked here. + check_findings.extend(await spawn_pending_checkpoints(ctx, state, current_step_id)) update = { "checker_success": checker_success, "current_step_id": current_step_id, - "operator_replied": False, + "operator_feedback": check_findings or None, } if checker_success: update["subagent_calls"] = [] return update +async def exit_settlement_node(state: State, ctx: ArtemisContext): + """Two-phase exit: settlement (unconditional) then final review (gated). + + Phase 1 — settlement barrier: the only intentional blocking wait in the + whole flow. Every started check attempt gets booked into the ledger before + exit, no matter which switches are on. + + Phase 2 — final review (``disable_final_check`` off): audits the USER'S + ORIGINAL GOAL plus all declared check items. Unmet verify criteria route + back into the loop (bounded by ``final_check_max_attempts``); assert + failures never do. + """ + state = strict_state(state, "exit_settlement") + logger.info("Starting exit_settlement_node") + update: dict = {"exit_settlement_route": "end"} + if not ctx.data_engine: + return update + + # Phase 1: settle every outstanding attempt (bounded by settlement_timeout). + await settle_all_checkpoints(ctx, state) + # Queued-but-never-spawned checkpoints are dropped here: their items simply + # remain unchecked and are reported as such. + ctx.pending_checkpoints.clear() + + base_dir = ctx.data_engine.base_dir + plan_text = "" + plan_path = get_note_file_path(base_dir, "task_plan") + if plan_path.exists(): + try: + plan_text = plan_path.read_text(encoding="utf-8") + except Exception as e: + logger.error(f"Failed to read task plan at settlement: {e}") + snapshot = parse_plan(plan_text) + + setup = ctx.execution_setup + user_stopped = bool(getattr(state, "user_stop_requested", False)) + halted = bool(getattr(ctx, "assert_halt", False)) + max_attempts = int(getattr(setup, "final_check_max_attempts", 3) or 3) if setup else 3 + + verify_blocked = False + blocked_findings: list[str] = [] + if final_check_enabled(ctx): + attempt_no = ctx.final_check_attempts + 1 + ctx.final_check_attempts = attempt_no + checkpoint_timeout = float(getattr(setup, "checkpoint_timeout", 180.0) or 180.0) + ledger = read_ledger(base_dir) + try: + report = await asyncio.wait_for( + run_final_check( + ctx, + goal=getattr(state, "initial_goal", ""), + plan_text=plan_text, + ledger=ledger, + check_items=list(snapshot.all_check_items), + ), + timeout=checkpoint_timeout, + ) + except Exception as e: + # Fail-open: release, but the verdict value stays inconclusive. + logger.warning(f"Final check errored ({e}); releasing fail-open.") + for ci in snapshot.all_check_items: + append_ledger_record( + base_dir, + { + "attempt_id": f"final#{attempt_no}", + "checkpoint_id": "final", + "item_text": ci.text, + "kind": ci.kind, + "when": ci.when, + "status": "inconclusive", + "evidence": f"final check error: {e}", + "anchor_step_id": None, + }, + ) + report = None + + if report is not None: + for v in report.verdicts: + when = next( + ( + ci.when + for ci in snapshot.all_check_items + if ci.kind == v.kind and ci.text == v.item_text + ), + "at_end", + ) + append_ledger_record( + base_dir, + { + "attempt_id": f"final#{attempt_no}", + "checkpoint_id": "final", + "item_text": v.item_text, + "kind": v.kind, + "when": when, + "status": v.status, + "evidence": v.evidence, + "suggestion": v.suggestion, + "anchor_step_id": None, + }, + ) + + passed = verdicts_allow_release(report) and not report.unmet_subgoals + if not passed: + if user_stopped or halted or attempt_no >= max_attempts: + verify_blocked = True + for v in report.verdicts: + if v.kind == "verify" and v.status == "failed": + blocked_findings.append( + f"[verify failed] '{v.item_text}': {v.evidence}" + ) + for text in report.unmet_subgoals: + blocked_findings.append(f"[unmet subgoal] '{text}'") + logger.warning( + "Final check found unmet verify criteria but the retry" + " budget is exhausted (or a stop/halt is latched);" + " ending with a blocked outcome." + ) + else: + findings: list[str] = [] + for text in report.unmet_subgoals: + if revert_subgoal_status(ctx, subgoal_hash(text)): + findings.append( + f"[final check] Subgoal '{text}' was set back to" + " in-progress: its acceptance criteria are not met." + ) + for v in report.verdicts: + if v.kind == "verify" and v.status == "failed": + suggestion = f" Suggestion: {v.suggestion}" if v.suggestion else "" + findings.append( + f"[verify failed] '{v.item_text}': {v.evidence}{suggestion}" + ) + update["operator_feedback"] = findings or None + update["exit_settlement_route"] = "continue" + return update + + # END path: assemble the machine-readable run outcome. + records = read_ledger(base_dir) + outcome = compute_run_outcome(snapshot, records, verify_blocked=verify_blocked) + if snapshot.all_check_items or records or verify_blocked: + update["run_outcome"] = outcome.model_dump() + # BLOCKED/partial wrap-ups carry the last findings in the metadata file. + extra = {"last_findings": blocked_findings} if blocked_findings else None + write_run_outcome(base_dir, outcome, extra) + logger.info( + f"Run outcome: task_status={outcome.task_status}," + f" tests(passed={outcome.tests.passed}, failed={outcome.tests.failed}," + f" inconclusive={outcome.tests.inconclusive}," + f" unchecked={outcome.tests.unchecked})" + ) + return update + + +def exit_settlement_gate(state: State) -> Literal["continue", "end"]: + route = getattr(state, "exit_settlement_route", None) + return "continue" if route == "continue" else "end" + + def execution_check_edge( state: State, ) -> Literal["execute_decisions", "review_subgoals"]: @@ -303,43 +415,6 @@ def _get_active_subgoal_hashes(ctx: ArtemisContext) -> tuple[str, str | None]: return "default", None -def validate_milestones( - content_before: str, content_after: str, similarity_threshold: float = 0.85 -) -> bool: - """Validates top-level milestones to see if async validation is needed. - - Returns True if structural changes or major wording changes ( str: - cleaned = re.sub(r"\W+", " ", s.lower()).strip() - return re.sub(r"\s+", " ", cleaned) - - for b, a in zip(before_milestones, after_milestones): - if b == a: - continue - - norm_b = normalize(b) - norm_a = normalize(a) - - ratio = difflib.SequenceMatcher(None, norm_b, norm_a).ratio() - if ratio < similarity_threshold: - return True - - return False - - class _CyFunctionDetectorMeta(type): def __instancecheck__(self, instance): name = type(instance).__name__ @@ -361,64 +436,185 @@ class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): def check_plan_mutation_rejections( content_before: str, content_after: str, state: State | None = None ) -> str | None: - """Checks if a task plan modification violates system safety or loop integrity rules.""" - lines_after = content_after.splitlines() - top_level_after = [line for line in lines_after if line.startswith("- [")] + """Checks if a task plan modification violates the machine-channel safety rules.""" + before = parse_plan(content_before) + after = parse_plan(content_after) # 1. Incomplete nested subgoals check - if top_level_after and all(line.startswith("- [x]") for line in top_level_after): - has_incomplete = any(re.match(r"^\s*-\s*\[[ /]\]", line) for line in lines_after) - if has_incomplete: - return ( - "Your changes to the task plan were not applied. Your changes would cause the" - " task to end completely, but there seem to be goals in the task plan" - " that have not been marked as completed. If you believe these goals" - " are already completed, please update their statuses." - ) - - # 2. Continuous monitoring / Loop milestone protection check - lines_before = content_before.splitlines() - top_level_before = [line for line in lines_before if line.startswith("- [")] - - continuous_pattern = re.compile(r"\[Loop\].*?(?:continuous|until\s*manual)", re.IGNORECASE) - has_continuous_before = any(continuous_pattern.search(line) for line in top_level_before) - - injected = getattr(state, "injected_instruction", None) if state else None - user_stopped = bool( - injected - and any( - w in str(injected).lower() - for w in ["stop", "finish", "end", "停止", "结束", "退出", "quit"] + if after.all_top_level_done and any( + (item.is_pending or item.is_active) and not item.is_top_level for item in after.items + ): + return ( + "Your changes to the task plan were not applied. Your changes would cause the" + " task to end completely, but there seem to be goals in the task plan" + " that have not been marked as completed. If you believe these goals" + " are already completed, please update their statuses." ) - ) - if has_continuous_before and not user_stopped: - has_continuous_after = any(continuous_pattern.search(line) for line in top_level_after) - if not has_continuous_after: + # 2. Continuous monitoring protection: a [Loop:continuous] milestone can only + # transition to [x] (or disappear) after the user injects an explicit stop + # signal (user_stop_requested); intent is never inferred from wording. + user_stopped = bool(state is not None and getattr(state, "user_stop_requested", False)) + + if before.continuous_top_level and not user_stopped: + if not after.continuous_top_level: return ( - "Your changes to the task plan were rejected. You cannot delete an active [Loop] " - "continuous monitoring milestone or remove its [Loop] tag. Please keep the [Loop] " - "milestone active (in progress [/]) and record each check cycle as an indented " - "subtask." + "Your changes to the task plan were rejected. You cannot delete an active" + " [Loop:continuous] continuous monitoring milestone or remove its tag." + " Please keep the milestone active (in progress [/]) and record each" + " check cycle as an indented subtask." + ) + if any(item.is_done for item in after.continuous_top_level): + return ( + "Your changes to the task plan were rejected. [Loop:continuous] continuous" + " monitoring milestones must remain in progress ([/]) and cannot be" + " unilaterally marked as completed [x] until the user injects an explicit" + " stop signal. Please keep it active ([/]) and record each polling check" + " as an indented subtask." ) - for line in top_level_after: - if continuous_pattern.search(line) and line.startswith("- [x]"): - return ( - "Your changes to the task plan were rejected. Ongoing continuous monitoring " - "milestones (exit condition: 'until manually stopped') must remain in progress " - "([/]) and cannot be unilaterally marked as completed [x]. Please keep it " - "active ([/]) and record each polling check as an indented subtask." - ) return None +def check_unintended_rewrite(content_before: str, content_after: str) -> str | None: + """Detects the hand-slip signature of a full-file rewrite: top-level milestones + whose status did not change but whose text drifted. Declared edits via + update_note (explicit target/replacement) are exempt. + """ + edits = unintended_milestone_edits(parse_plan(content_before), parse_plan(content_after)) + if not edits: + return None + + listing = "\n".join(f'- "{b}"\n -> "{a}"' for b, a in edits) + return ( + "Your changes to the task plan were not applied. Your rewrite reworded" + " top-level milestones whose status you did not change — this usually means" + " the plan was regenerated from memory and historical milestones drifted" + f" unintentionally:\n{listing}\nIf a rewording was intentional, apply it" + " explicitly with the update_note tool (target/replacement). Otherwise," + " re-apply only the changes you actually intended." + ) + + class NoteArgs(BaseModel): model_config = {"ignored_types": (CyFunctionDetector,)} key: str = Field(..., description=SAVE_NOTE_ARG_KEY_DESC) content: str = Field(..., description=SAVE_NOTE_ARG_CONTENT_DESC) +async def _process_plan_write( + ctx: ArtemisContext, + state: State | None, + task_plan_path, + content_before: str, + content_after: str, + result, + declared_intent: bool, +): + """Shared post-write pipeline for every task_plan mutation. + + Constitution order: hard rejections with rollback (machine-channel rules + only), then ratchet-baseline planner validation, then checker trigger on + new top-level completions. ``declared_intent`` marks surgical update_note + edits, which are exempt from the full-rewrite hand-slip check. + """ + rejection_message = check_plan_mutation_rejections(content_before, content_after, state) + if rejection_message is None and not declared_intent: + rejection_message = check_unintended_rewrite(content_before, content_after) + if rejection_message: + logger.info(f"Rejected task plan update: {rejection_message}") + if task_plan_path.exists(): + task_plan_path.write_text(content_before, encoding="utf-8") + + # Guidance, not a tool failure: status stays "success" so the Operator + # loop treats it as feedback content (matching the legacy envelope + # behavior where the SystemMessage carried no error status). + return ToolMessage(content=rejection_message, tool_call_id="", status="success") + + # Deterministic check-line guard (Operator-source writes only; the Planner + # node writes through unwrapped tools and retains revision authority): + # deleted/rewritten check lines are merged back in a pure text operation. + # This is content-driven and independent of any switch — a resumed plan + # with check lines stays protected even when checking is disabled. + merged = restore_missing_check_items(content_before, content_after) + if merged is not None: + logger.info( + "Plan write removed or rewrote declared check lines; merging them" + " back (check standards are protected deterministically)." + ) + content_after = merged + try: + task_plan_path.write_text(content_after, encoding="utf-8") + except Exception as e: + logger.error(f"Failed to write merged check lines: {e}") + + before = parse_plan(content_before) + after = parse_plan(content_after) + + # Ratchet baseline: judge the current plan against the last *validated* + # plan rather than the previous write, so many small edits cannot drift + # the milestones below any per-edit trigger. + if ctx.last_validated_plan is None: + ctx.last_validated_plan = content_before + + disable_planner_validation = ( + ctx.execution_setup and ctx.execution_setup.disable_planner_validation + ) + if not disable_planner_validation: + baseline = parse_plan(ctx.last_validated_plan) + if milestones_changed(baseline, after): + logger.info( + "Top-level task plan diverged from the validated baseline." + " Triggering async Planner Validation." + ) + + if ctx.planner_task and not ctx.planner_task.done(): + # A newer write supersedes the in-flight validation. + ctx.planner_task.cancel() + + if not milestones_changed(baseline, before): + # Last content still consistent with the baseline: preserves + # status progress made since the baseline as rollback target. + ctx.task_plan_content_before = content_before + elif ctx.task_plan_content_before is None: + ctx.task_plan_content_before = ctx.last_validated_plan + + ctx.pending_validated_plan = content_after + initial_goal = ( + state.initial_goal if state and hasattr(state, "initial_goal") else "Unknown Goal" + ) + + ctx.planner_task = asyncio.create_task( + run_async_planner_validation( + ctx, + initial_goal, + ctx.last_validated_plan, + content_after, + getattr(state, "operator_raw_thinking", None), + getattr(state, "operator_native_thinking", None), + ) + ) + + if isinstance(result, ToolMessage) and isinstance(result.content, str): + result.content = ( + f"{result.content}\n\nWe have applied your changes" + " to the task plan. A background verification task" + " is currently reviewing these changes. If there" + " are any issues with your modifications, you will" + " be notified shortly." + ) + + # Queue (never spawn) a checkpoint for EVERY newly completed top-level + # subgoal carrying on_complete check items. Spawning happens in + # execution_check_node after this turn's step is recorded, so the evidence + # anchor points at the correct step. + new_completions = new_top_level_completions(before, after) + if new_completions: + queue_checkpoints(ctx, state, after, new_completions, content_after) + + return result + + def wrap_note_tool(ctx: ArtemisContext, original_tool): async def wrapped_note_tool( @@ -428,125 +624,36 @@ def wrap_note_tool(ctx: ArtemisContext, original_tool): state: Annotated[State, InjectedState] = None, ): """Wrapped tool to intercept task plan updates.""" - if key == "task_plan" and ctx.data_engine: - task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") - - content_before = "" - if task_plan_path.exists(): - content_before = task_plan_path.read_text(encoding="utf-8") - - result = await invoke_tool_with_injection( - original_tool, - {"key": key, "content": content}, - tool_call_id, - state, - record_trace=False, - ) - - content_after = content - if task_plan_path.exists(): - content_after = task_plan_path.read_text(encoding="utf-8") - - rejection_message = check_plan_mutation_rejections(content_before, content_after, state) - if rejection_message: - logger.info(f"Rejected task plan update: {rejection_message}") - if task_plan_path.exists(): - task_plan_path.write_text(content_before, encoding="utf-8") - - return Command( - update={VALIDATOR_MESSAGES_KEY: [SystemMessage(content=rejection_message)]} - ) - - threshold = ( - ctx.execution_setup.planner_validation_threshold - if ctx.execution_setup - and hasattr(ctx.execution_setup, "planner_validation_threshold") - else 0.85 - ) - needs_validation = validate_milestones( - content_before, content_after, similarity_threshold=threshold - ) - disable_planner_validation = ( - ctx.execution_setup and ctx.execution_setup.disable_planner_validation - ) - - if needs_validation and not disable_planner_validation: - logger.info( - "Detected significant top-level task plan modification." - " Triggering async Planner Validation." - ) - - initial_goal = ( - state.initial_goal - if state and hasattr(state, "initial_goal") - else "Unknown Goal" - ) - ctx.task_plan_content_before = content_before - - ctx.planner_task = asyncio.create_task( - run_async_planner_validation( - ctx, - initial_goal, - content_before, - content_after, - getattr(state, "operator_raw_thinking", None), - getattr(state, "operator_native_thinking", None), - ) - ) - - if isinstance(result, Command) and VALIDATOR_MESSAGES_KEY in result.update: - msgs = result.update[VALIDATOR_MESSAGES_KEY] - if msgs and hasattr(msgs[0], "content"): - msgs[0].content = ( - f"{msgs[0].content}\n\nWe have applied your changes" - " to the task plan. A background verification task" - " is currently reviewing these changes. If there" - " are any issues with your modifications, you will" - " be notified shortly." - ) - - # Only match top-level subgoals, intentionally ignoring indented sub-subgoals - disable_checker = ctx.execution_setup and ctx.execution_setup.disable_checker - pattern = re.compile(r"^-\s*\[x\]\s*(.*)$", re.MULTILINE) - - before_matches = set(pattern.findall(content_before)) - after_matches = set(pattern.findall(content_after)) - - new_completions = after_matches - before_matches - - if new_completions and not disable_checker: - logger.info(f"Detected new top-level completions: {new_completions}.") - - all_matches_in_order = pattern.findall(content_after) - latest_completion = None - for match in reversed(all_matches_in_order): - if match in new_completions: - latest_completion = match - break - - if latest_completion: - logger.info(f"Triggering Checker for latest completion: {latest_completion}") - raw_perception = getattr(state, "operator_raw_data", None) - ui_hierarchy = getattr(state, "latest_ui_hierarchy", None) - ctx.checker_task = asyncio.create_task( - run_async_check( - ctx, - latest_completion, - raw_perception_data=raw_perception, - latest_ui_hierarchy=ui_hierarchy, - ) - ) - - return result - else: + args = {"key": key, "content": content} + if key != "task_plan" or not ctx.data_engine: return await invoke_tool_with_injection( - original_tool, - {"key": key, "content": content}, - tool_call_id, - state, - record_trace=False, + original_tool, args, tool_call_id, state, record_trace=False ) + task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") + + content_before = "" + if task_plan_path.exists(): + content_before = task_plan_path.read_text(encoding="utf-8") + + result = await invoke_tool_with_injection( + original_tool, args, tool_call_id, state, record_trace=False + ) + + content_after = content + if task_plan_path.exists(): + content_after = task_plan_path.read_text(encoding="utf-8") + + return await _process_plan_write( + ctx, + state, + task_plan_path, + content_before, + content_after, + result, + declared_intent=False, + ) + return StructuredTool.from_function( func=lambda *a, **kw: None, coroutine=wrapped_note_tool, @@ -573,121 +680,36 @@ def wrap_update_note_tool(ctx: ArtemisContext, original_tool): state: Annotated[State, InjectedState] = None, ): """Wrapped tool to intercept task plan updates via update_note.""" - if key == "task_plan" and ctx.data_engine: - task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") - - content_before = "" - if task_plan_path.exists(): - content_before = task_plan_path.read_text(encoding="utf-8") - - result = await invoke_tool_with_injection( - original_tool, - {"key": key, "target": target, "replacement": replacement}, - tool_call_id, - state, - record_trace=False, - ) - - content_after = "" - if task_plan_path.exists(): - content_after = task_plan_path.read_text(encoding="utf-8") - - rejection_message = check_plan_mutation_rejections(content_before, content_after, state) - if rejection_message: - logger.info(f"Rejected task plan update via update_note: {rejection_message}") - if task_plan_path.exists(): - task_plan_path.write_text(content_before, encoding="utf-8") - - return Command( - update={VALIDATOR_MESSAGES_KEY: [SystemMessage(content=rejection_message)]} - ) - - threshold = ( - ctx.execution_setup.planner_validation_threshold - if ctx.execution_setup - and hasattr(ctx.execution_setup, "planner_validation_threshold") - else 0.85 - ) - needs_validation = validate_milestones( - content_before, content_after, similarity_threshold=threshold - ) - disable_planner_validation = ( - ctx.execution_setup and ctx.execution_setup.disable_planner_validation - ) - - if needs_validation and not disable_planner_validation: - logger.info( - "Detected significant top-level task plan modification." - " Triggering async Planner Validation." - ) - - initial_goal = ( - state.initial_goal - if state and hasattr(state, "initial_goal") - else "Unknown Goal" - ) - ctx.task_plan_content_before = content_before - - ctx.planner_task = asyncio.create_task( - run_async_planner_validation( - ctx, - initial_goal, - content_before, - content_after, - getattr(state, "operator_raw_thinking", None), - getattr(state, "operator_native_thinking", None), - ) - ) - - if isinstance(result, Command) and VALIDATOR_MESSAGES_KEY in result.update: - msgs = result.update[VALIDATOR_MESSAGES_KEY] - if msgs and hasattr(msgs[0], "content"): - msgs[0].content = ( - f"{msgs[0].content}\n\nWe have applied your changes" - " to the task plan. A background verification task" - " is currently reviewing these changes. If there" - " are any issues with your modifications, you will" - " be notified shortly." - ) - - disable_checker = ctx.execution_setup and ctx.execution_setup.disable_checker - pattern = re.compile(r"^-\s*\[x\]\s*(.*)$", re.MULTILINE) - before_matches = set(pattern.findall(content_before)) - after_matches = set(pattern.findall(content_after)) - new_completions = after_matches - before_matches - - if new_completions and not disable_checker: - logger.info(f"Detected new top-level completions: {new_completions}.") - all_matches_in_order = pattern.findall(content_after) - latest_completion = None - for match in reversed(all_matches_in_order): - if match in new_completions: - latest_completion = match - break - - if latest_completion: - logger.info(f"Triggering Checker for latest completion: {latest_completion}") - raw_perception = getattr(state, "operator_raw_data", None) - ui_hierarchy = getattr(state, "latest_ui_hierarchy", None) - ctx.checker_task = asyncio.create_task( - run_async_check( - ctx, - latest_completion, - raw_perception_data=raw_perception, - latest_ui_hierarchy=ui_hierarchy, - ) - ) - - return result - else: + args = {"key": key, "target": target, "replacement": replacement} + if key != "task_plan" or not ctx.data_engine: return await invoke_tool_with_injection( - original_tool, - {"key": key, "target": target, "replacement": replacement}, - tool_call_id, - state, - record_trace=False, + original_tool, args, tool_call_id, state, record_trace=False ) + task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") + + content_before = "" + if task_plan_path.exists(): + content_before = task_plan_path.read_text(encoding="utf-8") + + result = await invoke_tool_with_injection( + original_tool, args, tool_call_id, state, record_trace=False + ) + + content_after = "" + if task_plan_path.exists(): + content_after = task_plan_path.read_text(encoding="utf-8") + + return await _process_plan_write( + ctx, + state, + task_plan_path, + content_before, + content_after, + result, + declared_intent=True, + ) + return StructuredTool.from_function( func=lambda *a, **kw: None, coroutine=wrapped_update_note, @@ -697,64 +719,83 @@ def wrap_update_note_tool(ctx: ArtemisContext, original_tool): ) +def convergence_gate( + state: State, ctx: ArtemisContext +) -> Literal["continue", "exit_settlement", "end"]: + logger.info("Starting convergence_gate") + + if not ctx.data_engine: + return "end" + + def _read_plan() -> str: + file_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") + if not file_path.exists(): + return "" + try: + return file_path.read_text(encoding="utf-8") + except Exception: + return "" + + def _terminal_route() -> Literal["exit_settlement", "end"]: + """Settlement and final review are decoupled from each other but share + the entry: any check item, in-flight/queued attempt, ledger record, or + an enabled final review forces the settlement node.""" + plan_snapshot = parse_plan(_read_plan()) + needs_settlement = ( + bool(plan_snapshot.all_check_items) + or bool(getattr(ctx, "checkpoint_tasks", None)) + or bool(getattr(ctx, "pending_checkpoints", None)) + or has_ledger_records(ctx.data_engine.base_dir) + or final_check_enabled(ctx) + ) + return "exit_settlement" if needs_settlement else "end" + + # A halt latched by an assert failure (policy 'halt') terminates the run + # regardless of plan progress. + if getattr(ctx, "assert_halt", False): + logger.warning("Assert-halt latched; routing to exit settlement.") + return _terminal_route() + + # Check planner-validation result + if hasattr(state, "checker_success") and not state.checker_success: + logger.info("Plan validation failed, returning to operator for retry.") + return "continue" + + file_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") + if not file_path.exists(): + logger.warning(f"Task plan file not found at {file_path}") + return "continue" + try: + task_plan_content = file_path.read_text(encoding="utf-8") + except Exception as e: + logger.error(f"Failed to read task plan: {e}") + return "continue" + + snapshot = parse_plan(task_plan_content) + + if not snapshot.has_top_level: + # An empty plan does not bypass an enabled final review: the review + # object is the user's original goal, not the plan. + logger.info("No subgoals found in file, ending") + return _terminal_route() + + if snapshot.all_top_level_done: + user_stopped = bool(getattr(state, "user_stop_requested", False)) if state else False + if snapshot.continuous_top_level and not user_stopped: + logger.warning( + "All subgoals marked [x] but task plan has a [Loop:continuous]" + " milestone without a user stop signal. Continuing." + ) + return "continue" + logger.info("All subgoals are completed, ending the goal") + return _terminal_route() + + return "continue" + + async def get_graph(ctx: ArtemisContext) -> CompiledStateGraph: graph_builder = StateGraph(State) - def convergence_gate(state: State) -> Literal["continue", "end"]: - logger.info("Starting convergence_gate") - - # Check checker result - if hasattr(state, "checker_success") and not state.checker_success: - logger.info("Checker failed, returning to operator for retry.") - return "continue" - - task_plan_content = "" - if ctx.data_engine: - file_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan") - if file_path.exists(): - try: - task_plan_content = file_path.read_text(encoding="utf-8") - except Exception as e: - logger.error(f"Failed to read task plan: {e}") - return "continue" - else: - logger.warning(f"Task plan file not found at {file_path}") - return "continue" - - lines = task_plan_content.splitlines() - - # Check for continuous monitoring loop that should not terminate prematurely - continuous_pattern = re.compile(r"\[Loop\].*?(?:continuous|until\s*manual)", re.IGNORECASE) - has_continuous_monitoring = any(continuous_pattern.search(line) for line in lines) - injected = getattr(state, "injected_instruction", None) if state else None - user_stopped = bool( - injected - and any( - w in str(injected).lower() - for w in ["stop", "finish", "end", "停止", "结束", "退出", "quit"] - ) - ) - - # Check for all completed - top_level_lines = [line for line in lines if line.startswith("- [")] - - if not top_level_lines: - logger.info("No subgoals found in file, ending") - return "end" - - all_done = all(line.startswith("- [x]") for line in top_level_lines) - if all_done: - if has_continuous_monitoring and not user_stopped: - logger.warning( - "All subgoals marked [x] but task plan has active continuous monitoring" - " without user stop signal. Continuing." - ) - return "continue" - logger.info("All subgoals are completed, ending the goal") - return "end" - - return "continue" - # Get native tools native_wrappers = [ save_note_wrapper, @@ -825,6 +866,7 @@ async def get_graph(ctx: ArtemisContext) -> CompiledStateGraph: graph_builder.add_node("execution_check", functools.partial(execution_check_node, ctx=ctx)) graph_builder.add_node("perception", functools.partial(perception_node, ctx=ctx)) graph_builder.add_node(node="convergence", action=convergence_node, defer=True) + graph_builder.add_node("exit_settlement", functools.partial(exit_settlement_node, ctx=ctx)) ## Linking nodes graph_builder.add_edge(START, "planner") @@ -843,7 +885,16 @@ async def get_graph(ctx: ArtemisContext) -> CompiledStateGraph: graph_builder.add_conditional_edges( source="convergence", - path=convergence_gate, + path=functools.partial(convergence_gate, ctx=ctx), + path_map={ + "continue": "perception", + "exit_settlement": "exit_settlement", + "end": END, + }, + ) + graph_builder.add_conditional_edges( + source="exit_settlement", + path=exit_settlement_gate, path_map={ "continue": "perception", "end": END, diff --git a/artemis/graph/perception.py b/artemis/graph/perception.py index 32a2edd..67dae2f 100644 --- a/artemis/graph/perception.py +++ b/artemis/graph/perception.py @@ -22,6 +22,7 @@ from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController from artemis.data_engine.trace import trace from artemis.graph.state import State +from artemis.graph.visibility import strict_state from artemis.utils.logger import get_logger from artemis.utils.ocr_api import is_ocr_configured, perform_ocr from artemis.utils.ocr_xml_fusion import ( @@ -71,15 +72,23 @@ def _should_skip_settling(state: State) -> bool: @trace(type="agent", name="perception") -def _check_injected_instruction_file(base_dir: str) -> str | None: +def _check_injected_instruction_file(base_dir: str) -> dict | None: + """Consumes a pending injected-instruction payload, if any. + + Returns the payload dict: `instruction` (str | None) plus the explicit + `release_loop` stop signal (bool) declared by the injecting caller. + """ instruction_path = Path(base_dir) / "injected_instruction.json" if instruction_path.exists(): with open(instruction_path, encoding="utf-8") as f: data = json.load(f) - injected_instruction = data.get("instruction") instruction_path.unlink() - logger.info(f"Read and deleted injected instruction: '{injected_instruction}'") - return injected_instruction + payload = { + "instruction": data.get("instruction"), + "release_loop": bool(data.get("release_loop")), + } + logger.info(f"Read and deleted injected instruction payload: {payload}") + return payload return None @@ -88,6 +97,7 @@ async def perception_node(state: State, ctx: ArtemisContext) -> dict: Always executes at the start of an operator cycle. """ + state = strict_state(state, "perception") logger.info("Entering Perception Node...") if not ctx.data_engine.current_step_id: @@ -96,10 +106,14 @@ async def perception_node(state: State, ctx: ArtemisContext) -> dict: # Check for injected instruction without blocking event loop injected_instruction = None + user_stop_requested = False if ctx.data_engine and ctx.data_engine.base_dir: - injected_instruction = await asyncio.to_thread( + injected_payload = await asyncio.to_thread( _check_injected_instruction_file, str(ctx.data_engine.base_dir) ) + if injected_payload: + injected_instruction = injected_payload.get("instruction") + user_stop_requested = bool(injected_payload.get("release_loop")) controller = UnifiedMobileController(ctx) @@ -171,16 +185,6 @@ async def perception_node(state: State, ctx: ArtemisContext) -> dict: ) logger.info(f"Offloaded perception data saving to background with image_name: {image_name}") - # Cache latest perception data on ArtemisContext for downstream zero-latency reuse (e.g. Checker) - ctx.latest_perception_data = { - "screenshot_b64": latest_screenshot_b64, - "latest_ui_hierarchy": fused_xml, - "xml_hierarchy": xml_hierarchy if xml_hierarchy is not None else [], - "ocr_results": ocr_results, - "width": device_data.width, - "height": device_data.height, - } - # Return state update update = { "latest_screenshot": screenshot_path, @@ -193,5 +197,7 @@ async def perception_node(state: State, ctx: ArtemisContext) -> dict: "height": device_data.height, }, "injected_instruction": injected_instruction, + # sticky_or reducer: once latched True, later False updates keep it True + "user_stop_requested": user_stop_requested, } return update diff --git a/artemis/graph/state.py b/artemis/graph/state.py index c65bcf4..24445b6 100644 --- a/artemis/graph/state.py +++ b/artemis/graph/state.py @@ -12,14 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Annotated, Any -from langchain_core.messages import AnyMessage -from langgraph.graph import add_messages -from artemis.config import AgentNode -from artemis.context import ArtemisContext -from artemis.utils.logger import get_logger +from typing import Annotated + from pydantic import BaseModel, ConfigDict +from artemis.utils.logger import get_logger + logger = get_logger(__name__) @@ -28,38 +26,91 @@ def take_last(a, b): return b +def sticky_or(a, b): + """Reducer that latches True: once set, the flag survives later updates.""" + return bool(a) or bool(b) + + class State(BaseModel): + """Graph channel state. + + Every field is explicit (``extra="forbid"``): an undeclared write or an + unknown constructor key fails loudly instead of becoming a silent ghost + key. Undeclared *reads* (``getattr(state, k, default)``) are guarded by the + node visibility manifest in ``artemis.graph.visibility`` instead. + """ + model_config = ConfigDict( arbitrary_types_allowed=True, - extra="allow", + extra="forbid", ) - messages: Annotated[list[AnyMessage], "Sequential messages", add_messages] - - remaining_steps: Annotated[int | None, "Remaining steps before the task is completed"] = None - - # planner related keys + # ── Control plane (cross-turn signals & routing) ───────────────────── initial_goal: Annotated[str, "Initial goal given by the user"] injected_instruction: Annotated[ str | None, "Injected instruction from user for micro-adjustment", take_last, ] = None + user_stop_requested: Annotated[ + bool, + "Latched True once the user externally signals stop (release_loop)," + " unlocking completion of [Loop:continuous] milestones", + sticky_or, + ] = False + checker_success: Annotated[ + bool | None, + "True if checker succeeded or not triggered, False if failed", + take_last, + ] = None + operator_feedback: Annotated[ + list[str] | None, + "Source-tagged findings ([checker]/[planner]/[final check]) injected" + " append-only into the Operator's next prompt (never switches the" + " prompt template)", + take_last, + ] = None + run_outcome: Annotated[ + dict | None, + "Machine-readable run outcome (task_status + test summary), populated" + " by exit settlement before END", + take_last, + ] = None + exit_settlement_route: Annotated[ + str | None, + "Routing decision produced by exit_settlement_node ('continue' | 'end')", + take_last, + ] = None - # operator related keys (perception) + # ── Perception (moves behind PerceptionStore in Phase 2) ───────────── latest_ui_hierarchy: Annotated[ list[dict] | None, "Latest UI hierarchy of the device", take_last - ] - latest_screenshot: Annotated[str | None, "Path to the latest screenshot", take_last] - focused_app_info: Annotated[str | None, "Focused app info", take_last] - device_date: Annotated[str | None, "Date of the device", take_last] + ] = None + latest_screenshot: Annotated[str | None, "Path to the latest screenshot", take_last] = None + operator_raw_data: Annotated[ + dict | None, + "Raw perception data from operator (screenshot, xml, ocr)", + take_last, + ] = None + indexed_points: Annotated[ + list[list[int]] | None, + "Active coordinate mappings matching indexed text elements", + take_last, + ] = None + indexed_elements: Annotated[ + list[dict] | None, + "Active interactable elements matching indexed text elements," + " containing text, bounds, class, and center coordinates.", + take_last, + ] = None - # operator related keys (decisions) + # ── Turn products (previous turn's decisions & results) ────────────── + current_step_id: Annotated[str | None, "Current step ID from data engine", take_last] = None structured_decisions: Annotated[ str | None, "Structured decisions made by the operator, for the validator to follow", take_last, - ] + ] = None operator_raw_thinking: Annotated[ str | None, "Raw thinking process of the operator", @@ -75,60 +126,23 @@ class State(BaseModel): "Short term memory / scratchpad for the operator", take_last, ] = None - complete_subgoals_by_ids: Annotated[ - list[str], - "List of subgoal IDs to complete", - take_last, - ] - current_agent: Annotated[str | None, "Current active agent", take_last] = None - operator_replan_reason: Annotated[str | None, "Reason for operator replan", take_last] = None - subgoal_plan: Annotated[list[Any] | None, "Subgoal plan", take_last] = None - operator_tactical_plan: Annotated[list[Any] | None, "Operator tactical plan", take_last] = None - - # validator related keys - validator_messages: Annotated[list[AnyMessage], "Sequential Validator messages", add_messages] last_execution_result: Annotated[ dict | None, "Last execution result from validator", take_last ] = None - current_step_id: Annotated[str | None, "Current step ID from data engine", take_last] = None - operator_raw_data: Annotated[ - dict | None, - "Raw perception data from operator (screenshot, xml, ocr)", - take_last, - ] = None - checker_success: Annotated[ - bool | None, - "True if checker succeeded or not triggered, False if failed", - take_last, - ] = None subagent_calls: Annotated[ list[str], "List of sub-agent calls in current session", take_last ] = [] - operator_replied: Annotated[bool | None, "True if operator replied to checker", take_last] = ( - None - ) operator_tool_limit_exceeded: Annotated[ bool | None, "True if operator exceeded tool call limit in the previous turn", take_last, ] = None - indexed_points: Annotated[ - list[list[int]] | None, - "Active coordinate mappings matching indexed text elements", - take_last, - ] = None - indexed_elements: Annotated[ - list[dict] | None, - "Active interactable elements matching indexed text elements," - " containing text, bounds, class, and center coordinates.", - take_last, - ] = None - async def asanitize_update( - self, - ctx: ArtemisContext, - update: dict, - agent: AgentNode | None = None, - ): - """Sanitizes the state update to ensure it is valid and apply side effect logic where required.""" - return update + @classmethod + def initial(cls, goal: str) -> "State": + """Single source for the graph's initial state. + + Every entrypoint (SDK, engine runners, tests) must construct the + first State through here so required fields stay in one place. + """ + return cls(initial_goal=goal) diff --git a/artemis/graph/visibility.py b/artemis/graph/visibility.py new file mode 100644 index 0000000..69b2edb --- /dev/null +++ b/artemis/graph/visibility.py @@ -0,0 +1,227 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Per-node State visibility manifests with test-time enforcement. + +``extra="forbid"`` on :class:`~artemis.graph.state.State` catches ghost +*writes*; ghost *reads* (``getattr(state, key, default)`` silently returning +the default) can only be caught by policing reads against a declared manifest. + +The manifest below is the declared visible domain of every graph node — the +authoritative answer to "which State fields does this agent see / produce". +Enforcement is zero-cost in production: :func:`strict_state` returns the state +unchanged unless ``ARTEMIS_STRICT_STATE=1`` (set in tests), in which case field +reads outside the node's manifest raise :class:`VisibilityError`. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any + +from artemis.graph.state import State + + +class VisibilityError(Exception): + """A node touched a State field outside its declared visible domain.""" + + +@dataclass(frozen=True) +class NodeVisibility: + reads: frozenset[str] + writes: frozenset[str] + + @property + def visible(self) -> frozenset[str]: + return self.reads | self.writes + + +def _vis(reads: set[str], writes: set[str]) -> NodeVisibility: + return NodeVisibility(reads=frozenset(reads), writes=frozenset(writes)) + + +#: Declared visible domain per node. Reads include fields touched by the tools +#: a node invokes with ``InjectedState`` (they run inside the node's turn). +#: Start generous, tighten over time — an entry here is a documented decision. +NODE_VISIBILITY: dict[str, NodeVisibility] = { + "perception": _vis( + reads={"structured_decisions"}, + writes={ + "latest_screenshot", + "latest_ui_hierarchy", + "operator_raw_data", + "injected_instruction", + "user_stop_requested", + }, + ), + "planner": _vis( + reads={"initial_goal", "latest_screenshot"}, + # Mutates latest_screenshot in place when it captures its own frame + # (planner runs before perception on the first turn). + writes={"latest_screenshot"}, + ), + "operator": _vis( + reads={ + "initial_goal", + "injected_instruction", + "user_stop_requested", + "operator_feedback", + "operator_raw_data", + "latest_ui_hierarchy", + "latest_screenshot", + "indexed_points", + "indexed_elements", + "short_term_memory", + "subagent_calls", + "current_step_id", + "operator_tool_limit_exceeded", + "structured_decisions", + "operator_raw_thinking", + "operator_native_thinking", + }, + writes={ + "structured_decisions", + "operator_raw_thinking", + "operator_native_thinking", + "short_term_memory", + "indexed_points", + "indexed_elements", + "current_step_id", + "subagent_calls", + "operator_tool_limit_exceeded", + }, + ), + "execution_check": _vis( + reads={ + "operator_raw_data", + "structured_decisions", + "operator_raw_thinking", + "operator_native_thinking", + "user_stop_requested", + "initial_goal", + }, + writes={ + "checker_success", + "structured_decisions", + "current_step_id", + "operator_feedback", + "subagent_calls", + }, + ), + "validator": _vis( + reads={ + "initial_goal", + "structured_decisions", + "latest_screenshot", + "latest_ui_hierarchy", + "current_step_id", + "operator_raw_data", + "operator_raw_thinking", + "operator_native_thinking", + "indexed_points", + "indexed_elements", + }, + writes={ + "last_execution_result", + # In-place turn-scoped mutations (formalized as TurnWorkspace in + # Phase 2): post-action screenshot and re-derived index mappings. + "latest_screenshot", + "indexed_points", + "indexed_elements", + }, + ), + "summarizer": _vis( + reads={ + "current_step_id", + "structured_decisions", + "operator_raw_thinking", + "operator_native_thinking", + "last_execution_result", + }, + writes=set(), + ), + "convergence": _vis( + reads={"checker_success", "user_stop_requested"}, + writes=set(), + ), + "exit_settlement": _vis( + reads={"user_stop_requested", "initial_goal"}, + writes={"exit_settlement_route", "operator_feedback", "run_outcome"}, + ), +} + +_STATE_FIELDS = frozenset(State.model_fields.keys()) + + +def strict_enabled() -> bool: + return os.environ.get("ARTEMIS_STRICT_STATE", "") == "1" + + +class StateView: + """Read-policing proxy handed to nodes under ``ARTEMIS_STRICT_STATE=1``. + + Only declared State *fields* are policed; methods and pydantic internals + pass through. Attribute writes delegate to the underlying State (in-place + turn mutations are Phase 2 scope) but are still checked against the + node's write set. + """ + + __slots__ = ("_state", "_node", "_visibility") + + def __init__(self, state: State, node: str): + object.__setattr__(self, "_state", state) + object.__setattr__(self, "_node", node) + object.__setattr__(self, "_visibility", NODE_VISIBILITY[node]) + + def __getattr__(self, name: str) -> Any: + if name in _STATE_FIELDS and name not in self._visibility.visible: + raise VisibilityError( + f"Node '{self._node}' read undeclared State field '{name}'." + " Declare it in NODE_VISIBILITY or stop reading it." + ) + return getattr(self._state, name) + + def __setattr__(self, name: str, value: Any) -> None: + if name in _STATE_FIELDS and name not in self._visibility.writes: + raise VisibilityError( + f"Node '{self._node}' mutated undeclared State field '{name}'." + " Declare it in NODE_VISIBILITY or stop writing it." + ) + setattr(self._state, name, value) + + +def strict_state(state: State, node: str) -> State | StateView: + """Wraps ``state`` in a policing view when strict mode is on (tests).""" + if strict_enabled() and isinstance(state, State): + return StateView(state, node) + return state + + +def check_update(node: str, update: dict | None) -> None: + """Validates a node's returned update keys against its write manifest.""" + if not update: + return + illegal = set(update) & _STATE_FIELDS - NODE_VISIBILITY[node].writes + if illegal: + raise VisibilityError(f"Node '{node}' returned undeclared update keys: {sorted(illegal)}.") + + +def validate_manifest() -> list[str]: + """Returns problems in the manifest itself (ghost fields). Empty = clean.""" + problems = [] + for node, vis in NODE_VISIBILITY.items(): + for field in vis.visible - _STATE_FIELDS: + problems.append(f"{node}: '{field}' is not a declared State field") + return problems diff --git a/artemis/llm/reliability.py b/artemis/llm/reliability.py new file mode 100644 index 0000000..beef7cf --- /dev/null +++ b/artemis/llm/reliability.py @@ -0,0 +1,252 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Failure classification, retry policies, and circuit breaking for LLM calls. + +This is the single source of truth for how ARTEMIS reacts to a failed model +call. Every provider/transport exception is converted into a ``Failure`` +carrying explicit recovery decisions, and every retry loop in the codebase is +expected to consult ``retry_policy_for`` instead of inventing its own rules. + +The taxonomy was promoted from ``artemis.agents.video_analyzer.reliability`` +(which now extends it with media-specific categories) so that the same +classification and circuit-breaker quality applies to every agent call, not +just the video pipeline. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +import random +import threading +import time +from typing import Any + + +class FailureCategory(StrEnum): + RATE_LIMIT = "rate_limit" + PROVIDER_UNAVAILABLE = "provider_unavailable" + TIMEOUT = "timeout" + CONNECTION = "connection" + AUTHENTICATION = "authentication" + BAD_REQUEST = "bad_request" + CANCELLED = "cancelled" + UNKNOWN = "unknown" + + +#: Categories that count toward circuit-breaker state: transient provider-side +#: trouble where hammering the endpoint makes things worse. +TRANSIENT_CATEGORIES: frozenset[str] = frozenset( + { + FailureCategory.RATE_LIMIT.value, + FailureCategory.PROVIDER_UNAVAILABLE.value, + FailureCategory.TIMEOUT.value, + FailureCategory.CONNECTION.value, + } +) + + +@dataclass(frozen=True) +class Failure: + category: FailureCategory + retryable: bool + should_fallback: bool + + +def extract_status_code(error: BaseException) -> int | None: + """Best-effort extraction of an HTTP-ish status code from an exception.""" + code = getattr(error, "code", None) or getattr(error, "status_code", None) + try: + return int(code) if code is not None else None + except (TypeError, ValueError): + return None + + +_RATE_LIMIT_MARKERS = ("rate limit", "quota", "resource_exhausted") +_UNAVAILABLE_MARKERS = ("unavailable", "overloaded", "high demand", "service unavailable") +_AUTH_MARKERS = ("unauthorized", "forbidden", "api key", "credential") + + +def classify_failure(error: BaseException) -> Failure: + """Convert a provider/transport exception into explicit recovery decisions.""" + if isinstance(error, KeyboardInterrupt) or type(error).__name__ == "CancelledError": + return Failure(FailureCategory.CANCELLED, False, False) + + code = extract_status_code(error) + message = str(error).lower() + + if code == 429 or any(marker in message for marker in _RATE_LIMIT_MARKERS): + return Failure(FailureCategory.RATE_LIMIT, True, True) + if code in {500, 502, 503, 504} or any(marker in message for marker in _UNAVAILABLE_MARKERS): + return Failure(FailureCategory.PROVIDER_UNAVAILABLE, True, True) + if isinstance(error, TimeoutError) or "timed out" in message or "timeout" in message: + return Failure(FailureCategory.TIMEOUT, True, True) + if isinstance(error, (ConnectionError, OSError)): + return Failure(FailureCategory.CONNECTION, True, True) + if code in {401, 403} or any(marker in message for marker in _AUTH_MARKERS): + # Not retryable against the same endpoint, but a fallback endpoint may + # use a different (working) credential. + return Failure(FailureCategory.AUTHENTICATION, False, True) + if code in {400, 404, 413, 415, 422}: + # Programmer/request errors: retrying is useless and falling back only + # hides the bug inside a different model's output. + return Failure(FailureCategory.BAD_REQUEST, False, False) + return Failure(FailureCategory.UNKNOWN, True, True) + + +# --------------------------------------------------------------------------- +# Typed terminal errors +# --------------------------------------------------------------------------- + + +class LLMCallError(Exception): + """Base class for terminal LLM call failures raised by the gateway.""" + + def __init__(self, message: str, *, failure: Failure, cause: BaseException | None = None): + super().__init__(message) + self.failure = failure + if cause is not None: + self.__cause__ = cause + + +class LLMPermanentError(LLMCallError): + """A non-retryable failure (auth, bad request). Retrying cannot help.""" + + +class LLMExhaustedError(LLMCallError): + """Retryable failures exhausted every attempt and the pause deadline.""" + + +# --------------------------------------------------------------------------- +# Retry policies +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class RetryPolicy: + """How many attempts a failure category deserves and how to space them.""" + + max_attempts: int + base_delay: float + max_delay: float + + def delay_for(self, attempt: int) -> float: + """Backoff delay before retry number ``attempt`` (1-based).""" + delay = min(self.max_delay, self.base_delay * (2 ** max(0, attempt - 1))) + return delay + random.uniform(0.1, 0.5) + + +_DEFAULT_RETRY_POLICIES: dict[FailureCategory, RetryPolicy] = { + FailureCategory.RATE_LIMIT: RetryPolicy(max_attempts=4, base_delay=10.0, max_delay=60.0), + FailureCategory.PROVIDER_UNAVAILABLE: RetryPolicy( + max_attempts=4, base_delay=5.0, max_delay=30.0 + ), + FailureCategory.TIMEOUT: RetryPolicy(max_attempts=3, base_delay=2.0, max_delay=15.0), + FailureCategory.CONNECTION: RetryPolicy(max_attempts=3, base_delay=2.0, max_delay=15.0), + FailureCategory.UNKNOWN: RetryPolicy(max_attempts=2, base_delay=1.0, max_delay=5.0), + # Non-retryable categories get a single attempt by definition. + FailureCategory.AUTHENTICATION: RetryPolicy(max_attempts=1, base_delay=0.0, max_delay=0.0), + FailureCategory.BAD_REQUEST: RetryPolicy(max_attempts=1, base_delay=0.0, max_delay=0.0), + FailureCategory.CANCELLED: RetryPolicy(max_attempts=1, base_delay=0.0, max_delay=0.0), +} + + +def retry_policy_for(category: FailureCategory) -> RetryPolicy: + return _DEFAULT_RETRY_POLICIES.get(category, _DEFAULT_RETRY_POLICIES[FailureCategory.UNKNOWN]) + + +# --------------------------------------------------------------------------- +# Circuit breaker +# --------------------------------------------------------------------------- + + +@dataclass +class _CircuitState: + consecutive_failures: int = 0 + open_until: float = 0.0 + half_open_in_flight: bool = False + + +class CircuitBreaker: + """Small circuit breaker keyed by provider/model identity. + + Only transient categories (see ``TRANSIENT_CATEGORIES``) trip the breaker; + programmer errors and auth failures never open it. ``failure`` may be any + object with a ``category`` attribute whose value matches this module's + category strings, so domain extensions (e.g. the video pipeline's failure + type) plug in without conversion. + """ + + def __init__(self, threshold: int = 3, cooldown_seconds: float = 60.0) -> None: + self.threshold = max(1, int(threshold)) + self.cooldown_seconds = max(0.1, float(cooldown_seconds)) + self._states: dict[str, _CircuitState] = {} + self._lock = threading.RLock() + + def allow(self, key: str, *, now: float | None = None) -> bool: + current = time.time() if now is None else now + with self._lock: + state = self._states.get(key) + if state is None: + return True + if state.open_until: + if current < state.open_until or state.half_open_in_flight: + return False + # Half-open: permit exactly one trial. Other concurrent calls + # continue to bypass the primary until it succeeds or fails. + state.half_open_in_flight = True + state.consecutive_failures = max(0, self.threshold - 1) + return True + return not state.half_open_in_flight + + def open_remaining(self, key: str, *, now: float | None = None) -> float: + """Seconds until the circuit for ``key`` half-opens (0 if not open).""" + current = time.time() if now is None else now + with self._lock: + state = self._states.get(key) + if state is None or not state.open_until: + return 0.0 + return max(0.0, state.open_until - current) + + def record_success(self, key: str) -> None: + with self._lock: + self._states.pop(key, None) + + def record_failure(self, key: str, failure: Any, *, now: float | None = None) -> None: + category = str(getattr(failure, "category", "")) + if category not in TRANSIENT_CATEGORIES: + with self._lock: + state = self._states.get(key) + if state is not None and state.half_open_in_flight: + self._states.pop(key, None) + return + current = time.time() if now is None else now + with self._lock: + state = self._states.setdefault(key, _CircuitState()) + state.half_open_in_flight = False + state.consecutive_failures += 1 + if state.consecutive_failures >= self.threshold: + state.open_until = current + self.cooldown_seconds + + def snapshot(self) -> dict[str, dict[str, Any]]: + with self._lock: + return { + key: { + "consecutive_failures": state.consecutive_failures, + "open_until": state.open_until, + "half_open_in_flight": state.half_open_in_flight, + } + for key, state in self._states.items() + } diff --git a/artemis/llm/structured.py b/artemis/llm/structured.py new file mode 100644 index 0000000..5f1aeab --- /dev/null +++ b/artemis/llm/structured.py @@ -0,0 +1,194 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unified structured-output parsing for LLM responses. + +Single pipeline replacing the ad-hoc JSON extraction scattered across agents: +strip thinking tags → extract the JSON payload (fenced or balanced-span) → +parse → tolerant repair pass → optional schema validation. A failed parse +returns a typed ``ParseFailure`` carrying the raw text and the reason — never +a raw string masquerading as parsed data. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +import re +from typing import Any + +from pydantic import BaseModel, ValidationError + +from artemis.llm.parser import extract_thinking_tags + + +@dataclass(frozen=True) +class ParseFailure: + """A parse miss: keeps the raw text and the reason, explicitly typed.""" + + raw: str + error: str + + +class StructuredOutputError(ValueError): + """Raised when a required structured response could not be parsed.""" + + def __init__(self, failure: ParseFailure): + super().__init__(failure.error) + self.failure = failure + + +def content_to_text(content: Any) -> str: + """Flatten a LangChain message content (str or block list) to plain text.""" + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for item in content: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict) and item.get("type") != "thinking": + text = item.get("text") + if isinstance(text, str): + parts.append(text) + return "".join(parts) + return "" if content is None else str(content) + + +_FENCE_RE = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE) + + +def _balanced_span(text: str) -> str | None: + """Return the first balanced {...} or [...] span, string/escape aware.""" + start = None + for i, ch in enumerate(text): + if ch in "{[": + start = i + break + if start is None: + return None + stack: list[str] = [] + in_string = False + escaped = False + for j in range(start, len(text)): + ch = text[j] + if in_string: + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == '"': + in_string = False + continue + if ch == '"': + in_string = True + elif ch in "{[": + stack.append(ch) + elif ch in "}]": + if not stack or (stack[-1], ch) not in {("{", "}"), ("[", "]")}: + return None + stack.pop() + if not stack: + return text[start : j + 1] + return None + + +def extract_json_candidate(text: str) -> str | None: + """Locate the JSON payload inside prose: fenced blocks first, then bare spans.""" + text = text.strip() + for match in _FENCE_RE.finditer(text): + span = _balanced_span(match.group(1).strip()) + if span: + return span + return _balanced_span(text) + + +def repair_json(text: str) -> str: + """Remove comment and trailing-comma noise outside of strings.""" + out: list[str] = [] + i = 0 + n = len(text) + in_string = False + escaped = False + while i < n: + ch = text[i] + if in_string: + out.append(ch) + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == '"': + in_string = False + i += 1 + continue + if ch == '"': + in_string = True + out.append(ch) + i += 1 + continue + if ch == "/" and i + 1 < n and text[i + 1] == "/": + while i < n and text[i] != "\n": + i += 1 + continue + if ch == "/" and i + 1 < n and text[i + 1] == "*": + i += 2 + while i + 1 < n and not (text[i] == "*" and text[i + 1] == "/"): + i += 1 + i += 2 + continue + if ch == ",": + k = i + 1 + while k < n and text[k] in " \t\r\n": + k += 1 + if k < n and text[k] in "]}": + i += 1 + continue + out.append(ch) + i += 1 + return "".join(out) + + +def parse_structured(text: Any, schema: type[BaseModel] | None = None) -> Any | ParseFailure: + """Parse an LLM response into JSON data (optionally schema-validated). + + Returns the parsed object (or a validated ``schema`` instance), or a + ``ParseFailure`` describing why the text could not be parsed. Callers must + check ``isinstance(result, ParseFailure)`` — the raw text is never + silently passed through as a result. + """ + if not isinstance(text, str): + text = content_to_text(text) + if not text or not text.strip(): + return ParseFailure(raw=text or "", error="empty response") + + cleaned, _ = extract_thinking_tags(text) + candidate = extract_json_candidate(cleaned) + if candidate is None: + return ParseFailure(raw=text, error="no JSON object or array found in response") + + last_error = "" + for variant in (candidate, repair_json(candidate)): + try: + data = json.loads(variant) + except json.JSONDecodeError as decode_error: + last_error = str(decode_error) + continue + if schema is not None: + try: + return schema.model_validate(data) + except ValidationError as validation_error: + return ParseFailure(raw=text, error=f"schema validation failed: {validation_error}") + return data + return ParseFailure(raw=text, error=f"invalid JSON: {last_error}") diff --git a/artemis/mcp/action_executor.py b/artemis/mcp/action_executor.py index 34e5f11..93c55cf 100644 --- a/artemis/mcp/action_executor.py +++ b/artemis/mcp/action_executor.py @@ -42,6 +42,7 @@ from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController from artemis.data_engine.trace import TraceSpan from artemis.mcp.action_manifest import OPTIONAL_ACTIONS, REQUIRED_ACTIONS +from artemis.mcp.action_specs import exception_prefix from artemis.mcp.action_session import ActionSession, get_action_session from artemis.mcp.action_types import ActionCode, ActionResult from artemis.mcp.actuators.adb import AdbActuator @@ -66,19 +67,6 @@ logger = get_logger(__name__) __all__ = ["McpActionExecutor"] -# Wording for wrapped exceptions; matches the historical executor `except` arms. -_EXCEPTION_PREFIX = { - "click": "Error during click", - "click_sequence": "Error executing click sequence", - "long_press": "Error during long press", - "input_text": "Error during input text", - "swipe": "Error during swipe", - "press_key": "Error during press_key", - "manage_app": "Error during manage_app", - "wait_for_delay": "Error during wait_for_delay", - "wait_for_text": "Error during wait_for_text", -} - class _ArgError(ValueError): """Argument-translation failure whose message is already fully formatted.""" @@ -94,19 +82,14 @@ class McpActionExecutor: actuator: AdbActuator | None = None, ): self.ctx = ctx - self.actuator = ( - actuator or getattr(ctx, "actuator", None) or AdbActuator(ctx, controller) - ) + self.actuator = actuator or getattr(ctx, "actuator", None) or AdbActuator(ctx, controller) self.controller = self.actuator.controller self._session: ActionSession | None = None @property def action_tool_names(self) -> frozenset[str]: """Device actions plus backend extension names -- the dynamic dispatch set.""" - return ( - (REQUIRED_ACTIONS | OPTIONAL_ACTIONS) - | {e.name for e in self.actuator.extensions()} - ) + return (REQUIRED_ACTIONS | OPTIONAL_ACTIONS) | {e.name for e in self.actuator.extensions()} async def _session_or_start(self) -> ActionSession: if self._session is None or not self._session.started: @@ -129,9 +112,7 @@ class McpActionExecutor: return await self._execute_agent_tool(raw_name, name, args, tool_call_id, state) if raw_name in self.action_tool_names: - return await self._execute_device_action( - raw_name, name, args, tool_call_id, state - ) + return await self._execute_device_action(raw_name, name, args, tool_call_id, state) return ToolExecutionResult( tool_call_id=tool_call_id, @@ -188,15 +169,12 @@ class McpActionExecutor: try: img_bytes = Path(shot_path).read_bytes() except Exception as read_err: - logger.warning( - f"Failed to read observed screenshot: {read_err}" - ) + logger.warning(f"Failed to read observed screenshot: {read_err}") except _ArgError as e: res = ActionResult.failure(raw_name, str(e), code=ActionCode.INVALID_ARGS) message = str(e) except Exception as e: - prefix = _EXCEPTION_PREFIX.get(raw_name, f"Error during {raw_name}") - message = f"{prefix}: {e}" + message = f"{exception_prefix(raw_name)}: {e}" res = ActionResult.failure(raw_name, message, detail=repr(e)) if shot_path: @@ -365,9 +343,7 @@ class McpActionExecutor: pass if not isinstance(sequence, (list, tuple)): - raise _ArgError( - f"Error during click sequence: Invalid sequence format: {sequence}" - ) + raise _ArgError(f"Error during click sequence: Invalid sequence format: {sequence}") width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080 height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400 @@ -389,15 +365,11 @@ class McpActionExecutor: elif isinstance(target, (list, tuple)) and len(target) == 2: nx, ny = int(target[0]), int(target[1]) else: - raise _ArgError( - f"Error during click sequence: Invalid target format: {raw_target}" - ) + raise _ArgError(f"Error during click sequence: Invalid target format: {raw_target}") resolved.append([nx, ny]) return resolved - def _translate_swipe( - self, args: dict[str, Any], state: Any - ) -> tuple[str, dict[str, Any], Any]: + def _translate_swipe(self, args: dict[str, Any], state: Any) -> tuple[str, dict[str, Any], Any]: width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080 height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400 default_duration = args.get("duration", 400) @@ -411,7 +383,7 @@ class McpActionExecutor: direction=target, target=args.get("target"), indexed_elements=getattr(state, "indexed_elements", None) if state else None, - ui_hierarchy=getattr(state, "ui_tree", None) if state else None, + ui_hierarchy=getattr(state, "latest_ui_hierarchy", None) if state else None, width=width, height=height, duration=final_duration, @@ -441,10 +413,7 @@ class McpActionExecutor: def finalize(res: ActionResult) -> str: if not res.ok: return f"Error dragging: {res.detail}" - return ( - "Swipe completed successfully. Swiped from" - f" [{x1}, {y1}] to [{x2}, {y2}]." - ) + return f"Swipe completed successfully. Swiped from [{x1}, {y1}] to [{x2}, {y2}]." return ( "swipe", @@ -494,9 +463,7 @@ class McpActionExecutor: text_summary=text, ) - async def _read_note( - self, key: str, start_line: int | None, end_line: int | None - ) -> str: + async def _read_note(self, key: str, start_line: int | None, end_line: int | None) -> str: try: base_dir = self.ctx.data_engine.base_dir if self.ctx.data_engine else None if not base_dir: @@ -514,9 +481,7 @@ class McpActionExecutor: except Exception as e: return format_list_notes_failure(str(e)) - async def _ask_explorer( - self, query: str, context_feedback: str | None, state: Any - ) -> str: + async def _ask_explorer(self, query: str, context_feedback: str | None, state: Any) -> str: # Imported lazily: the explorer stack is heavy and recursive (it spawns an LLM # sub-agent), which is also why it must never live behind the action server. from artemis.tools.explorer_tool import _run_explorer_logic diff --git a/artemis/mcp/action_manifest.py b/artemis/mcp/action_manifest.py index c26db8a..99fc681 100644 --- a/artemis/mcp/action_manifest.py +++ b/artemis/mcp/action_manifest.py @@ -122,7 +122,6 @@ BACKEND_INDEPENDENT_TOOLS: frozenset[str] = frozenset( "run_adb_command", "manage_task", "analyze_task_output", - "reply_to_checker", "report_task_status", "report_failure_analysis", } @@ -289,8 +288,7 @@ def filter_declarations( for decl in declarations # Only device actions are gated. Backend-independent tools and anything the # manifest does not classify (e.g. an agent's own bespoke tool) pass through. - if _declaration_name(decl) not in DEVICE_ACTIONS - or _declaration_name(decl) in available + if _declaration_name(decl) not in DEVICE_ACTIONS or _declaration_name(decl) in available ] for ext in actuator.extensions(): diff --git a/artemis/mcp/action_server.py b/artemis/mcp/action_server.py index 91bfef2..7369f74 100644 --- a/artemis/mcp/action_server.py +++ b/artemis/mcp/action_server.py @@ -37,6 +37,11 @@ from mcp.server.fastmcp import FastMCP from mcp.types import CallToolResult, ImageContent, TextContent from artemis.mcp.action_manifest import validate_actuator +from artemis.mcp.action_specs import ( + exception_prefix, + make_wire_handler, + wire_dialects, +) from artemis.mcp.action_types import ActionCode, ActionResult, ObserveResult from artemis.mcp.actuators.base import Actuator from artemis.mcp.observation import observe as observe_impl @@ -46,19 +51,6 @@ logger = get_logger(__name__) __all__ = ["build_action_server"] -# Exception wrapping per tool; wording matches the historical executor `except` arms. -_EXCEPTION_PREFIX = { - "click": "Error during click", - "click_sequence": "Error executing click sequence", - "long_press": "Error during long press", - "input_text": "Error during input text", - "swipe": "Error during swipe", - "press_key": "Error during press_key", - "manage_app": "Error during manage_app", - "wait_for_delay": "Error during wait_for_delay", - "wait_for_text": "Error during wait_for_text", -} - def _wrap(res: ActionResult) -> CallToolResult: return CallToolResult( @@ -69,9 +61,8 @@ def _wrap(res: ActionResult) -> CallToolResult: def _wrap_exception(action: str, exc: Exception) -> CallToolResult: - prefix = _EXCEPTION_PREFIX.get(action, f"Error during {action}") return _wrap( - ActionResult.failure(action, f"{prefix}: {exc}", detail=repr(exc)) + ActionResult.failure(action, f"{exception_prefix(action)}: {exc}", detail=repr(exc)) ) @@ -86,166 +77,17 @@ def build_action_server(actuator: Actuator, name: str = "artemis_actions") -> Fa mcp = FastMCP(name) # --- Device actions (registered only when implemented) --------------------------- + # Declarations and argument conversion both come from the canonical manifest + # (artemis/mcp/action_specs.py); this server contributes only the transport + # wrapping, so the served schema cannot drift from the manifest. - if "click" in caps: - - @mcp.tool() - async def click( - target: list[int], times: int = 1, delay_ms: int = 100 - ) -> Annotated[CallToolResult, ActionResult]: - """Tap at a 0-1000 normalized [x, y] coordinate.""" - try: - return _wrap( - await actuator.click( - int(target[0]), int(target[1]), times=times, delay_ms=delay_ms - ) - ) - except Exception as e: - return _wrap_exception("click", e) - - if "click_sequence" in caps: - - @mcp.tool() - async def click_sequence( - sequence: list[list[int]], delay_ms: int = 50 - ) -> Annotated[CallToolResult, ActionResult]: - """Tap a series of 0-1000 normalized [x, y] points in one atomic burst.""" - try: - points = [(int(p[0]), int(p[1])) for p in sequence] - return _wrap(await actuator.click_sequence(points, delay_ms=delay_ms)) - except Exception as e: - return _wrap_exception("click_sequence", e) - - if "long_press" in caps: - - @mcp.tool() - async def long_press( - target: list[int], duration_ms: int = 1000 - ) -> Annotated[CallToolResult, ActionResult]: - """Long-press at a 0-1000 normalized [x, y] coordinate.""" - try: - return _wrap( - await actuator.long_press( - int(target[0]), int(target[1]), duration_ms=duration_ms - ) - ) - except Exception as e: - return _wrap_exception("long_press", e) - - if "input_text" in caps: - - @mcp.tool() - async def input_text( - text: str, - target: list[int] | None = None, - clear_exist: bool = True, - ) -> Annotated[CallToolResult, ActionResult]: - """Type text, optionally focusing a 0-1000 normalized [x, y] target first.""" - try: - norm = (int(target[0]), int(target[1])) if target else None - return _wrap( - await actuator.input_text(text, norm, clear_exist=clear_exist) - ) - except Exception as e: - return _wrap_exception("input_text", e) - - if "swipe" in caps: - - @mcp.tool() - async def swipe( - start: list[int], end: list[int], duration_ms: int = 800 - ) -> Annotated[CallToolResult, ActionResult]: - """Swipe between two 0-1000 normalized [x, y] points.""" - try: - return _wrap( - await actuator.swipe( - (int(start[0]), int(start[1])), - (int(end[0]), int(end[1])), - duration_ms, - ) - ) - except Exception as e: - return _wrap_exception("swipe", e) - - if "press_key" in caps: - - @mcp.tool() - async def press_key(key: str) -> Annotated[CallToolResult, ActionResult]: - """Press a device key (home, back, enter, delete, tab, search, menu, app_switch).""" - try: - return _wrap(await actuator.press_key(key)) - except Exception as e: - return _wrap_exception("press_key", e) - - if "manage_app" in caps: - - @mcp.tool() - async def manage_app( - action: str, app_name: str - ) -> Annotated[CallToolResult, ActionResult]: - """Launch or stop an app by human-readable name or package.""" - try: - return _wrap(await actuator.manage_app(action, app_name)) - except Exception as e: - return _wrap_exception("manage_app", e) - - if "wait_for_delay" in caps: - - @mcp.tool() - async def wait_for_delay( - time_in_ms: int, - ) -> Annotated[CallToolResult, ActionResult]: - """Wait for a fixed number of milliseconds.""" - try: - return _wrap(await actuator.wait_for_delay(time_in_ms)) - except Exception as e: - return _wrap_exception("wait_for_delay", e) - - if "wait_for_text" in caps: - - @mcp.tool() - async def wait_for_text( - text: str, wait_state: str = "appear", timeout_ms: int = 5000 - ) -> Annotated[CallToolResult, ActionResult]: - """Wait for text to appear on or disappear from the screen.""" - try: - return _wrap(await actuator.wait_for_text(text, wait_state, timeout_ms)) - except Exception as e: - return _wrap_exception("wait_for_text", e) - - if "open_link" in caps: - - @mcp.tool() - async def open_link(url: str) -> Annotated[CallToolResult, ActionResult]: - """Open a URL on the device.""" - try: - return _wrap(await actuator.open_link(url)) - except Exception as e: - return _wrap_exception("open_link", e) - - if "erase_one_char" in caps: - - @mcp.tool() - async def erase_one_char() -> Annotated[CallToolResult, ActionResult]: - """Erase a single character in the focused field.""" - try: - return _wrap(await actuator.erase_one_char()) - except Exception as e: - return _wrap_exception("erase_one_char", e) - - if "focus_and_clear_text" in caps: - - @mcp.tool() - async def focus_and_clear_text( - target: list[int], - ) -> Annotated[CallToolResult, ActionResult]: - """Focus the field at a 0-1000 normalized [x, y] and clear its text.""" - try: - return _wrap( - await actuator.focus_and_clear_text(int(target[0]), int(target[1])) - ) - except Exception as e: - return _wrap_exception("focus_and_clear_text", e) + for spec in wire_dialects(): + if spec.name in caps: + mcp.add_tool( + make_wire_handler(spec, actuator, _wrap, _wrap_exception), + name=spec.name, + description=spec.wire.description, + ) # --- Internal observation tools (never declared to an LLM) ----------------------- @@ -315,7 +157,9 @@ def build_action_server(actuator: Actuator, name: str = "artemis_actions") -> Fa except Exception as e: payload = {"ok": False, "error": str(e)} return CallToolResult( - content=[TextContent(type="text", text="screenshot" if payload["ok"] else str(payload))], + content=[ + TextContent(type="text", text="screenshot" if payload["ok"] else str(payload)) + ], structuredContent=payload, isError=False, ) @@ -332,11 +176,16 @@ def build_action_server(actuator: Actuator, name: str = "artemis_actions") -> Fa elements = await coro payload: dict[str, Any] = {"ok": True, "elements": elements} except TimeoutError: - payload = {"ok": False, "error": f"ui hierarchy fetch timed out after {timeout_ms}ms"} + payload = { + "ok": False, + "error": f"ui hierarchy fetch timed out after {timeout_ms}ms", + } except Exception as e: payload = {"ok": False, "error": str(e)} return CallToolResult( - content=[TextContent(type="text", text="ui_hierarchy" if payload["ok"] else str(payload))], + content=[ + TextContent(type="text", text="ui_hierarchy" if payload["ok"] else str(payload)) + ], structuredContent=payload, isError=False, ) diff --git a/artemis/mcp/action_specs.py b/artemis/mcp/action_specs.py new file mode 100644 index 0000000..784eae1 --- /dev/null +++ b/artemis/mcp/action_specs.py @@ -0,0 +1,961 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The canonical device-action manifest: every dialect of every action, side by side. + +Historically each device action was declared independently at three model-facing +sites -- the Operator's inline LangChain shells, the Validator/Flash/FailureAnalyzer +``ToolDeclaration`` constants, and the action MCP server's function signatures -- and +the declarations drifted apart (``click.target`` took an element index in one place +and only coordinates in another). This module is now the single place all three are +*defined*; the historical sites import their surface from here, generated. + +An action has up to three dialects, and the differences between them are **deliberate +and declared**, not accidental: + +* ``operator`` -- the LangChain shell tools the Operator binds. The Operator receives + an indexed "Visible UI Elements" list and its prompt teaches index-first targeting, + so target parameters accept ``int`` element indices as well as normalized + coordinates. The shells are declaration-only ("Action Recorded"): the Operator's + translate step lowers them into structured decisions. +* ``declaration`` -- the JSON ``ToolDeclaration`` bound by the Validator's + FailureAnalyzer and the FlashRunner. These agents act on coordinates directly (the + element list they see carries coordinates), so target parameters are coordinate + pairs and the wording teaches the act-then-observe loop ("The screen after X will + be returned automatically"). +* ``wire`` -- what the action MCP server serves, mirroring the actuator protocol + one-to-one: normalized coordinates, millisecond durations, no addressing sugar. + Client-side executors (``McpActionExecutor``, ``MobileActionExecutor``) lower the + agent dialects onto it. + +Each ``ActionSpec.param_bridge`` maps operator parameter names onto declaration +parameter names, so every rename (``duration`` -> ``duration_ms``, ``gesture`` -> +``action``) is written down exactly once. ``tests/unit/mcp/test_action_specs.py`` +re-derives the bridge from the generated schemas, so an undeclared divergence -- +today's definition of drift -- fails CI instead of shipping. + +The classification of actions (required/optional/internal) stays in +``action_manifest``; this module holds their schemas and teaching text. +""" + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from functools import cache +import inspect +from typing import Annotated, Any, Literal + +from langchain_core.tools import StructuredTool +from mcp.types import CallToolResult +from pydantic import Field, create_model + +from artemis.core.tool_declaration import ToolDeclaration +from artemis.mcp.action_types import ActionResult + +__all__ = [ + "ACTION_SPECS", + "ActionSpec", + "DeclarationDialect", + "EXCEPTION_PREFIXES", + "OPERATOR_SHELL_ORDER", + "OperatorDialect", + "ParamSpec", + "WireDialect", + "exception_prefix", + "make_wire_handler", + "operator_shell_tool", + "tool_declaration", + "wire_dialects", +] + + +# --- Spec structure ------------------------------------------------------------------ + + +@dataclass(frozen=True) +class ParamSpec: + """One parameter of a Python-typed dialect (operator shell or wire).""" + + name: str + annotation: Any + description: str | None = None + required: bool = True + default: Any = None + + +@dataclass(frozen=True) +class OperatorDialect: + """The Operator's LangChain shell: index-capable targets, declaration-only body.""" + + description: str + params: tuple[ParamSpec, ...] + + +@dataclass(frozen=True) +class DeclarationDialect: + """The Validator/Flash/FailureAnalyzer JSON declaration, wording preserved.""" + + description: str + parameters: dict[str, Any] + + +@dataclass(frozen=True) +class WireDialect: + """The action-server tool mirroring one actuator method. + + ``bind`` receives the actuator and the validated, default-filled argument dict + and performs the (tiny) call-site conversion the historical hand-written wrapper + performed -- typically ``int()`` coercion of coordinate pairs. + """ + + description: str + params: tuple[ParamSpec, ...] + bind: Callable[[Any, dict[str, Any]], Awaitable[ActionResult]] + + +@dataclass(frozen=True) +class ActionSpec: + """All dialects of one canonical device action, plus their declared differences. + + Attributes: + name: The canonical tool name every dialect shares. + operator: Operator shell dialect, or ``None`` when the Operator does not bind + this action. + declaration: Validator/Flash dialect, or ``None`` when no JSON declaration + exists (an action can be wire-only, reachable through executors but never + declared to a model directly). + wire: Action-server dialect; ``None`` only for purely virtual actions. + param_bridge: operator param name -> declaration param name, for every + operator param, when both dialects exist. Identical names map to + themselves; renames are the deliberate dialect differences. + differences: Human-readable record of every deliberate cross-dialect + divergence (semantics, not just spelling). Empty means the dialects + align modulo the documented dialect philosophy above. + """ + + name: str + operator: OperatorDialect | None = None + declaration: DeclarationDialect | None = None + wire: WireDialect | None = None + param_bridge: dict[str, str] = field(default_factory=dict) + differences: str = "" + + +# --- Shared type aliases ------------------------------------------------------------- + +#: Operator-dialect target: element index into the indexed UI list, or a normalized +#: [x, y] pair. Only the Operator sees this union; see the module docstring. +IndexOrCoords = int | list[int] + +Direction = Literal["up", "down", "left", "right"] + +_COORD_PAIR_JSON = { + "type": "array", + "items": {"type": "integer"}, +} + + +# --- Wire bindings ------------------------------------------------------------------- +# Each binding reproduces the exact conversion of the historical hand-written FastMCP +# wrapper it replaces. Arguments arrive validated and default-filled. + + +async def _wire_click(actuator: Any, a: dict[str, Any]) -> ActionResult: + return await actuator.click( + int(a["target"][0]), int(a["target"][1]), times=a["times"], delay_ms=a["delay_ms"] + ) + + +async def _wire_click_sequence(actuator: Any, a: dict[str, Any]) -> ActionResult: + points = [(int(p[0]), int(p[1])) for p in a["sequence"]] + return await actuator.click_sequence(points, delay_ms=a["delay_ms"]) + + +async def _wire_long_press(actuator: Any, a: dict[str, Any]) -> ActionResult: + return await actuator.long_press( + int(a["target"][0]), int(a["target"][1]), duration_ms=a["duration_ms"] + ) + + +async def _wire_input_text(actuator: Any, a: dict[str, Any]) -> ActionResult: + target = a["target"] + norm = (int(target[0]), int(target[1])) if target else None + return await actuator.input_text(a["text"], norm, clear_exist=a["clear_exist"]) + + +async def _wire_swipe(actuator: Any, a: dict[str, Any]) -> ActionResult: + start, end = a["start"], a["end"] + return await actuator.swipe( + (int(start[0]), int(start[1])), (int(end[0]), int(end[1])), a["duration_ms"] + ) + + +async def _wire_press_key(actuator: Any, a: dict[str, Any]) -> ActionResult: + return await actuator.press_key(a["key"]) + + +async def _wire_manage_app(actuator: Any, a: dict[str, Any]) -> ActionResult: + return await actuator.manage_app(a["action"], a["app_name"]) + + +async def _wire_wait_for_delay(actuator: Any, a: dict[str, Any]) -> ActionResult: + return await actuator.wait_for_delay(a["time_in_ms"]) + + +async def _wire_wait_for_text(actuator: Any, a: dict[str, Any]) -> ActionResult: + return await actuator.wait_for_text(a["text"], a["wait_state"], a["timeout_ms"]) + + +async def _wire_open_link(actuator: Any, a: dict[str, Any]) -> ActionResult: + return await actuator.open_link(a["url"]) + + +async def _wire_erase_one_char(actuator: Any, a: dict[str, Any]) -> ActionResult: + return await actuator.erase_one_char() + + +async def _wire_focus_and_clear_text(actuator: Any, a: dict[str, Any]) -> ActionResult: + return await actuator.focus_and_clear_text(int(a["target"][0]), int(a["target"][1])) + + +# --- The manifest -------------------------------------------------------------------- + +_SPECS: tuple[ActionSpec, ...] = ( + ActionSpec( + name="click", + operator=OperatorDialect( + description=( + "[ACTION] Click on the target location on the screen (supports element" + " index or absolute normalized coordinates)." + ), + params=( + ParamSpec( + "target", + IndexOrCoords, + "Click target. Can be an element index number (int, e.g. 3) OR" + " normalized coordinates (list of 2 integers, e.g. [500, 600]).", + ), + ParamSpec( + "times", + int, + "Number of consecutive clicks on this target. Use this for" + " double-clicks or multi-clicks (e.g. 7 to enter developer" + " mode). Default is 1.", + required=False, + default=1, + ), + ParamSpec( + "delay_ms", + int, + "Delay in milliseconds between consecutive clicks. Default is 100.", + required=False, + default=100, + ), + ), + ), + declaration=DeclarationDialect( + description=( + "[ACTION] Tap/click on the target coordinate on the screen. The screen" + " after click will be returned automatically. For buttons, checkboxes," + " tabs, icons, items in a list, tap ON the element. For text fields /" + " search bars / input boxes, tap INSIDE the input box to focus it." + ), + parameters={ + "type": "object", + "properties": { + "target": { + "type": "array", + "items": {"type": "integer"}, + "description": "Normalized coordinates [x, y] in 0-1000 scale.", + }, + "times": { + "type": "integer", + "description": ( + "Number of taps to perform (default 1). Set to 2 for double click." + ), + }, + "delay_ms": { + "type": "integer", + "description": "Delay between taps in milliseconds (default 100).", + }, + }, + "required": ["target"], + }, + ), + wire=WireDialect( + description="Tap at a 0-1000 normalized [x, y] coordinate.", + params=( + ParamSpec("target", list[int]), + ParamSpec("times", int, required=False, default=1), + ParamSpec("delay_ms", int, required=False, default=100), + ), + bind=_wire_click, + ), + param_bridge={"target": "target", "times": "times", "delay_ms": "delay_ms"}, + differences=( + "target: operator accepts an element index or a coordinate pair; the" + " declaration and wire dialects take coordinates only." + ), + ), + ActionSpec( + name="click_sequence", + declaration=DeclarationDialect( + description=( + "[ACTION] Executes a sequence of taps one by one in order on the" + " specified targets (e.g. [[500, 280], [885, 362]]). The screen will be returned ONLY after all clicks" + " in the sequence have completed." + ), + parameters={ + "type": "object", + "properties": { + "sequence": { + "type": "array", + "items": { + "type": "array", + "items": {"type": "integer"}, + "description": ( + "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280])," + " or a single integer element index." + ), + }, + "description": ( + "List of targets to tap in sequence, e.g. [[500, 280], [885, 362]]." + ), + }, + "delay_ms": { + "type": "integer", + "description": ( + "Delay between consecutive taps in milliseconds (default 50ms)." + ), + }, + }, + "required": ["sequence"], + }, + ), + wire=WireDialect( + description=("Tap a series of 0-1000 normalized [x, y] points in one atomic burst."), + params=( + ParamSpec("sequence", list[list[int]]), + ParamSpec("delay_ms", int, required=False, default=50), + ), + bind=_wire_click_sequence, + ), + differences=( + "Declared to Flash and the disappeared-target FailureAnalyzer strategies" + " only; sequence entries may be element indices at the declaration layer" + " (resolved client-side), while the wire takes coordinate pairs only." + ), + ), + ActionSpec( + name="long_press", + operator=OperatorDialect( + description=( + "[ACTION] Long press on the target location on the screen (supports" + " element index or absolute normalized coordinates)." + ), + params=( + ParamSpec( + "target", + IndexOrCoords, + "Long press target. Can be an element index number (int, e.g." + " 3) OR normalized coordinates (list of 2 integers, e.g. [500," + " 600]).", + ), + ParamSpec( + "duration", + int, + "Long press duration in milliseconds (default 1000).", + required=False, + default=1000, + ), + ), + ), + declaration=DeclarationDialect( + description=( + "[ACTION] Long press on a target coordinate on the screen. The screen" + " after long pressing will be returned automatically." + ), + parameters={ + "type": "object", + "properties": { + "target": { + "type": "array", + "items": {"type": "integer"}, + "description": "Normalized coordinates [x, y] in 0-1000 scale.", + }, + "duration_ms": { + "type": "integer", + "description": "Duration of press in milliseconds (default 1000ms).", + }, + }, + "required": ["target"], + }, + ), + wire=WireDialect( + description="Long-press at a 0-1000 normalized [x, y] coordinate.", + params=( + ParamSpec("target", list[int]), + ParamSpec("duration_ms", int, required=False, default=1000), + ), + bind=_wire_long_press, + ), + param_bridge={"target": "target", "duration": "duration_ms"}, + differences=( + "target: operator accepts an element index or a coordinate pair." + " duration: the operator spells the duration parameter `duration` (its" + " structured decisions and recorded traces carry that key); the" + " declaration and wire dialects spell it `duration_ms`. Executors accept" + " both spellings." + ), + ), + ActionSpec( + name="input_text", + operator=OperatorDialect( + description=( + "[ACTION] Type text into the target input field (supports replacing" + " whole text or appending to the end, and multi-line strings with" + " '\\n')." + ), + params=( + ParamSpec( + "text", + str, + "The text content to input. Supports multi-line content with '\\n'.", + ), + ParamSpec( + "target", + IndexOrCoords, + "Input target field. Can be an input box element index number" + " (int, e.g. 3) OR normalized coordinates (list of 2 integers," + " e.g. [500, 600]).", + ), + ParamSpec( + "clear_exist", + bool, + "Whether to clear existing text before typing. True (default):" + " clear/replace entire text. False: append at the end of" + " existing content.", + required=False, + default=True, + ), + ), + ), + declaration=DeclarationDialect( + description=( + "[ACTION] Type text into an input field on the screen. The screen after" + " typing will be returned automatically. Automatically taps inside the" + " input box at target [x, y] to focus before typing." + ), + parameters={ + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "Text to type into the focused input field.", + }, + "target": { + "type": "array", + "items": {"type": "integer"}, + "description": "Coordinates [x, y] of the input box in 0-1000 scale.", + }, + "clear_exist": { + "type": "boolean", + "description": ( + "Whether to clear existing text in the input box before typing" + " (default True)." + ), + }, + }, + "required": ["text", "target"], + }, + ), + wire=WireDialect( + description=("Type text, optionally focusing a 0-1000 normalized [x, y] target first."), + params=( + ParamSpec("text", str), + ParamSpec("target", list[int] | None, required=False, default=None), + ParamSpec("clear_exist", bool, required=False, default=True), + ), + bind=_wire_input_text, + ), + param_bridge={"target": "target", "text": "text", "clear_exist": "clear_exist"}, + differences=( + "target: operator accepts an element index or a coordinate pair, and" + " requires a target; the wire dialect allows omitting the target to type" + " into the already-focused field." + ), + ), + ActionSpec( + name="swipe", + operator=OperatorDialect( + description=( + "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n" + "\n" + "• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n" + "• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n" + "\n" + "Args:\n" + " direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n" + " start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n" + " end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n" + " target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n" + " gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n" + " duration: Optional gesture duration in milliseconds (default 800)." + ), + params=( + ParamSpec( + "direction", + Direction | None, + "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below)," + " 'down' (drags top-to-bottom, scrolling up to reveal content above)," + " 'left' (drags right-to-left, scrolling right)," + " 'right' (drags left-to-right, scrolling left).", + required=False, + default=None, + ), + ParamSpec( + "start", + list[int] | None, + "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise," + " local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop).", + required=False, + default=None, + ), + ParamSpec( + "end", + list[int] | None, + "End normalized coordinates [end_x, end_y] in 0-1000 scale for precise," + " local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop).", + required=False, + default=None, + ), + ParamSpec( + "target", + int | list[int] | str | None, + "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.", + required=False, + default=None, + ), + ParamSpec( + "gesture", + Direction | list[int] | None, + "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right')" + " OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale.", + required=False, + default=None, + ), + ParamSpec( + "duration", + int | None, + "Optional swipe/drag duration in milliseconds (default 800). For drag-and-drop," + " list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars)," + " set duration >= 1000 (e.g. 1500). If omitted for directional swipe, duration is computed automatically.", + required=False, + default=None, + ), + ), + ), + declaration=DeclarationDialect( + description=( + "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen. The screen and UI hierarchy after swipe will be returned automatically.\n\n" + "• Directional Scrolling ('direction' or 'action'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n" + "• Precise Coordinate Gestures ('start', 'end' or coordinates list): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary." + ), + parameters={ + "type": "object", + "properties": { + "direction": { + "type": "string", + "enum": ["up", "down", "left", "right"], + "description": ( + "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below)," + " 'down' (drags top-to-bottom, scrolling up to reveal content above)," + " 'left' (drags right-to-left, scrolling right)," + " 'right' (drags left-to-right, scrolling left)." + ), + }, + "start": { + "type": "array", + "items": {"type": "integer"}, + "description": ( + "Start normalized coordinates [start_x, start_y] in 0-1000 scale." + ), + }, + "end": { + "type": "array", + "items": {"type": "integer"}, + "description": ( + "End normalized coordinates [end_x, end_y] in 0-1000 scale." + ), + }, + "target": { + "description": ( + "Optional target element index (e.g. 2) or container bounds" + " [left, top, right, bottom] to scope the directional swipe within." + ), + }, + "action": { + "description": ( + "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right')" + " OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." + ), + }, + "duration": { + "type": "integer", + "description": ( + "Optional swipe/drag duration in milliseconds (default 800). For drag-and-drop," + " list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars)," + " set duration >= 1000 (e.g. 1500). If omitted for directional swipe, duration is computed automatically." + ), + }, + }, + }, + ), + wire=WireDialect( + description="Swipe between two 0-1000 normalized [x, y] points.", + params=( + ParamSpec("start", list[int]), + ParamSpec("end", list[int]), + ParamSpec("duration_ms", int, required=False, default=800), + ), + bind=_wire_swipe, + ), + param_bridge={ + "direction": "direction", + "start": "start", + "end": "end", + "target": "target", + "gesture": "action", + "duration": "duration", + }, + differences=( + "gesture/action: the legacy combined direction-or-coordinates parameter is" + " spelled `gesture` in the operator dialect and `action` in the" + " declaration dialect (`action` collides with the operator's structured" + " decision verb field). Smart directional swipes exist only in the agent" + " dialects; the wire takes a resolved start/end pair, and direction" + " resolution happens client-side against the live UI tree." + ), + ), + ActionSpec( + name="press_key", + operator=OperatorDialect( + description=( + "[ACTION] Press a physical or virtual system button (e.g.\n" + "\n" + "ENTER, BACK, HOME, APP_SWITCH)." + ), + params=( + ParamSpec( + "key", + Literal["ENTER", "BACK", "HOME", "APP_SWITCH"], + "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH).", + ), + ), + ), + declaration=DeclarationDialect( + description=( + "[ACTION] Press a physical or virtual system button. The screen after" + " pressing will be returned automatically." + ), + parameters={ + "type": "object", + "properties": { + "key": { + "type": "string", + "description": ( + "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH)." + ), + } + }, + "required": ["key"], + }, + ), + wire=WireDialect( + description=( + "Press a device key (home, back, enter, delete, tab, search, menu, app_switch)." + ), + params=(ParamSpec("key", str),), + bind=_wire_press_key, + ), + param_bridge={"key": "key"}, + differences=( + "key vocabulary: the operator dialect is a closed uppercase enum (its" + " translate step emits Android KEYCODE_* names); the declaration dialect" + " is an open string taught the same uppercase names; the wire takes the" + " actuator's bare lowercase key words. `to_canonical_call` and the" + " executors normalize between them." + ), + ), + ActionSpec( + name="manage_app", + operator=OperatorDialect( + description="[ACTION] Launch or force stop a specified application.", + params=( + ParamSpec("action", Literal["launch", "stop"], "The action type."), + ParamSpec( + "app_name", + str, + "Display name or package name of the application.", + ), + ), + ), + declaration=DeclarationDialect( + description=( + "[ACTION] Launch or force stop a specified application. The screen" + " after launch/stop will be returned automatically." + ), + parameters={ + "type": "object", + "properties": { + "action": { + "type": "string", + "description": "The action type ('launch' or 'stop').", + }, + "app_name": { + "type": "string", + "description": "Display name or package name of the application.", + }, + }, + "required": ["action", "app_name"], + }, + ), + wire=WireDialect( + description="Launch or stop an app by human-readable name or package.", + params=( + ParamSpec("action", str), + ParamSpec("app_name", str), + ), + bind=_wire_manage_app, + ), + param_bridge={"action": "action", "app_name": "app_name"}, + ), + ActionSpec( + name="wait_for_delay", + operator=OperatorDialect( + description=( + "[ACTION] Pause execution and wait for a specified duration in milliseconds.\n" + "\n" + "Use this whenever you need time to elapse—whether for UI loading, animations,\n" + "screen transitions, or longer scheduled delays and intervals specified in the task." + ), + params=( + ParamSpec( + "time_in_ms", + int, + "The exact duration to wait in milliseconds. Accurately convert the" + " required time duration into milliseconds based on your objective" + " or plan (e.g., 2000 for 2s, 5000 for 5s, 60000 for 1 minute," + " 180000 for 3 minutes, 300000 for 5 minutes).", + ), + ), + ), + declaration=DeclarationDialect( + description=( + "[ACTION] Pause execution and wait for a specified duration in milliseconds." + " The screen after pause will be returned automatically." + ), + parameters={ + "type": "object", + "properties": { + "time_in_ms": { + "type": "integer", + "description": ( + "The exact duration to pause in milliseconds (e.g., 2000 for 2s," + " 60000 for 1 min, 180000 for 3 mins, 300000 for 5 mins)." + " Convert any required waiting duration into milliseconds." + ), + } + }, + "required": ["time_in_ms"], + }, + ), + wire=WireDialect( + description="Wait for a fixed number of milliseconds.", + params=(ParamSpec("time_in_ms", int),), + bind=_wire_wait_for_delay, + ), + param_bridge={"time_in_ms": "time_in_ms"}, + ), + ActionSpec( + name="wait_for_text", + declaration=None, # Listed in the legacy tool-name sets but never declared to + # any LLM; kept wire-reachable for executors and external MCP clients. + wire=WireDialect( + description="Wait for text to appear on or disappear from the screen.", + params=( + ParamSpec("text", str), + ParamSpec("wait_state", str, required=False, default="appear"), + ParamSpec("timeout_ms", int, required=False, default=5000), + ), + bind=_wire_wait_for_text, + ), + ), + ActionSpec( + name="open_link", + wire=WireDialect( + description="Open a URL on the device.", + params=(ParamSpec("url", str),), + bind=_wire_open_link, + ), + ), + ActionSpec( + name="erase_one_char", + wire=WireDialect( + description="Erase a single character in the focused field.", + params=(), + bind=_wire_erase_one_char, + ), + ), + ActionSpec( + name="focus_and_clear_text", + wire=WireDialect( + description=("Focus the field at a 0-1000 normalized [x, y] and clear its text."), + params=(ParamSpec("target", list[int]),), + bind=_wire_focus_and_clear_text, + ), + ), +) + +#: Canonical manifest, keyed by action name, in declaration order. +ACTION_SPECS: dict[str, ActionSpec] = {spec.name: spec for spec in _SPECS} + +#: The order the Operator binds its shells (historically the factory-dict order, +#: identical to the prompt's "Physical device actions" enumeration order). +OPERATOR_SHELL_ORDER: tuple[str, ...] = ( + "click", + "input_text", + "swipe", + "press_key", + "manage_app", + "wait_for_delay", + "long_press", +) + +#: Exception wording per action; matches the historical executor `except` arms. +EXCEPTION_PREFIXES: dict[str, str] = { + "click": "Error during click", + "click_sequence": "Error executing click sequence", + "long_press": "Error during long press", + "input_text": "Error during input text", + "swipe": "Error during swipe", + "press_key": "Error during press_key", + "manage_app": "Error during manage_app", + "wait_for_delay": "Error during wait_for_delay", + "wait_for_text": "Error during wait_for_text", +} + + +def exception_prefix(action: str) -> str: + """Returns the historical human-readable exception prefix for an action.""" + return EXCEPTION_PREFIXES.get(action, f"Error during {action}") + + +# --- Projections --------------------------------------------------------------------- + + +@cache +def _operator_args_model(name: str): + """Builds (once) the pydantic argument model for an operator shell.""" + dialect = ACTION_SPECS[name].operator + if dialect is None: + raise ValueError(f"Action '{name}' has no operator dialect.") + fields: dict[str, Any] = {} + for p in dialect.params: + if p.required: + fields[p.name] = (p.annotation, Field(description=p.description)) + else: + fields[p.name] = ( + p.annotation, + Field(default=p.default, description=p.description), + ) + return create_model(name, **fields) + + +def operator_shell_tool(name: str) -> StructuredTool: + """Builds the Operator's declaration-only shell for one action. + + The body is never executed by the Operator loop -- action calls are translated + into structured decisions instead -- but returns the historical marker string for + any caller that does invoke it. + """ + dialect = ACTION_SPECS[name].operator + if dialect is None: + raise ValueError(f"Action '{name}' has no operator dialect.") + return StructuredTool( + name=name, + description=dialect.description, + args_schema=_operator_args_model(name), + func=lambda **kwargs: "Action Recorded", + ) + + +def tool_declaration(name: str) -> ToolDeclaration: + """Builds the Validator/Flash ``ToolDeclaration`` for one action.""" + dialect = ACTION_SPECS[name].declaration + if dialect is None: + raise ValueError(f"Action '{name}' has no declaration dialect.") + return ToolDeclaration( + name=name, + description=dialect.description, + parameters=dialect.parameters, + ) + + +def wire_dialects() -> tuple[ActionSpec, ...]: + """Returns every spec that has a wire dialect, in manifest order.""" + return tuple(spec for spec in _SPECS if spec.wire is not None) + + +def make_wire_handler( + spec: ActionSpec, + actuator: Any, + wrap: Callable[[ActionResult], CallToolResult], + wrap_exception: Callable[[str, Exception], CallToolResult], +) -> Callable[..., Awaitable[CallToolResult]]: + """Builds the FastMCP tool function for one wire dialect. + + The returned coroutine carries an explicit ``__signature__`` so FastMCP derives + the same argument model a literal ``async def`` produced historically, and the + same ``Annotated[CallToolResult, ActionResult]`` return annotation so the + structured-output schema is unchanged. + """ + wire = spec.wire + if wire is None: + raise ValueError(f"Action '{spec.name}' has no wire dialect.") + + async def handler(**kwargs: Any) -> Any: + try: + return wrap(await wire.bind(actuator, kwargs)) + except Exception as e: # pylint: disable=broad-exception-caught + return wrap_exception(spec.name, e) + + return_annotation = Annotated[CallToolResult, ActionResult] + parameters = [ + inspect.Parameter( + p.name, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + annotation=p.annotation, + default=inspect.Parameter.empty if p.required else p.default, + ) + for p in wire.params + ] + handler.__name__ = spec.name + handler.__qualname__ = spec.name + handler.__doc__ = wire.description + handler.__signature__ = inspect.Signature( # type: ignore[attr-defined] + parameters, return_annotation=return_annotation + ) + handler.__annotations__ = {p.name: p.annotation for p in wire.params} + handler.__annotations__["return"] = return_annotation + return handler diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py index 1f352c6..b999718 100644 --- a/artemis/sdk/agent.py +++ b/artemis/sdk/agent.py @@ -35,7 +35,6 @@ import uuid from adbutils import AdbClient from dotenv import load_dotenv from google import genai -from langchain_core.messages import AIMessage from langchain_google_genai import ChatGoogleGenerativeAI from PIL import Image from pydantic import BaseModel @@ -117,14 +116,13 @@ class Agent: session_id: str | None = None, ): raw_sid = ( - session_id - or os.getenv("ARTEMIS_SESSION_ID") - or os.getenv("ARTEMIS_CLOUD_SESSION_ID") + session_id or os.getenv("ARTEMIS_SESSION_ID") or os.getenv("ARTEMIS_CLOUD_SESSION_ID") ) self._session_id: str | None = str(raw_sid).strip() if raw_sid else None target_dev = device_serial or device_id if config is None: from artemis.sdk.builders import Builders + builder = Builders.AgentConfig if target_dev: builder.for_device(DevicePlatform.ANDROID, target_dev) @@ -203,7 +201,9 @@ class Agent: raise DeviceNotFoundError(error_msg) # Initialize clients - publish_startup_progress("device_check", "Checking the Android device", session_id=self._session_id) + publish_startup_progress( + "device_check", "Checking the Android device", session_id=self._session_id + ) if os.environ.get("ARTEMIS_CLOUD_MODE") != "1": self._init_clients( device_id=device_id, @@ -222,7 +222,9 @@ class Agent: device_id=device_id, platform=platform ) logger.info(self._device_context.to_str()) - publish_startup_progress("device_ready", "Android device connected", session_id=self._session_id) + publish_startup_progress( + "device_ready", "Android device connected", session_id=self._session_id + ) # Asynchronously pre-warm LLM connection pools in the background asyncio.create_task(self._prewarm_llm_connections(api_key)) @@ -236,9 +238,13 @@ class Agent: """Pre-warms the HTTP2/gRPC connection pools for both Native GenAI and LangChain clients in the background.""" if os.environ.get("ARTEMIS_FAKE_LLM") == "1": logger.info("ARTEMIS_FAKE_LLM=1 — skipping real LLM connection pre-warming.") - publish_startup_progress("model_ready", "Model connection is ready (fake LLM)", session_id=self._session_id) + publish_startup_progress( + "model_ready", "Model connection is ready (fake LLM)", session_id=self._session_id + ) return - publish_startup_progress("model_warmup", "Warming the model connection", session_id=self._session_id) + publish_startup_progress( + "model_warmup", "Warming the model connection", session_id=self._session_id + ) logger.info("Starting background pre-warming of Gemini API connection pools...") try: key = api_key @@ -248,7 +254,9 @@ class Agent: if not key: logger.warning("Skipping LLM pre-warming: No API key available.") publish_startup_progress( - "model_ready", "Model connection will initialize on first use", session_id=self._session_id + "model_ready", + "Model connection will initialize on first use", + session_id=self._session_id, ) return @@ -265,11 +273,15 @@ class Agent: return_exceptions=True, ) logger.success("Gemini API connection pools successfully pre-warmed.") - publish_startup_progress("model_ready", "Model connection is ready", session_id=self._session_id) + publish_startup_progress( + "model_ready", "Model connection is ready", session_id=self._session_id + ) except Exception as e: logger.warning(f"Failed to pre-warm LLM connections: {e}") publish_startup_progress( - "model_ready", "Model connection will initialize on first use", session_id=self._session_id + "model_ready", + "Model connection will initialize on first use", + session_id=self._session_id, ) async def install_apk(self, apk_path: str | Path) -> None: @@ -549,13 +561,17 @@ class Agent: or active_owner.token == self._device_context.device_id ) ) - device_lock = None if already_held else DeviceExecutionLock( - self._device_context.device_id, - description=f"{request.goal[:120]}", - concurrency_mode=effective_mode, - max_concurrency=effective_max, - session_id=str(sess_id) if sess_id else None, - ingress=os.getenv("ARTEMIS_TASK_INGRESS") or "agent", + device_lock = ( + None + if already_held + else DeviceExecutionLock( + self._device_context.device_id, + description=f"{request.goal[:120]}", + concurrency_mode=effective_mode, + max_concurrency=effective_max, + session_id=str(sess_id) if sess_id else None, + ingress=os.getenv("ARTEMIS_TASK_INGRESS") or "agent", + ) ) try: if device_lock is not None and os.environ.get("ARTEMIS_CLOUD_MODE") != "1": @@ -707,7 +723,6 @@ class Agent: context.data_engine.end_session("failed") return None - print_ai_response_to_stderr(graph_result=last_state) output = await self._extract_output( task_name=task_name, ctx=context, @@ -715,7 +730,43 @@ class Agent: output_config=output_config, state=last_state, ) - logger.info(f"✅ Automation '{task_name}' is success ✅") + # Run outcome (goal axis x test axis) drives the + # wrap-up instead of an unconditional success path. + run_outcome = getattr(last_state, "run_outcome", None) + context.run_outcome = run_outcome + output = attach_test_summary(output, run_outcome) + task_status_axis = ( + run_outcome.get("task_status") if run_outcome else None + ) + tests_failed = ( + int((run_outcome.get("tests") or {}).get("failed", 0)) + if run_outcome + else 0 + ) + if task_status_axis == "blocked": + err = ( + f"[{task_name}] Task ended blocked: verify" + " criteria unmet after exhausting the final" + " check budget." + ) + logger.warning(err) + await task.finalize( + content=output, + state=last_state_snapshot, + error=err, + ) + if context.data_engine: + context.data_engine.end_session("failed") + return output + + if tests_failed > 0: + logger.warning( + f"[{task_name}] Task completed, but" + f" {tests_failed} assertion(s) failed —" + " see the test summary." + ) + else: + logger.info(f"✅ Automation '{task_name}' is success ✅") await task.finalize(content=output, state=last_state_snapshot) if context.data_engine: context.data_engine.end_session("completed") @@ -723,13 +774,18 @@ class Agent: return output finally: if recording_started: + async def _safe_stop_recording(): try: - logger.info(f"[{task_name}] Stopping automated screen recording...") + logger.info( + f"[{task_name}] Stopping automated screen recording..." + ) controller = get_controller(context) return await controller.stop_video_recording() except Exception as e: - logger.error(f"[{task_name}] Error stopping screen recording: {e}") + logger.error( + f"[{task_name}] Error stopping screen recording: {e}" + ) return None stop_task = asyncio.create_task(_safe_stop_recording()) @@ -982,8 +1038,16 @@ class Agent: else False, video_recording_tools_enabled=self._config.video_recording_tools_enabled, disable_checker=self._config.disable_checker, + disable_midway_checks=self._config.disable_midway_checks, + disable_final_check=self._config.disable_final_check, checker_max_iterations=self._config.checker_max_iterations, - checker_max_chat_rounds=self._config.checker_max_chat_rounds, + final_check_max_attempts=self._config.final_check_max_attempts, + checkpoint_max_repairs=self._config.checkpoint_max_repairs, + max_concurrent_checkpoints=self._config.max_concurrent_checkpoints, + checkpoint_timeout=self._config.checkpoint_timeout, + settlement_timeout=self._config.settlement_timeout, + assert_failure_policy=self._config.assert_failure_policy, + disable_device_probes=self._config.disable_device_probes, disable_planner_validation=self._config.disable_planner_validation, planner_validation_threshold=self._config.planner_validation_threshold, enable_committee=self._config.enable_committee, @@ -1058,7 +1122,7 @@ class Agent: logger.error(f"[{task_name}] Failed to clean up temp trace folder: {e}") return - status = "_PASS" if task.status == "completed" else "_FAIL" + status = resolve_trace_suffix(task.status, getattr(context, "run_outcome", None)) ts = task.created_at.strftime("%Y-%m-%dT%H-%M-%S") new_name = f"{exec_setup_ctx.trace_name}{status}_{ts}" @@ -1133,28 +1197,10 @@ class Agent: except Exception as e: logger.error(f"[{task_name}] Failed to generate structured output: {e}") return None - if state and state.messages: - for msg in reversed(state.messages): - if msg.content: - logger.info(f"Fallback output from message: {msg.content}") - record_events(output_path=request.llm_output_path, events=msg.content) - return msg.content return None def _get_graph_state(self, task: Task): - return State( - messages=[], - initial_goal=task.request.goal, - subgoal_plan=[], - latest_ui_hierarchy=None, - latest_screenshot=None, - focused_app_info=None, - device_date=None, - structured_decisions=None, - complete_subgoals_by_ids=[], - remaining_steps=task.request.max_steps, - validator_messages=[], - ) + return State.initial(task.request.goal) def _init_clients( self, @@ -1229,8 +1275,44 @@ def _validate_and_prepare_file(file_path: Path): raise AgentTaskRequestError(f"Error creating file '{file_path}': {e}") -def print_ai_response_to_stderr(graph_result: State): - for msg in reversed(graph_result.messages): - if isinstance(msg, AIMessage): - print(msg.content, file=sys.stderr) - return +def attach_test_summary(output, run_outcome: dict | None): + """Surfaces the machine-readable test summary in the task's return value. + + Only applies when the run actually had check items (the summary carries at + least one counted item) — otherwise the historical return shape is kept + untouched. String outputs are wrapped into a dict so that callers such as + ``mobile_run_task`` receive the summary without parsing report prose. + """ + if not run_outcome: + return output + tests = run_outcome.get("tests") or {} + total = ( + int(tests.get("passed", 0)) + + int(tests.get("failed", 0)) + + int(tests.get("inconclusive", 0)) + + int(tests.get("unchecked", 0)) + ) + if total <= 0: + return output + summary = {"task_status": run_outcome.get("task_status"), **tests} + if isinstance(output, dict): + merged = dict(output) + merged.setdefault("test_summary", summary) + return merged + if output is None: + return {"test_summary": summary} + if isinstance(output, str): + return {"result": output, "test_summary": summary} + # Typed/structured outputs keep their shape; the summary stays available in + # run_outcome.json and the session state. + return output + + +def resolve_trace_suffix(task_status: str, run_outcome: dict | None) -> str: + """Trace naming: assertion failures must be distinguishable from _PASS.""" + if task_status != "completed": + return "_FAIL" + tests = (run_outcome or {}).get("tests") or {} + if int(tests.get("failed", 0)) > 0: + return "_TESTFAIL" + return "_PASS" diff --git a/artemis/sdk/builders/agent_config_builder.py b/artemis/sdk/builders/agent_config_builder.py index eb74ff3..5228db4 100644 --- a/artemis/sdk/builders/agent_config_builder.py +++ b/artemis/sdk/builders/agent_config_builder.py @@ -88,7 +88,15 @@ class AgentConfigBuilder: self._committee_debate_rounds = agent_cfg.committee.debate_rounds self._disable_checker = not agent_cfg.checker.enabled self._checker_max_iterations = agent_cfg.checker.max_iterations - self._checker_max_chat_rounds = agent_cfg.checker.max_chat_rounds + self._disable_midway_checks = not agent_cfg.checker.midway_checks + self._disable_final_check = not agent_cfg.checker.final_check + self._final_check_max_attempts = agent_cfg.checker.final_check_max_attempts + self._checkpoint_max_repairs = agent_cfg.checker.checkpoint_max_repairs + self._max_concurrent_checkpoints = agent_cfg.checker.max_concurrent_checkpoints + self._checkpoint_timeout = agent_cfg.checker.checkpoint_timeout + self._settlement_timeout = agent_cfg.checker.settlement_timeout + self._assert_failure_policy = agent_cfg.checker.assert_failure_policy + self._disable_device_probes = not agent_cfg.checker.device_probes self._outputter = agent_cfg.outputter self._disable_outputter = not agent_cfg.outputter.enabled self._flash = agent_cfg.flash @@ -162,9 +170,7 @@ class AgentConfigBuilder: """Target a specific Android device by its ADB serial number.""" return self.for_device(DevicePlatform.ANDROID, device_serial) - def with_concurrency_mode( - self, mode: str - ) -> "AgentConfigBuilder": + def with_concurrency_mode(self, mode: str) -> "AgentConfigBuilder": """Configure concurrency mode: 'global' (1 task globally) or 'per_device' (1 task per device).""" self._concurrency_mode = str(mode).strip().lower() return self @@ -244,20 +250,62 @@ class AgentConfigBuilder: self, enabled: bool = True, max_iterations: int | None = None, - max_chat_rounds: int | None = None, + midway_checks: bool | None = None, + final_check: bool | None = None, + final_check_max_attempts: int | None = None, + checkpoint_max_repairs: int | None = None, + max_concurrent_checkpoints: int | None = None, + checkpoint_timeout: float | None = None, + settlement_timeout: float | None = None, + assert_failure_policy: str | None = None, + device_probes: bool | None = None, ) -> "AgentConfigBuilder": - """Configure Checker subgoal verification and rollback settings. + """Configure plan-driven checkpoint verification and the final review. Args: - enabled: Whether checker visual verification is enabled + enabled: Master switch (compat alias); False disables both gates max_iterations: Optional maximum iterations for Checker reasoning (1-50) - max_chat_rounds: Optional maximum debate rounds with Operator (1-10) + midway_checks: Whether plan-declared midway checkpoints run + final_check: Whether the exit final review runs + final_check_max_attempts: Max final-review attempts before a blocked exit + checkpoint_max_repairs: Per-checkpoint verify-fail repair quota + max_concurrent_checkpoints: Concurrency cap for checkpoint attempts + checkpoint_timeout: Per-attempt timeout in seconds + settlement_timeout: Aggregate exit-settlement timeout in seconds + assert_failure_policy: 'continue' or 'halt' on assert failures + device_probes: Whether Checker read-only device probes are registered """ self._disable_checker = not enabled if max_iterations is not None: self._checker_max_iterations = max_iterations - if max_chat_rounds is not None: - self._checker_max_chat_rounds = max_chat_rounds + if midway_checks is not None: + self._disable_midway_checks = not midway_checks + if final_check is not None: + self._disable_final_check = not final_check + if final_check_max_attempts is not None: + self._final_check_max_attempts = final_check_max_attempts + if checkpoint_max_repairs is not None: + self._checkpoint_max_repairs = checkpoint_max_repairs + if max_concurrent_checkpoints is not None: + self._max_concurrent_checkpoints = max_concurrent_checkpoints + if checkpoint_timeout is not None: + self._checkpoint_timeout = checkpoint_timeout + if settlement_timeout is not None: + self._settlement_timeout = settlement_timeout + if assert_failure_policy is not None: + self._assert_failure_policy = assert_failure_policy + if device_probes is not None: + self._disable_device_probes = not device_probes + return self + + def with_midway_checks(self, enabled: bool = True) -> "AgentConfigBuilder": + """Enable or disable plan-declared midway checkpoints.""" + self._disable_midway_checks = not enabled + return self + + def with_final_check(self, enabled: bool = True) -> "AgentConfigBuilder": + """Enable or disable the exit final review.""" + self._disable_final_check = not enabled return self def with_disable_planner_validation(self, disable: bool = True) -> "AgentConfigBuilder": @@ -521,8 +569,16 @@ class AgentConfigBuilder: video_recording_tools_enabled=self._video_recording_tools_enabled, force_web_accessibility=self._force_web_accessibility, disable_checker=self._disable_checker, + disable_midway_checks=self._disable_midway_checks, + disable_final_check=self._disable_final_check, checker_max_iterations=self._checker_max_iterations, - checker_max_chat_rounds=self._checker_max_chat_rounds, + final_check_max_attempts=self._final_check_max_attempts, + checkpoint_max_repairs=self._checkpoint_max_repairs, + max_concurrent_checkpoints=self._max_concurrent_checkpoints, + checkpoint_timeout=self._checkpoint_timeout, + settlement_timeout=self._settlement_timeout, + assert_failure_policy=self._assert_failure_policy, + disable_device_probes=self._disable_device_probes, disable_planner_validation=self._disable_planner_validation, planner_validation_threshold=self._planner_validation_threshold, enable_committee=self._enable_committee, diff --git a/artemis/sdk/types/agent.py b/artemis/sdk/types/agent.py index dc5cedb..1faa125 100644 --- a/artemis/sdk/types/agent.py +++ b/artemis/sdk/types/agent.py @@ -112,8 +112,16 @@ class AgentConfig(BaseModel): video_recording_tools_enabled: bool = Field(default_factory=detect_video_tools_enabled) force_web_accessibility: bool = False disable_checker: bool = True + disable_midway_checks: bool = False + disable_final_check: bool = False checker_max_iterations: int = 20 - checker_max_chat_rounds: int = 4 + final_check_max_attempts: int = 3 + checkpoint_max_repairs: int = 2 + max_concurrent_checkpoints: int = 3 + checkpoint_timeout: float = 180.0 + settlement_timeout: float = 120.0 + assert_failure_policy: Literal["continue", "halt"] = "continue" + disable_device_probes: bool = False disable_planner_validation: bool = True planner_validation_threshold: float = 0.85 enable_committee: bool = False diff --git a/artemis/services/llm.py b/artemis/services/llm.py index dccd911..547552d 100644 --- a/artemis/services/llm.py +++ b/artemis/services/llm.py @@ -25,7 +25,6 @@ from dataclasses import replace import functools import logging from pathlib import Path -import random import re import sys import time @@ -33,6 +32,7 @@ from typing import Any, Literal, TypeVar, overload from uuid import uuid4 from langchain_core.language_models.chat_models import BaseChatModel +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage from artemis.config import ( PAUSE_FILE, @@ -46,7 +46,22 @@ from artemis.config import ( ) from artemis.context import ArtemisContext from artemis.data_engine.trace import CURRENT_TRACE_ID, DataEngineCallbackHandler +from artemis.llm.reliability import ( + CircuitBreaker, + FailureCategory, + LLMCallError, + LLMExhaustedError, + LLMPermanentError, + classify_failure, + retry_policy_for, +) from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider +from artemis.llm.structured import ( + ParseFailure, + StructuredOutputError, + content_to_text, + parse_structured, +) from artemis.utils.logger import get_logger # Logger for internal messages @@ -69,26 +84,32 @@ _ACTIVE_LLM_REQUEST: ContextVar[dict[str, Any] | None] = ContextVar( ) -def _provider_name_from_call(args: tuple[Any, ...]) -> str | None: - wrapper = args[0] if args else None - endpoint = getattr(wrapper, "endpoint", None) - provider = getattr(endpoint, "provider", None) - if provider is None: - return None - return str(getattr(provider, "value", provider)) - - -def _begin_llm_request(args: tuple[Any, ...]) -> Token: +def _begin_llm_request(provider: str | None) -> Token: return _ACTIVE_LLM_REQUEST.set( { "request_id": str(uuid4()), - "provider": _provider_name_from_call(args), + "provider": provider, "started_at": time.time(), "retries": [], } ) +# Whether the current async context has a configured fallback model waiting +# behind this call (set by with_fallback). When a fallback exists, exhausting +# retries hands over to it immediately instead of pausing the whole task. +_FALLBACK_AVAILABLE: ContextVar[bool] = ContextVar("llm_fallback_available", default=False) + +# Shared circuit breaker keyed by "provider:model". Transient failures open +# it; while open, concurrent calls wait out the cooldown instead of hammering +# a provider that is already melting down. +_ENDPOINT_BREAKER = CircuitBreaker(threshold=3, cooldown_seconds=30.0) + +# Endpoints observed to not support streaming. Detected once, loudly, then +# remembered so subsequent calls go straight to non-streaming invocation. +_NON_STREAMING_ENDPOINTS: set[str] = set() + + def _get_current_data_engine(): if _CURRENT_DATA_ENGINE is not None: return _CURRENT_DATA_ENGINE @@ -170,6 +191,52 @@ def _record_llm_retry( llm_logger.warning("Failed to publish LLM retry event: %s", publish_error) +def _record_llm_event(name: str, payload: dict[str, Any], *, status: str = "retrying") -> None: + """Persist and publish one degradation/lifecycle event on the LLM IO path. + + Every degradation (fallback, stream downgrade, stream reset, giving up) + must be observable: WARNING-level logging is handled by callers, this + records the structured trace + live event, best-effort. + """ + request = _ACTIVE_LLM_REQUEST.get() + engine = _get_current_data_engine() + if not engine or not getattr(engine, "current_session_id", None): + return + + full_payload = dict(payload) + if request is not None: + full_payload.setdefault("request_id", request.get("request_id")) + if request.get("provider"): + full_payload.setdefault("provider", request.get("provider")) + full_payload = {key: value for key, value in full_payload.items() if value is not None} + + step_id = getattr(engine, "current_step_id", None) + trace_id = None + try: + trace_id = engine.record_trace( + type="llm_call", + name=name, + payload=full_payload, + step_id=step_id, + parent_trace_id=CURRENT_TRACE_ID.get(), + status=status, + ) + except Exception as trace_error: + llm_logger.warning("Failed to persist LLM event trace %s: %s", name, trace_error) + try: + engine._publish( + name, + { + **full_payload, + "step_id": str(step_id) if step_id else None, + "trace_id": str(trace_id) if trace_id else None, + "timestamp": time.time(), + }, + ) + except Exception as publish_error: + llm_logger.warning("Failed to publish LLM event %s: %s", name, publish_error) + + class _ProviderRetryTelemetryHandler(logging.Handler): """Turn provider SDK retry log records into structured Artemis telemetry.""" @@ -217,8 +284,7 @@ class _ProviderRetryTelemetryHandler(logging.Handler): def _install_provider_retry_telemetry() -> None: provider_logger = logging.getLogger("google_genai._api_client") if any( - isinstance(handler, _ProviderRetryTelemetryHandler) - for handler in provider_logger.handlers + isinstance(handler, _ProviderRetryTelemetryHandler) for handler in provider_logger.handlers ): return provider_logger.addHandler(_ProviderRetryTelemetryHandler(level=logging.INFO)) @@ -285,126 +351,149 @@ def _handle_llm_pause_and_resume(last_error: Exception) -> Path: return pause_file -def robust_retry_async(func: Callable) -> Callable: - """Decorator to add robust retry and pause/resume logic to an async LLM call.""" +async def _wait_for_resume(pause_file: Path) -> bool: + """Wait for the pause file to be cleared. Returns False on deadline.""" + deadline = float(getattr(settings, "LLM_PAUSE_TIMEOUT_SECONDS", 0.0) or 0.0) + waited = 0.0 + while pause_file.exists(): + if deadline > 0 and waited >= deadline: + return False + await asyncio.sleep(1) + waited += 1 + return True - @functools.wraps(func) - async def wrapper(*args, **kwargs): - while True: - request_token = _begin_llm_request(args) - max_retries = 3 - retry_delay = 0.0 - last_error = None - for attempt in range(max_retries): + +async def _wait_for_breaker(key: str) -> None: + """Wait out an open circuit instead of hammering a melting provider.""" + waited = 0.0 + warned = False + while not _ENDPOINT_BREAKER.allow(key): + if not warned: + warned = True + llm_logger.warning(f"LLM circuit open for {key}; waiting for cooldown...") + step = min(max(_ENDPOINT_BREAKER.open_remaining(key), 0.5), 2.0) + await asyncio.sleep(step) + waited += step + if waited >= 120.0: + # Never deadlock behind a stuck half-open trial; proceed anyway. + break + + +async def _run_with_recovery[T]( + call_once: Callable[[], Awaitable[T]], + *, + provider: str | None = None, + endpoint_key: str | None = None, +) -> T: + """Execute one LLM call under the classified retry / pause / breaker policy. + + Failure handling is decided per category (see artemis.llm.reliability): + + - Retryable categories back off per policy; each retry is recorded for UI + transparency via _record_llm_retry. + - Non-retryable categories (auth, bad request) raise LLMPermanentError + immediately: retrying cannot help and pausing would hang the task. + - When retryable attempts are exhausted: if a fallback model is waiting + (with_fallback), raise LLMExhaustedError so it takes over immediately; + otherwise pause the task (bounded by settings.LLM_PAUSE_TIMEOUT_SECONDS) + and retry from scratch on resume. + """ + while True: + request_token = _begin_llm_request(provider) + last_error: Exception | None = None + last_failure = None + attempts: dict[FailureCategory, int] = {} + pause_file: Path | None = None + try: + while True: + if endpoint_key: + await _wait_for_breaker(endpoint_key) try: - result = await func(*args, **kwargs) - _ACTIVE_LLM_REQUEST.reset(request_token) - return result - except Exception as e: - last_error = e - err_str = str(e).lower() - if attempt == max_retries - 1: - break - if attempt > 0: - retry_delay = 0.5 * (2 ** (attempt - 1)) + random.uniform(0.1, 0.3) - if any( - err in err_str - for err in [ - "503", - "throttled", - "overloaded", - "429", - "quota", - "resource_exhausted", - ] - ): - retry_delay = max(retry_delay, 10.0) - if retry_delay > 0: - llm_logger.warning(f"LLM call failed, retrying in {retry_delay:.2f}s...") - - await asyncio.sleep(retry_delay) - except BaseException: - _ACTIVE_LLM_REQUEST.reset(request_token) + result = await call_once() + except (KeyboardInterrupt, asyncio.CancelledError): + raise + except LLMCallError: + # Already classified terminal by a nested recovery layer. raise - - try: - pause_file = _handle_llm_pause_and_resume(last_error) - finally: - _ACTIVE_LLM_REQUEST.reset(request_token) - while pause_file.exists(): - await asyncio.sleep(1) - - llm_logger.info("Resume signal received, retrying LLM call...") - - current_engine = _get_current_data_engine() - if current_engine: - current_engine._publish("task_resumed", {}) - - return wrapper - - -def robust_retry_astream(func: Callable) -> Callable: - """Decorator to add robust retry and pause/resume logic to an async stream LLM call.""" - - @functools.wraps(func) - async def wrapper(*args, **kwargs): - while True: - request_token = _begin_llm_request(args) - max_retries = 3 - retry_delay = 0.0 - last_error = None - for attempt in range(max_retries): - try: - gen = func(*args, **kwargs) - first = await gen.__anext__() - yield first - async for chunk in gen: - yield chunk - _ACTIVE_LLM_REQUEST.reset(request_token) - return except Exception as e: - last_error = e - err_str = str(e).lower() - if attempt == max_retries - 1: - break - if attempt > 0: - retry_delay = 0.5 * (2 ** (attempt - 1)) + random.uniform(0.1, 0.3) - if any( - err in err_str - for err in [ - "503", - "throttled", - "overloaded", - "429", - "quota", - "resource_exhausted", - ] - ): - retry_delay = max(retry_delay, 10.0) - if retry_delay > 0: + failure = classify_failure(e) + if failure.category is FailureCategory.CANCELLED: + raise + if endpoint_key: + _ENDPOINT_BREAKER.record_failure(endpoint_key, failure) + if not failure.retryable: llm_logger.warning( - f"LLM stream handshake throttled, retrying in {retry_delay:.2f}s..." + f"LLM call failed permanently ({failure.category.value}): {e}" ) + _record_llm_event( + "llm_gave_up", + { + "error": str(e)[:1000], + "category": failure.category.value, + "retryable": False, + }, + status="failed", + ) + raise LLMPermanentError(str(e), failure=failure, cause=e) from e + last_error, last_failure = e, failure + attempt = attempts.get(failure.category, 0) + 1 + attempts[failure.category] = attempt + policy = retry_policy_for(failure.category) + if attempt >= policy.max_attempts: + break + delay = policy.delay_for(attempt) + _record_llm_retry( + str(e), + delay, + attempt=attempt, + max_retries=policy.max_attempts, + provider=provider, + ) + llm_logger.warning( + f"LLM call failed ({failure.category.value}), retrying in {delay:.2f}s..." + ) + await asyncio.sleep(delay) + else: + if endpoint_key: + _ENDPOINT_BREAKER.record_success(endpoint_key) + return result - await asyncio.sleep(retry_delay) - except BaseException: - _ACTIVE_LLM_REQUEST.reset(request_token) - raise + # Retryable attempts exhausted. + if _FALLBACK_AVAILABLE.get(): + _record_llm_event( + "llm_gave_up", + { + "error": str(last_error)[:1000], + "category": last_failure.category.value, + "retryable": True, + "handover": "fallback", + }, + status="failed", + ) + raise LLMExhaustedError( + str(last_error), failure=last_failure, cause=last_error + ) from last_error + pause_file = _handle_llm_pause_and_resume(last_error) + finally: + _ACTIVE_LLM_REQUEST.reset(request_token) + if not await _wait_for_resume(pause_file): + llm_logger.error( + "LLM pause deadline" + f" ({settings.LLM_PAUSE_TIMEOUT_SECONDS:.0f}s) exceeded; giving up." + ) try: - pause_file = _handle_llm_pause_and_resume(last_error) - finally: - _ACTIVE_LLM_REQUEST.reset(request_token) - while pause_file.exists(): - await asyncio.sleep(1) + pause_file.unlink(missing_ok=True) + except OSError: + pass + raise LLMExhaustedError( + str(last_error), failure=last_failure, cause=last_error + ) from last_error - llm_logger.info("Resume signal received, retrying LLM stream...") - - current_engine = _get_current_data_engine() - if current_engine: - current_engine._publish("task_resumed", {}) - - return wrapper + llm_logger.info("Resume signal received, retrying LLM call...") + current_engine = _get_current_data_engine() + if current_engine: + current_engine._publish("task_resumed", {}) def _inject_parent_trace_id(trace_id, *args, **kwargs): @@ -524,49 +613,229 @@ class RobustChatModelWrapper: self.base_model.__ror__(other), self.ctx, endpoint=self.endpoint ) - @robust_retry_async + def _provider_value(self) -> str | None: + provider = getattr(self.endpoint, "provider", None) + if provider is None: + return None + return str(getattr(provider, "value", provider)) + + def _endpoint_key(self) -> str: + if self.endpoint is not None: + return f"{self._provider_value()}:{self.endpoint.model_name}" + return type(self.base_model).__name__ + + def _traced_call(self, args: tuple, kwargs: dict) -> tuple[tuple, dict, Any]: + trace_id = None + if self.ctx and self.ctx.data_engine: + trace_id = CURRENT_TRACE_ID.get() + if trace_id: + args, kwargs = _inject_parent_trace_id(trace_id, *args, **kwargs) + return args, kwargs, trace_id + async def ainvoke(self, *args, **kwargs): - trace_id = None - if self.ctx and self.ctx.data_engine: - trace_id = CURRENT_TRACE_ID.get() - if trace_id: - args, kwargs = _inject_parent_trace_id(trace_id, *args, **kwargs) - return await self.base_model.ainvoke(*args, **kwargs) + args, kwargs, _ = self._traced_call(args, kwargs) + return await _run_with_recovery( + functools.partial(self.base_model.ainvoke, *args, **kwargs), + provider=self._provider_value(), + endpoint_key=self._endpoint_key(), + ) - @robust_retry_astream - async def astream(self, *args, **kwargs): - trace_id = None - if self.ctx and self.ctx.data_engine: - trace_id = CURRENT_TRACE_ID.get() + async def complete(self, *args, **kwargs): + """Single completion entry point: returns the full final message. - if trace_id: - args, kwargs = _inject_parent_trace_id(trace_id, *args, **kwargs) + Streaming is a transport detail handled internally — provider chunks + are forwarded to the live UI as deltas and accumulated here, so + downstream consumers only ever see one complete message (or a typed + LLMCallError). A mid-stream failure discards the partial output, + signals the UI to drop it, and retries the whole call per policy; + partial or duplicated chunks can never reach message history. + """ + args, kwargs, _ = self._traced_call(args, kwargs) + emit_deltas = bool(self.ctx and self.ctx.data_engine) + return await _run_with_recovery( + functools.partial(self._complete_attempt, emit_deltas, *args, **kwargs), + provider=self._provider_value(), + endpoint_key=self._endpoint_key(), + ) + + def _emit_stream_delta(self, stream_exec_id, chunk) -> None: + content = getattr(chunk, "content", None) + if not content: + return + text_to_stream = "" + thinking_to_stream = "" + if isinstance(content, str): + text_to_stream = content + elif isinstance(content, list): + for item in content: + if isinstance(item, dict): + if item.get("type") == "text": + text_to_stream += item.get("text", "") + elif item.get("type") == "thinking": + thinking_to_stream += item.get("thinking", "") + if text_to_stream: + self.ctx.data_engine.stream_output(stream_exec_id, text_to_stream, is_thinking=False) + if thinking_to_stream: + self.ctx.data_engine.stream_output(stream_exec_id, thinking_to_stream, is_thinking=True) + + async def _complete_attempt(self, emit_deltas: bool, *args, **kwargs): + endpoint_key = self._endpoint_key() + if endpoint_key in _NON_STREAMING_ENDPOINTS: + return await self.base_model.ainvoke(*args, **kwargs) stream_exec_id = uuid4() + full_response = None + try: + async for chunk in self.base_model.astream(*args, **kwargs): + if full_response is None: + full_response = chunk + else: + full_response = full_response + chunk + if emit_deltas: + self._emit_stream_delta(stream_exec_id, chunk) + except (KeyboardInterrupt, asyncio.CancelledError): + raise + except Exception as stream_error: + if full_response is not None: + # Mid-stream failure: the partial output must never reach + # message history. Tell the UI to discard it, then let the + # recovery policy retry the whole call with a fresh stream id. + llm_logger.warning( + f"LLM stream broke mid-response ({stream_error}); discarding partial output." + ) + _record_llm_event( + "llm_stream_reset", + { + "stream_exec_id": str(stream_exec_id), + "error": str(stream_error)[:500], + "category": classify_failure(stream_error).category.value, + }, + ) + raise + if _is_stream_unsupported_error(stream_error): + _NON_STREAMING_ENDPOINTS.add(endpoint_key) + llm_logger.warning( + f"Endpoint {endpoint_key} does not support streaming" + f" ({stream_error}); switching to non-streaming calls." + ) + _record_llm_event( + "llm_stream_downgrade", + {"endpoint": endpoint_key, "error": str(stream_error)[:500]}, + ) + return await self.base_model.ainvoke(*args, **kwargs) + raise + if full_response is None: + llm_logger.warning("LLM stream yielded no chunks; using non-streaming call.") + return await self.base_model.ainvoke(*args, **kwargs) + return full_response - async for chunk in self.base_model.astream(*args, **kwargs): - if trace_id and self.ctx and self.ctx.data_engine and chunk.content: - text_to_stream = "" - thinking_to_stream = "" - if isinstance(chunk.content, str): - text_to_stream = chunk.content - elif isinstance(chunk.content, list): - for item in chunk.content: - if isinstance(item, dict): - if item.get("type") == "text": - text_to_stream += item.get("text", "") - elif item.get("type") == "thinking": - thinking_to_stream += item.get("thinking", "") + async def astream(self, *args, **kwargs): + """Deprecated compatibility shim: yields exactly one final message. - if text_to_stream: - self.ctx.data_engine.stream_output( - stream_exec_id, text_to_stream, is_thinking=False - ) - if thinking_to_stream: - self.ctx.data_engine.stream_output( - stream_exec_id, thinking_to_stream, is_thinking=True - ) - yield chunk + The old chunk-level astream retry could re-deliver already-yielded + chunks after a mid-stream failure, corrupting accumulated message + history. Streaming now happens inside complete(); live token deltas + still reach the UI through the data engine. + """ + yield await self.complete(*args, **kwargs) + + +def _is_stream_unsupported_error(error: BaseException) -> bool: + """Detect 'this endpoint/model cannot stream' as opposed to a transient failure.""" + if isinstance(error, (NotImplementedError, AttributeError, TypeError)): + return True + message = str(error).lower() + return "stream" in message and any( + marker in message + for marker in ("not support", "unsupported", "not implemented", "not available") + ) + + +async def acomplete(llm, *args, **kwargs): + """Get one complete response from any chat model object. + + This is the single call-shape agents should use. For gateway-managed + models (RobustChatModelWrapper) it delegates to complete(), which owns + streaming, classified retries, and telemetry. For raw models and test + doubles it preserves the legacy accumulate-stream-else-ainvoke shape. + """ + if isinstance(llm, RobustChatModelWrapper): + return await llm.complete(*args, **kwargs) + + try: + full_response = None + async for chunk in llm.astream(*args, **kwargs): + if full_response is None: + full_response = chunk + else: + full_response = full_response + chunk + if full_response is not None: + return full_response + except (KeyboardInterrupt, asyncio.CancelledError): + raise + except Exception as stream_error: + llm_logger.warning( + f"astream unavailable on {type(llm).__name__} ({stream_error}); using ainvoke." + ) + return await llm.ainvoke(*args, **kwargs) + + +async def acomplete_structured( + llm, + messages: list, + *, + schema=None, + correction_attempts: int = 1, +): + """Complete and parse a JSON response, with one corrective re-ask. + + On a parse failure the model is shown its own output's parse error and + asked to re-emit valid JSON (up to ``correction_attempts`` times) — giving + the model a chance to see its mistake instead of feeding garbage + downstream. If it still fails, raises StructuredOutputError (never + returns raw text masquerading as parsed data). Each repair round emits an + llm_parse_repair telemetry event. + """ + response = await acomplete(llm, messages) + text = content_to_text(getattr(response, "content", "")) + parsed = parse_structured(text, schema=schema) + attempt = 0 + while isinstance(parsed, ParseFailure) and attempt < correction_attempts: + attempt += 1 + llm_logger.warning( + f"Structured output parse failed ({parsed.error});" + f" asking the model to correct itself (attempt {attempt})..." + ) + _record_llm_event( + "llm_parse_repair", + {"error": parsed.error[:500], "attempt": attempt}, + ) + correction_messages = [ + *messages, + response if isinstance(response, BaseMessage) else AIMessage(content=text), + HumanMessage( + content=( + "Your previous reply could not be parsed as the required" + f" JSON ({parsed.error}). Re-emit ONLY the corrected JSON" + " payload, with no surrounding prose or code fences." + ) + ), + ] + response = await acomplete(llm, correction_messages) + text = content_to_text(getattr(response, "content", "")) + parsed = parse_structured(text, schema=schema) + if isinstance(parsed, ParseFailure): + _record_llm_event( + "llm_gave_up", + { + "error": f"structured output unparseable: {parsed.error[:400]}", + "category": "bad_request", + "retryable": False, + }, + status="failed", + ) + raise StructuredOutputError(parsed) + return parsed async def invoke_llm_with_timeout_message[T]( @@ -578,9 +847,7 @@ async def invoke_llm_with_timeout_message[T]( llm_task = asyncio.create_task(llm_call) waiter_task = asyncio.create_task(asyncio.sleep(timeout_seconds)) try: - done, _ = await asyncio.wait( - {llm_task, waiter_task}, return_when=asyncio.FIRST_COMPLETED - ) + done, _ = await asyncio.wait({llm_task, waiter_task}, return_when=asyncio.FIRST_COMPLETED) if llm_task in done: return llm_task.result() @@ -846,12 +1113,55 @@ async def with_fallback[T]( fallback_call: Callable[[], Awaitable[T]], none_should_fallback: bool = True, ) -> T: + """Run main_call, switching to fallback_call only when it can actually help. + + Falling back is an explicit, observed decision: the failure is classified + and only categories where a different endpoint might succeed trigger the + fallback (a bad request would just hide the bug inside a weaker model's + output). Every switch is logged at WARNING and recorded as an + llm_fallback telemetry event. While main_call runs, the recovery layer + knows a fallback exists and hands over immediately on retry exhaustion + instead of pausing the task. + """ + + def _switch(reason: str, category: str | None, error: str | None) -> None: + llm_logger.warning( + f"❗ Main LLM inference failed ({reason}" + f"{f': {error}' if error else ''}). Falling back..." + ) + _record_llm_event( + "llm_fallback", + { + "reason": reason, + "category": category, + "error": error[:500] if error else None, + }, + ) + + # The contextvar must be reset BEFORE fallback_call runs: the fallback has + # no further fallback behind it, so its own exhaustion should pause. + fallback_token = _FALLBACK_AVAILABLE.set(True) try: result = await main_call() - if result is None and none_should_fallback: - llm_logger.warning("Main LLM inference returned None. Falling back...") - return await fallback_call() - return result - except Exception as e: - llm_logger.warning(f"❗ Main LLM inference failed: {e}. Falling back...") + except (KeyboardInterrupt, asyncio.CancelledError): + raise + except LLMCallError as e: + _FALLBACK_AVAILABLE.reset(fallback_token) + if not e.failure.should_fallback: + raise + _switch("terminal_error", e.failure.category.value, str(e)) return await fallback_call() + except Exception as e: + _FALLBACK_AVAILABLE.reset(fallback_token) + failure = classify_failure(e) + if failure.category is FailureCategory.CANCELLED or not failure.should_fallback: + raise + _switch("error", failure.category.value, str(e)) + return await fallback_call() + else: + _FALLBACK_AVAILABLE.reset(fallback_token) + + if result is None and none_should_fallback: + _switch("empty_result", None, None) + return await fallback_call() + return result diff --git a/artemis/tools/actions/__init__.py b/artemis/tools/actions/__init__.py deleted file mode 100644 index f6db1a6..0000000 --- a/artemis/tools/actions/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Universal device action tools.""" - -from artemis.tools.actions.device_actions import ( - ClickArgs, - InputTextArgs, - LongPressArgs, - PressKeyArgs, - SwipeArgs, - WaitForDelayArgs, - click, - input_text, - long_press, - press_key, - swipe, - wait_for_delay, -) - -__all__ = [ - "click", - "long_press", - "input_text", - "swipe", - "press_key", - "wait_for_delay", - "ClickArgs", - "LongPressArgs", - "InputTextArgs", - "SwipeArgs", - "PressKeyArgs", - "WaitForDelayArgs", -] diff --git a/artemis/tools/actions/device_actions.py b/artemis/tools/actions/device_actions.py deleted file mode 100644 index 2595ef8..0000000 --- a/artemis/tools/actions/device_actions.py +++ /dev/null @@ -1,250 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Universal device action tools implemented on top of BaseDeviceDriver.""" - -import asyncio -from typing import Any, Literal -from artemis.drivers.base import BaseDeviceDriver -from artemis.tools.base import artemis_tool -from artemis.utils.coordinates import ( - compute_smart_swipe_coordinates, - parse_swipe_parameters, -) -from artemis.utils.logger import get_logger -from pydantic import BaseModel, Field - -logger = get_logger(__name__) - - -# Schemas -class ClickArgs(BaseModel): - target: list[int] = Field( - ..., description="Click target normalized coordinates [x, y] in 0-1000 scale." - ) - - -class LongPressArgs(BaseModel): - target: list[int] = Field( - ..., description="Long press target normalized coordinates [x, y] in 0-1000 scale." - ) - duration: int = Field(1000, description="Long press duration in milliseconds (default 1000).") - - -class InputTextArgs(BaseModel): - text: str = Field(..., description="The text content to input.") - target: list[int] = Field( - ..., description="Input target field normalized coordinates [x, y] in 0-1000 scale." - ) - clear_exist: bool = Field( - True, description="Whether to clear existing text in the input box before typing." - ) - - -class SwipeArgs(BaseModel): - direction: Literal["up", "down", "left", "right"] | None = Field( - None, - description=( - "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below)," - " 'down' (drags top-to-bottom, scrolling up to reveal content above)," - " 'left' (drags right-to-left, scrolling right)," - " 'right' (drags left-to-right, scrolling left)." - ), - ) - start: list[int] | None = Field( - None, - description=( - "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise," - " local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." - ), - ) - end: list[int] | None = Field( - None, - description=( - "End normalized coordinates [end_x, end_y] in 0-1000 scale for precise," - " local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." - ), - ) - action: Literal["up", "down", "left", "right"] | list[int] | None = Field( - None, - description=( - "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right')" - " OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." - ), - ) - duration: int = Field( - 800, - description=( - "Swipe/drag duration in milliseconds (default 800). For drag-and-drop, " - "list reordering, or sliding/adjusting sliders (e.g. volume, brightness, SeekBars), " - "set duration >= 1000 (e.g. 1500)." - ), - ) - - -class PressKeyArgs(BaseModel): - key: str = Field( - ..., - description="Key name to press (e.g. 'home', 'back', 'enter', 'delete', 'power').", - ) - - -class WaitForDelayArgs(BaseModel): - seconds: float = Field( - ..., - description=( - "Duration in seconds to wait (e.g., 2 for 2s, 60 for 1 min, 180 for 3 mins)." - " Use whenever time needs to elapse for animations, loading, or scheduled delays." - ), - ) - - -# Universal Tool Definitions -@artemis_tool( - name="click", - description="[ACTION] Clicks on the specified screen coordinate [x, y] in normalized 0-1000 scale.", - args_schema=ClickArgs, - category="action", -) -async def click(target: list[int], driver: BaseDeviceDriver) -> str: - width, height = driver.screen_size - abs_x = int(target[0] * width / 1000.0) - abs_y = int(target[1] * height / 1000.0) - success = await driver.tap(abs_x, abs_y) - return ( - f"Click executed at normalized {target} (pixels: {abs_x}, {abs_y})" - if success - else f"Click failed at {target}" - ) - - -@artemis_tool( - name="long_press", - description="[ACTION] Long presses on the specified screen coordinate [x, y] in normalized 0-1000 scale.", - args_schema=LongPressArgs, - category="action", -) -async def long_press( - target: list[int], duration: int = 1000, driver: BaseDeviceDriver = None -) -> str: - width, height = driver.screen_size - abs_x = int(target[0] * width / 1000.0) - abs_y = int(target[1] * height / 1000.0) - success = await driver.long_press(abs_x, abs_y, duration_ms=duration) - return ( - f"Long press executed at normalized {target} for {duration}ms" - if success - else f"Long press failed at {target}" - ) - - -@artemis_tool( - name="input_text", - description="[ACTION] Clicks an input field target [x, y] and types text into it.", - args_schema=InputTextArgs, - category="action", -) -async def input_text( - text: str, target: list[int], clear_exist: bool = True, driver: BaseDeviceDriver = None -) -> str: - # First tap to focus - width, height = driver.screen_size - abs_x = int(target[0] * width / 1000.0) - abs_y = int(target[1] * height / 1000.0) - await driver.tap(abs_x, abs_y) - await asyncio.sleep(0.2) - - # Then input text - success = await driver.input_text(text, clear_existing=clear_exist) - return ( - f"Input text '{text}' into target {target}" - if success - else f"Failed to input text into {target}" - ) - - -@artemis_tool( - name="swipe", - description=( - "[ACTION] Performs smart directional swipe for browsing, or precise coordinate drag/slider gestures." - ), - args_schema=SwipeArgs, - category="action", -) -async def swipe( - action: Any = None, - direction: str | None = None, - start: list[int] | None = None, - end: list[int] | None = None, - duration: int = 800, - driver: BaseDeviceDriver = None, - **kwargs: Any, -) -> str: - params = dict(kwargs) - if action is not None: - params["action"] = action - if direction is not None: - params["direction"] = direction - if start is not None: - params["start"] = start - if end is not None: - params["end"] = end - params["duration"] = duration - - kind, target, dur = parse_swipe_parameters(params, default_duration=duration) - - if kind == "direction" and isinstance(target, str): - width, height = driver.screen_size - sx, sy, ex, ey, smart_dur = compute_smart_swipe_coordinates( - direction=target, - target=params.get("target"), - width=width, - height=height, - duration=dur, - ) - success = await driver.swipe(sx, sy, ex, ey, duration_ms=smart_dur) - return f"Swiped {target}" if success else f"Swipe {target} failed" - elif kind == "coords" and isinstance(target, list) and len(target) == 4: - width, height = driver.screen_size - start_x = int(target[0] * width / 1000.0) - start_y = int(target[1] * height / 1000.0) - end_x = int(target[2] * width / 1000.0) - end_y = int(target[3] * height / 1000.0) - success = await driver.swipe(start_x, start_y, end_x, end_y, duration_ms=dur) - return ( - f"Swiped from {target[:2]} to {target[2:]}" if success else "Swipe coordinates failed" - ) - return f"Invalid swipe action parameter: {action or direction or (start, end)}" - - -@artemis_tool( - name="press_key", - description="[ACTION] Presses a hardware or system key on the device ('home', 'back', 'enter', 'delete', 'power').", - args_schema=PressKeyArgs, - category="action", -) -async def press_key(key: str, driver: BaseDeviceDriver) -> str: - success = await driver.press_key(key) - return f"Key '{key}' pressed" if success else f"Failed to press key '{key}'" - - -@artemis_tool( - name="wait_for_delay", - description="[ACTION] Pauses execution for a specified number of seconds to allow UI loading or transition.", - args_schema=WaitForDelayArgs, - category="action", -) -async def wait_for_delay(seconds: float, driver: BaseDeviceDriver = None) -> str: - await asyncio.sleep(seconds) - return f"Waited for {seconds} seconds." diff --git a/artemis/tools/committee_tool.py b/artemis/tools/committee_tool.py index 2d55068..fb73b61 100644 --- a/artemis/tools/committee_tool.py +++ b/artemis/tools/committee_tool.py @@ -198,13 +198,6 @@ async def _execute_committee( logger.error(f"Failed to build plan and history in committee: {e}") history_summary += f"Error building plan and history: {e}" - # Append immediate failure reason if present in state - replan_reason = ( - getattr(state, "operator_replan_reason", None) if state is not None else None - ) - if replan_reason: - history_summary += f"\n\n### Current Failure Reason\n{replan_reason}\n" - # Read failed plans history to ensure the committee avoids # repeating failed strategies failed_plans_history = "" diff --git a/artemis/tools/mobile/exec_tools.py b/artemis/tools/mobile/exec_tools.py index a304c6d..1c06858 100644 --- a/artemis/tools/mobile/exec_tools.py +++ b/artemis/tools/mobile/exec_tools.py @@ -17,10 +17,8 @@ from typing import Any, Literal from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool -from langgraph.types import Command from pydantic import BaseModel, Field -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController from artemis.data_engine.trace import trace_langchain_tool @@ -206,19 +204,12 @@ class ClickTool(ArtemisTool): success = False outcome = f"Error during click: {e}" - if st and callable(getattr(st, "asanitize_update", None)): - tool_message = ToolMessage( + if st is not None: + return ToolMessage( tool_call_id=tcid or "", content=outcome, status="success" if success else "error", ) - return Command( - update=await st.asanitize_update( - ctx=ctx, - update={VALIDATOR_MESSAGES_KEY: [tool_message]}, - agent="validator", - ) - ) return outcome @@ -306,19 +297,12 @@ class LongPressTool(ArtemisTool): success = False outcome = f"Error during long press: {e}" - if st and callable(getattr(st, "asanitize_update", None)): - tool_message = ToolMessage( + if st is not None: + return ToolMessage( tool_call_id=tcid or "", content=outcome, status="success" if success else "error", ) - return Command( - update=await st.asanitize_update( - ctx=ctx, - update={VALIDATOR_MESSAGES_KEY: [tool_message]}, - agent="validator", - ) - ) return outcome @@ -414,19 +398,12 @@ class InputTextTool(ArtemisTool): success = False outcome = f"Error during input text: {e}" - if st and callable(getattr(st, "asanitize_update", None)): - tool_message = ToolMessage( + if st is not None: + return ToolMessage( tool_call_id=tcid or "", content=outcome, status="success" if success else "error", ) - return Command( - update=await st.asanitize_update( - ctx=ctx, - update={VALIDATOR_MESSAGES_KEY: [tool_message]}, - agent="validator", - ) - ) return outcome @@ -545,7 +522,7 @@ class SwipeTool(ArtemisTool): if kind == "direction" and isinstance(target, str): indexed_elems = getattr(state, "indexed_elements", None) if state else None - ui_hier = getattr(state, "ui_tree", None) if state else None + ui_hier = getattr(state, "latest_ui_hierarchy", None) if state else None start_x, start_y, end_x, end_y, smart_dur = compute_smart_swipe_coordinates( direction=target, target=params.get("target"), @@ -575,19 +552,12 @@ class SwipeTool(ArtemisTool): success = False outcome = f"Error during swipe: {e}" - if st and callable(getattr(st, "asanitize_update", None)): - tool_message = ToolMessage( + if st is not None: + return ToolMessage( tool_call_id=tcid or "", content=outcome, status="success" if success else "error", ) - return Command( - update=await st.asanitize_update( - ctx=ctx, - update={VALIDATOR_MESSAGES_KEY: [tool_message]}, - agent="validator", - ) - ) return outcome @@ -664,19 +634,12 @@ class PressKeyTool(ArtemisTool): success = False outcome = f"Error during press key: {e}" - if st and callable(getattr(st, "asanitize_update", None)): - tool_message = ToolMessage( + if st is not None: + return ToolMessage( tool_call_id=tcid or "", content=outcome, status="success" if success else "error", ) - return Command( - update=await st.asanitize_update( - ctx=ctx, - update={VALIDATOR_MESSAGES_KEY: [tool_message]}, - agent="validator", - ) - ) return outcome diff --git a/artemis/tools/mobile/launch_app.py b/artemis/tools/mobile/launch_app.py index 1f0db1c..999de34 100644 --- a/artemis/tools/mobile/launch_app.py +++ b/artemis/tools/mobile/launch_app.py @@ -16,11 +16,9 @@ from typing import Any from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool -from langgraph.types import Command from pydantic import BaseModel, Field from artemis.agents.hopper.hopper import HopperOutput, hopper -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.controllers.platform_specific_commands_controller import ( list_packages_async, @@ -176,23 +174,14 @@ class LaunchAppTool(ArtemisTool): outcome = f"Error during launch app: {e}" error_msg = str(e) - if st and callable(getattr(st, "asanitize_update", None)): + if st is not None: additional_kwargs = {} if success else ({"error": error_msg} if error_msg else {}) - tool_message = ToolMessage( + return ToolMessage( tool_call_id=tcid or "", content=outcome, additional_kwargs=additional_kwargs, status="success" if success else "error", ) - return Command( - update=await st.asanitize_update( - ctx=ctx, - update={ - VALIDATOR_MESSAGES_KEY: [tool_message], - }, - agent="validator", - ) - ) return outcome diff --git a/artemis/tools/mobile/ocr.py b/artemis/tools/mobile/ocr.py index c6af58d..866379c 100644 --- a/artemis/tools/mobile/ocr.py +++ b/artemis/tools/mobile/ocr.py @@ -19,10 +19,8 @@ from typing import Any from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool -from langgraph.types import Command from pydantic import BaseModel -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.data_engine.trace import trace_langchain_tool from artemis.drivers.base import BaseDeviceDriver @@ -169,8 +167,8 @@ class OcrRecognitionTool(ArtemisTool): has_failed = True error_message = f"Failed to perform OCR: {str(e)}" - if st and callable(getattr(st, "asanitize_update", None)): - tool_message = ToolMessage( + if st is not None: + return ToolMessage( tool_call_id=tcid or "", content=ocr_recognition_wrapper.on_failure_fn(error_message) if has_failed @@ -178,15 +176,6 @@ class OcrRecognitionTool(ArtemisTool): additional_kwargs={"error": error_message} if has_failed else {}, status="error" if has_failed else "success", ) - return Command( - update=await st.asanitize_update( - ctx=ctx, - update={ - VALIDATOR_MESSAGES_KEY: [tool_message], - }, - agent="validator", - ) - ) if has_failed: return ocr_recognition_wrapper.on_failure_fn(error_message) diff --git a/artemis/tools/object_detection_tool.py b/artemis/tools/object_detection_tool.py index 234364a..e0351c6 100644 --- a/artemis/tools/object_detection_tool.py +++ b/artemis/tools/object_detection_tool.py @@ -95,7 +95,7 @@ async def _run_object_detection_logic( if not target_image: error_msg = "No image path provided and no screenshot available in state." - if state and callable(getattr(state, "asanitize_update", None)): + if state is not None: return await _create_error_command(ctx, state, tool_call_id, error_msg, active_wrapper) return f"Object Detection failed: {error_msg}" @@ -107,12 +107,12 @@ async def _run_object_detection_logic( span.result = result output = json.dumps(result) if not isinstance(result, str) else result - if state and callable(getattr(state, "asanitize_update", None)): + if state is not None: return await _create_success_command(ctx, state, tool_call_id, output, active_wrapper) return output except Exception as e: logger.error(f"Object detection execution failed: {e}") - if state and callable(getattr(state, "asanitize_update", None)): + if state is not None: return await _create_error_command(ctx, state, tool_call_id, str(e), active_wrapper) return f"Object Detection failed: {e}" diff --git a/artemis/tools/scratchpad.py b/artemis/tools/scratchpad.py index dcc946c..35b65b0 100644 --- a/artemis/tools/scratchpad.py +++ b/artemis/tools/scratchpad.py @@ -16,10 +16,8 @@ from typing import Any from langchain_core.messages import ToolMessage from langchain_core.tools.base import BaseTool -from langgraph.types import Command from pydantic import BaseModel, Field -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.data_engine.trace import trace_langchain_tool from artemis.drivers.base import BaseDeviceDriver @@ -155,27 +153,13 @@ class SaveNoteTool(ArtemisTool): agent_outcome = f"Failed to save note {k}.md: {e}" status = "error" - if st and callable(getattr(st, "asanitize_update", None)): - tool_message = ToolMessage( + if st is not None: + return ToolMessage( tool_call_id=tcid, content=agent_outcome, status=status, ) - current_agent = "operator" - if hasattr(st, "current_agent") and st.current_agent: - current_agent = st.current_agent - - return Command( - update=await st.asanitize_update( - ctx=ctx, - update={ - VALIDATOR_MESSAGES_KEY: [tool_message], - }, - agent=current_agent, - ), - ) - return agent_outcome @@ -249,27 +233,13 @@ class AppendNoteTool(ArtemisTool): agent_outcome = f"Failed to append note {k}.md: {e}" status = "error" - if st and callable(getattr(st, "asanitize_update", None)): - tool_message = ToolMessage( + if st is not None: + return ToolMessage( tool_call_id=tcid, content=agent_outcome, status=status, ) - current_agent = "operator" - if hasattr(st, "current_agent") and st.current_agent: - current_agent = st.current_agent - - return Command( - update=await st.asanitize_update( - ctx=ctx, - update={ - VALIDATOR_MESSAGES_KEY: [tool_message], - }, - agent=current_agent, - ), - ) - return agent_outcome @@ -348,27 +318,13 @@ class UpdateNoteTool(ArtemisTool): agent_outcome = f"Failed to update note '{k}': {e}" status = "error" - if st and callable(getattr(st, "asanitize_update", None)): - tool_message = ToolMessage( + if st is not None: + return ToolMessage( tool_call_id=tcid, content=agent_outcome, status=status, ) - current_agent = "operator" - if hasattr(st, "current_agent") and st.current_agent: - current_agent = st.current_agent - - return Command( - update=await st.asanitize_update( - ctx=ctx, - update={ - VALIDATOR_MESSAGES_KEY: [tool_message], - }, - agent=current_agent, - ), - ) - return agent_outcome @@ -447,27 +403,13 @@ class ReadNoteTool(ArtemisTool): agent_outcome = f"Failed to read note '{k}': {e}" status = "error" - if st and callable(getattr(st, "asanitize_update", None)): - tool_message = ToolMessage( + if st is not None: + return ToolMessage( tool_call_id=tcid, content=agent_outcome, status=status, ) - current_agent = "operator" - if hasattr(st, "current_agent") and st.current_agent: - current_agent = st.current_agent - - return Command( - update=await st.asanitize_update( - ctx=ctx, - update={ - VALIDATOR_MESSAGES_KEY: [tool_message], - }, - agent=current_agent, - ), - ) - return agent_outcome @@ -523,27 +465,13 @@ class ListNotesTool(ArtemisTool): agent_outcome = f"Failed to list notes: {e}" status = "error" - if st and callable(getattr(st, "asanitize_update", None)): - tool_message = ToolMessage( + if st is not None: + return ToolMessage( tool_call_id=tcid, content=agent_outcome, status=status, ) - current_agent = "operator" - if hasattr(st, "current_agent") and st.current_agent: - current_agent = st.current_agent - - return Command( - update=await st.asanitize_update( - ctx=ctx, - update={ - VALIDATOR_MESSAGES_KEY: [tool_message], - }, - agent=current_agent, - ), - ) - return agent_outcome diff --git a/artemis/tools/tool_wrapper.py b/artemis/tools/tool_wrapper.py index b009f6e..39de65d 100644 --- a/artemis/tools/tool_wrapper.py +++ b/artemis/tools/tool_wrapper.py @@ -19,13 +19,12 @@ from typing import Annotated, Any, get_args, get_origin from unittest.mock import MagicMock, Mock import uuid +from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool from langchain_core.tools.base import InjectedToolCallId from langgraph.prebuilt import InjectedState -from langgraph.types import Command from pydantic import BaseModel -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.data_engine import engine as engine_mod from artemis.data_engine.trace import CURRENT_TRACE_ID, smart_serialize @@ -169,12 +168,7 @@ async def invoke_tool_with_injection( def get_tool_result_content(result: Any) -> Any: - """Extracts the content from a tool result, handling Command objects.""" - if isinstance(result, Command): - updates = result.update - if VALIDATOR_MESSAGES_KEY in updates: - msgs = updates[VALIDATOR_MESSAGES_KEY] - if msgs and hasattr(msgs[0], "content"): - return msgs[0].content - return result + """Extracts the content from a tool result, handling ToolMessage returns.""" + if isinstance(result, ToolMessage): + return result.content return result diff --git a/artemis/tools/video_tool.py b/artemis/tools/video_tool.py index d054b6e..eb56f37 100644 --- a/artemis/tools/video_tool.py +++ b/artemis/tools/video_tool.py @@ -23,11 +23,9 @@ from typing import Any, Literal from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool -from langgraph.types import Command from pydantic import BaseModel, Field from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.data_engine.trace import trace_langchain_tool from artemis.drivers.base import BaseDeviceDriver @@ -148,21 +146,12 @@ class VideoAnalyzerTool(ArtemisTool): agent_outcome = f"Error running video analyzer: {e}" status = "error" - if st and callable(getattr(st, "asanitize_update", None)): - tool_message = ToolMessage( + if st is not None: + return ToolMessage( tool_call_id=tcid or "", content=agent_outcome, status=status, ) - return Command( - update=await st.asanitize_update( - ctx=ctx, - update={ - VALIDATOR_MESSAGES_KEY: [tool_message], - }, - agent="validator", - ), - ) if status == "failed": return f"Video analysis failed: {agent_outcome}" diff --git a/artemis/utils/file.py b/artemis/utils/file.py index f313cbd..7b8d97d 100644 --- a/artemis/utils/file.py +++ b/artemis/utils/file.py @@ -13,11 +13,8 @@ # limitations under the License. import json -from pathlib import Path import re -import shutil from typing import IO -from unittest.mock import MagicMock def strip_json_comments(text: str) -> str: @@ -28,28 +25,3 @@ def strip_json_comments(text: str) -> str: def load_jsonc(file: IO) -> dict: return json.loads(strip_json_comments(file.read())) - - -def create_snapshot(src_dir: Path, snapshot_dir: Path): - """Creates a snapshot of a directory by copying it.""" - if ( - isinstance(src_dir, MagicMock) - or isinstance(snapshot_dir, MagicMock) - or "MagicMock" in str(src_dir) - or "MagicMock" in str(snapshot_dir) - ): - return - if not src_dir.exists(): - return - if snapshot_dir.exists(): - shutil.rmtree(snapshot_dir) - shutil.copytree(src_dir, snapshot_dir) - - -def restore_snapshot(snapshot_dir: Path, dst_dir: Path): - """Restores a directory from a snapshot.""" - if not snapshot_dir.exists(): - raise FileNotFoundError(f"Snapshot directory {snapshot_dir} does not exist.") - if dst_dir.exists(): - shutil.rmtree(dst_dir) - shutil.copytree(snapshot_dir, dst_dir) diff --git a/artemis/utils/plan_grammar.py b/artemis/utils/plan_grammar.py new file mode 100644 index 0000000..544a73b --- /dev/null +++ b/artemis/utils/plan_grammar.py @@ -0,0 +1,362 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Single source of truth for the task-plan grammar (the "machine channel"). + +``task_plan.md`` is a dual-channel document: + +- **Machine channel**: the checkbox micro-grammar parsed here (status chars, + indentation, ``[Loop]`` / ``[Loop:continuous]`` tags). Harness code may only + ever read this channel. +- **Semantic channel**: all remaining free text. Only LLM agents interpret it; + harness code must never branch on wording. + +Every deterministic decision the harness makes about the plan (termination, +loop protection, validation/checker triggers, hand-slip detection) derives +from the :class:`PlanSnapshot` produced by :func:`parse_plan`. The same +constants render :data:`PLAN_GRAMMAR_SPEC`, the grammar description injected +into agent prompts via the ``{{ plan_grammar }}`` template variable — so the +contract the agents are taught and the contract the code enforces cannot +drift apart. +""" + +from __future__ import annotations + +import hashlib +import re +from collections import Counter +from dataclasses import dataclass, field +from typing import Literal + +STATUS_PENDING = " " +STATUS_ACTIVE = "/" +STATUS_DONE = "x" +STATUS_BLOCKED = "!" +STATUS_CHARS = STATUS_PENDING + STATUS_ACTIVE + STATUS_DONE + STATUS_BLOCKED + +LOOP_TAG = "[Loop]" +CONTINUOUS_LOOP_TAG = "[Loop:continuous]" + +CHECKBOX_LINE_RE = re.compile( + r"^(?P\s*)-\s*\[(?P[" + re.escape(STATUS_CHARS) + r"])\]\s*(?P.*)$" +) +_LOOP_TAG_RE = re.compile(r"\[Loop(?::continuous)?\]") +_CONTINUOUS_TAG_RE = re.compile(re.escape(CONTINUOUS_LOOP_TAG)) + +#: Check lines are indented attachments under a top-level subgoal +#: (``- verify: ...`` / ``- assert: ...``) or task-level free items +#: (``- assert@end: ...`` at zero indent). The optional ``@end`` suffix moves +#: the judgment moment from the anchored subgoal's completion to task exit. +CHECK_LINE_RE = re.compile( + r"^(?P\s*)-\s*(?Pverify|assert)(?P@end)?\s*:\s*(?P\S.*?)\s*$" +) + + +def subgoal_hash(text: str) -> str: + """Stable identity hash for a subgoal text (used for per-subgoal artifacts).""" + return hashlib.md5(text.strip().encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class PlanItem: + """One checkbox line of the plan's machine channel.""" + + indent: int + status: str + text: str + line_no: int + + @property + def is_top_level(self) -> bool: + return self.indent == 0 + + @property + def is_pending(self) -> bool: + return self.status == STATUS_PENDING + + @property + def is_active(self) -> bool: + return self.status == STATUS_ACTIVE + + @property + def is_done(self) -> bool: + return self.status == STATUS_DONE + + @property + def is_loop(self) -> bool: + return bool(_LOOP_TAG_RE.search(self.text)) + + @property + def is_continuous(self) -> bool: + return bool(_CONTINUOUS_TAG_RE.search(self.text)) + + @property + def key(self) -> str: + return subgoal_hash(self.text) + + +@dataclass(frozen=True) +class CheckItem: + """One declared check line of the plan's machine channel. + + ``verify`` marks a completion acceptance criterion (a failure triggers + repair); ``assert`` marks a test assertion (a failure is recorded as a + legitimate test result, never repaired). + """ + + kind: Literal["verify", "assert"] + when: Literal["on_complete", "at_end"] + text: str + parent_key: str | None + + @property + def key(self) -> str: + """Stable identity of the check line (kind + when + text + anchor).""" + return subgoal_hash(f"{self.kind}|{self.when}|{self.parent_key}|{self.text}") + + +@dataclass(frozen=True) +class PlanSnapshot: + """Read-only derived view of a plan. Never persisted, never written back.""" + + items: tuple[PlanItem, ...] + check_items: tuple[CheckItem, ...] = field(default=()) + + @property + def top_level(self) -> tuple[PlanItem, ...]: + return tuple(i for i in self.items if i.is_top_level) + + @property + def has_top_level(self) -> bool: + return bool(self.top_level) + + @property + def all_top_level_done(self) -> bool: + top = self.top_level + return bool(top) and all(i.is_done for i in top) + + @property + def continuous_top_level(self) -> tuple[PlanItem, ...]: + return tuple(i for i in self.top_level if i.is_continuous) + + @property + def milestone_texts(self) -> tuple[str, ...]: + return tuple(i.text.strip() for i in self.top_level) + + def first_active(self) -> PlanItem | None: + for item in self.items: + if item.is_active: + return item + return None + + def last_active(self) -> PlanItem | None: + for item in reversed(self.items): + if item.is_active: + return item + return None + + def parent_of(self, item: PlanItem) -> PlanItem | None: + """Nearest top-level item above an indented item.""" + parent = None + for candidate in self.items: + if candidate.line_no >= item.line_no: + break + if candidate.is_top_level: + parent = candidate + return parent + + def check_items_of(self, item: PlanItem) -> tuple[CheckItem, ...]: + """Check lines anchored to the given top-level subgoal.""" + return tuple(ci for ci in self.check_items if ci.parent_key == item.key) + + @property + def task_level_check_items(self) -> tuple[CheckItem, ...]: + return tuple(ci for ci in self.check_items if ci.parent_key is None) + + @property + def all_check_items(self) -> tuple[CheckItem, ...]: + return self.check_items + + +def parse_plan(content: str | None) -> PlanSnapshot: + """Parses plan markdown into its machine channel. Non-checkbox lines are ignored. + + Check lines (``- verify:`` / ``- assert:``) are parsed into + :class:`CheckItem` entries; they never contribute to the checkbox item list + and therefore never influence milestone hashing, drift detection, or + completion triggers. + """ + items: list[PlanItem] = [] + checks: list[CheckItem] = [] + current_top: PlanItem | None = None + if content: + for line_no, line in enumerate(content.splitlines()): + match = CHECKBOX_LINE_RE.match(line) + if match: + item = PlanItem( + indent=len(match.group("indent")), + status=match.group("status"), + text=match.group("text").strip(), + line_no=line_no, + ) + items.append(item) + if item.is_top_level: + current_top = item + continue + check_match = CHECK_LINE_RE.match(line) + if check_match: + indented = bool(check_match.group("indent")) + at_end = bool(check_match.group("at_end")) + parent_key = current_top.key if (indented and current_top) else None + checks.append( + CheckItem( + kind=check_match.group("kind"), + when="at_end" if at_end else "on_complete", + text=check_match.group("text").strip(), + parent_key=parent_key, + ) + ) + return PlanSnapshot(items=tuple(items), check_items=tuple(checks)) + + +def milestones_changed(before: PlanSnapshot, after: PlanSnapshot) -> bool: + """True if the top-level milestone texts differ (status flips don't count).""" + return before.milestone_texts != after.milestone_texts + + +def _check_multiset(snapshot: PlanSnapshot) -> Counter: + return Counter((ci.kind, ci.when, ci.text, ci.parent_key) for ci in snapshot.check_items) + + +def check_items_changed(before: PlanSnapshot, after: PlanSnapshot) -> bool: + """True if the multiset of check lines differs between the two snapshots.""" + return _check_multiset(before) != _check_multiset(after) + + +def missing_check_items(before: PlanSnapshot, after: PlanSnapshot) -> list[CheckItem]: + """Check items present in ``before`` but absent from ``after`` (multiset diff). + + Additions in ``after`` are ignored — adding check lines is allowed; only + deletions/rewrites of existing ones are guarded. + """ + deficit = _check_multiset(before) - _check_multiset(after) + missing: list[CheckItem] = [] + for ci in before.check_items: + sig = (ci.kind, ci.when, ci.text, ci.parent_key) + if deficit.get(sig, 0) > 0: + deficit[sig] -= 1 + missing.append(ci) + return missing + + +def render_check_line(item: CheckItem, indent: int = 2) -> str: + suffix = "@end" if item.when == "at_end" else "" + prefix = " " * indent if item.parent_key is not None else "" + return f"{prefix}- {item.kind}{suffix}: {item.text}" + + +def restore_missing_check_items(content_before: str, content_after: str) -> str | None: + """Deterministic guard: merge check lines deleted/rewritten by a plan write + back into the new content. + + Returns the merged plan text, or ``None`` when nothing is missing. Lines + whose parent subgoal still exists are re-anchored directly under it; lines + whose parent subgoal was removed are converted to task-level ``@end`` items + appended at the end of the plan. + """ + before = parse_plan(content_before) + after = parse_plan(content_after) + missing = missing_check_items(before, after) + if not missing: + return None + + lines = (content_after or "").splitlines() + top_line_by_key: dict[str, int] = {} + for item in after.items: + if item.is_top_level: + top_line_by_key.setdefault(item.key, item.line_no) + + inserts: dict[int, list[str]] = {} + tail: list[str] = [] + for ci in missing: + if ci.parent_key is not None and ci.parent_key in top_line_by_key: + inserts.setdefault(top_line_by_key[ci.parent_key], []).append(render_check_line(ci)) + else: + orphan = CheckItem(kind=ci.kind, when="at_end", text=ci.text, parent_key=None) + tail.append(render_check_line(orphan)) + + merged: list[str] = [] + for line_no, line in enumerate(lines): + merged.append(line) + if line_no in inserts: + merged.extend(inserts[line_no]) + merged.extend(tail) + return "\n".join(merged) + ("\n" if (content_after or "").endswith("\n") else "") + + +def new_top_level_completions(before: PlanSnapshot, after: PlanSnapshot) -> list[str]: + """Top-level texts newly marked done, in plan order.""" + before_done = {i.text for i in before.top_level if i.is_done} + return [i.text for i in after.top_level if i.is_done and i.text not in before_done] + + +def unintended_milestone_edits(before: PlanSnapshot, after: PlanSnapshot) -> list[tuple[str, str]]: + """Hand-slip signature of a full-file rewrite: positional top-level pairs + whose status char is unchanged but whose text was reworded. + + Only defined when the milestone count is unchanged; structural changes + (add/remove) are a declared replan and are judged by planner validation + instead. + """ + top_before = before.top_level + top_after = after.top_level + if len(top_before) != len(top_after): + return [] + return [ + (b.text, a.text) + for b, a in zip(top_before, top_after) + if b.status == a.status and b.text != a.text + ] + + +_PLAN_GRAMMAR_BASE = f"""### Task Plan Grammar (machine-enforced contract) +Every checklist line MUST match `- [] `: +- Status characters: `[{STATUS_PENDING}]` pending, `[{STATUS_ACTIVE}]` in progress, `[{STATUS_DONE}]` completed, `[{STATUS_BLOCKED}]` blocked. +- Top-level lines (no indentation) are strategic milestones; their sub-tasks are indented by 2 spaces. All other text in the note is free-form context. +- `{LOOP_TAG}` tags a BOUNDED iterative milestone: declare its exit boundary (e.g., `(Exit: ; Interval: )`) and mark it `[{STATUS_DONE}]` only once that exit condition is verifiably met. +- `{CONTINUOUS_LOOP_TAG}` tags an UNBOUNDED continuous-monitoring milestone: it must stay `[{STATUS_ACTIVE}]` and can never be marked `[{STATUS_DONE}]`, deleted, or untagged by you — the system mechanically rejects such edits. Only an explicit external stop signal injected by the user unlocks its completion. +- The task terminates only when every top-level milestone is `[{STATUS_DONE}]`.""" + +_CHECK_GRAMMAR_EXTENSION = """### Check Line Grammar (declared verification standards) +A top-level milestone may carry indented check lines, and the plan may end with task-level check lines at zero indentation: +- ` - verify: ` — an acceptance criterion for its parent milestone. It is judged by an independent Checker at the moment the milestone is marked completed; a failed verify triggers a repair loop. +- ` - assert: ` — a test assertion. It is judged by the independent Checker and a failure is recorded verbatim as a legitimate test result; assertions are NEVER repaired, worked around, or satisfied by constructing state. +- `- verify@end: ...` / `- assert@end: ...` — the `@end` suffix defers judgment to task exit, using the final device state. Items without `@end` are judged with the evidence available when their anchored milestone completes. +- Capability boundary (declare honestly, never promise more): all checks are POST-HOC audits over recorded evidence. They detect and record violations after the fact; they CANNOT block an action from happening, so "B must not run before A is confirmed" ordering rules are verified retroactively, not enforced. Transient prompts that were never captured in the execution history cannot be recovered — prefer expressing expectations as persistently probeable state.""" + + +def render_plan_grammar_spec(include_checks: bool) -> str: + """Renders the grammar spec taught to agents. + + ``include_checks=False`` (both check gates disabled) returns exactly the + base grammar — zero context pollution from the checking feature. + """ + if include_checks: + return _PLAN_GRAMMAR_BASE + "\n\n" + _CHECK_GRAMMAR_EXTENSION + return _PLAN_GRAMMAR_BASE + + +#: Backward-compatible module constant: the base grammar without the check +#: extension. Prompt injection sites use :func:`render_plan_grammar_spec`. +PLAN_GRAMMAR_SPEC = _PLAN_GRAMMAR_BASE diff --git a/artemis/utils/task_tree.py b/artemis/utils/task_tree.py index 30e0e32..5d23df2 100644 --- a/artemis/utils/task_tree.py +++ b/artemis/utils/task_tree.py @@ -13,12 +13,13 @@ # limitations under the License. import ast -import hashlib import json from pathlib import Path import re from typing import Any +from artemis.utils.plan_grammar import parse_plan + def format_action_clean(action_obj) -> str: if not action_obj: @@ -952,53 +953,26 @@ def get_active_subgoal_hashes(task_plan: str) -> tuple[str, str | None]: return "default", None try: - lines = task_plan.split("\n") - active_line_idx = -1 - # Search from bottom to top to prioritize the most specific deeply nested active subgoal - for i in range(len(lines) - 1, -1, -1): - if lines[i].strip().startswith("- [/]"): - active_line_idx = i - break + snapshot = parse_plan(task_plan) - if active_line_idx != -1: - active_line = lines[active_line_idx] - active_text = active_line.strip()[5:].strip() - - # Check if it is indented (sub-subgoal) - if active_line.startswith(" "): - # Find parent (first non-indented line above it) - parent_text = "" - for i in range(active_line_idx - 1, -1, -1): - if not lines[i].startswith(" ") and lines[i].strip().startswith("- ["): - parent_text = lines[i].strip()[5:].strip() - break - - # Consolidate under the parent Level 1 subgoal - if parent_text: - parent_hash = hashlib.md5(parent_text.encode("utf-8")).hexdigest() - return parent_hash, None - - # If not indented, it is a top-level Level 1 subgoal - active_hash = hashlib.md5(active_text.encode("utf-8")).hexdigest() - return active_hash, None + # Bottom-most active item wins (most specific deeply nested subgoal) + active_item = snapshot.last_active() + if active_item is not None: + if not active_item.is_top_level: + parent = snapshot.parent_of(active_item) + if parent is not None: + return parent.key, None + return active_item.key, None # No active subgoal found. Check if all are completed. - top_level_lines = [line for line in lines if line.strip().startswith("- [")] - all_done = all(line.strip().startswith("- [x]") for line in top_level_lines) - if all_done: + if all(item.is_done for item in snapshot.top_level): return "default", None - # SAFE FALLBACK: Check if all subgoals are not started (all are "[ ]") - all_pending = all(line.strip().startswith("- [ ]") for line in top_level_lines) - if all_pending: - # If all are pending, we are at the start of the task. - # The active subgoal is guaranteed to be the first one. - for line in lines: - stripped = line.strip() - if stripped.startswith("- [ ]"): - first_pending_text = stripped[5:].strip() - first_hash = hashlib.md5(first_pending_text.encode("utf-8")).hexdigest() - return first_hash, None + # SAFE FALLBACK: if every item is untouched, the first pending one is active. + if all(item.is_pending for item in snapshot.top_level): + for item in snapshot.items: + if item.is_pending: + return item.key, None except Exception: pass @@ -1011,13 +985,7 @@ def get_completed_subgoal_hashes(task_plan: str) -> set[str]: if not task_plan: return set() - completed_hashes = set() - - pattern = re.compile(r"^-\s*\[x\]\s*(.*)$", re.MULTILINE) - for match in pattern.findall(task_plan): - completed_hashes.add(hashlib.md5(match.strip().encode("utf-8")).hexdigest()) - - return completed_hashes + return {item.key for item in parse_plan(task_plan).top_level if item.is_done} def get_all_subgoal_aliases(target_hash: str, base_dir: str | Path | None = None) -> set[str]: diff --git a/artemis/utils/verification.py b/artemis/utils/verification.py deleted file mode 100644 index 3b50718..0000000 --- a/artemis/utils/verification.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -from pathlib import Path -from artemis.utils.logger import get_logger -from artemis.utils.notes import get_notes_dir - -logger = get_logger(__name__) - - -def get_verification_chat_path(base_dir: str | Path, subgoal_hash: str) -> Path: - """Gets the path to the verification chat file for a given subgoal.""" - notes_dir = get_notes_dir(base_dir) - return notes_dir / f"verification_chat_{subgoal_hash}.json" - - -def read_verification_chat(chat_path: Path) -> list[dict]: - """Reads the verification chat history from the given path.""" - if not chat_path.exists(): - return [] - try: - return json.loads(chat_path.read_text(encoding="utf-8")) - except Exception as e: - logger.error(f"Failed to load verification chat from {chat_path}: {e}") - return [] - - -def get_verification_chat_rounds(turns: list[dict]) -> tuple[int, int]: - """Calculates the maximum rounds for operator and checker from the turns.""" - max_op = max( - [t.get("round", 0) for t in turns if t.get("role") == "operator"], - default=0, - ) - max_chk = max( - [t.get("round", 0) for t in turns if t.get("role") == "checker"], - default=0, - ) - return max_op, max_chk - - -def append_verification_chat(chat_path: Path, role: str, content: str, round_num: int) -> bool: - """Appends a new turn to the verification chat file.""" - turns = read_verification_chat(chat_path) - turns.append({"role": role, "round": round_num, "content": content}) - try: - chat_path.write_text(json.dumps(turns, indent=2, ensure_ascii=False), encoding="utf-8") - return True - except Exception as e: - logger.error(f"Failed to save verification chat to {chat_path}: {e}") - return False diff --git a/config/artemis.jsonc b/config/artemis.jsonc index a505852..d2f368b 100644 --- a/config/artemis.jsonc +++ b/config/artemis.jsonc @@ -167,11 +167,19 @@ "enabled": false, "debate_rounds": 2 }, - // 🔎 Checker: Visual subgoal verification and automatic checkpoint rollback + // 🔎 Checker: Plan-declared checkpoint verification (optimistic async) + exit final review "checker": { - "enabled": false, + "enabled": false, // Master switch (compat alias): false disables both gates below + "midway_checks": true, // Plan-declared midway checkpoints (verify/assert on subgoal completion) + "final_check": true, // Exit final review against the user's original goal "max_iterations": 20, - "max_chat_rounds": 4 + "final_check_max_attempts": 3, + "checkpoint_max_repairs": 2, + "max_concurrent_checkpoints": 3, + "checkpoint_timeout": 180.0, + "settlement_timeout": 120.0, + "assert_failure_policy": "continue", // 'continue' or 'halt' on assert failures + "device_probes": true // Checker's enumerated read-only device probes }, // 🎬 Video Analyzer & Screen Recording: Continuous screen recording via scrcpy and temporal analysis "video_analyzer": { diff --git a/docs/plans/checker-two-gate-redesign.md b/docs/plans/checker-two-gate-redesign.md new file mode 100644 index 0000000..4526bba --- /dev/null +++ b/docs/plans/checker-two-gate-redesign.md @@ -0,0 +1,370 @@ +# Checker 全流程验证重构（计划驱动检查点 + 乐观异步 + 出口结算） + +> 交接文档：本计划自包含，执行者不需要任何前置对话上下文。 +> 日期：2026-08-30（v2.1 实施版）。文中行号以当日工作区为准，**执行前一律先按内容搜索确认，行号可能漂移**。 + +--- + +## 0. 工作区纪律（先读） + +- 本工作区可能有**多个并发 Claude 会话**在改代码，工作树里有大量未提交改动（部分不是你产生的）。 + - **严禁 `git stash`**（会卷走别人的半成品）。 + - 需要对比基线用 `git show HEAD:` 或临时 worktree，不要动工作区里已有的未提交修改。 +- 不要 commit/push，除非用户明确要求。 +- 每个 Work Package 完成后跑对应的单测（见 §8），基础设施类测试可用 `ARTEMIS_FAKE_LLM=1` 环境变量走假 LLM。 + +--- + +## 1. 背景与结论（已定，不要重新设计） + +### 1.1 旧中途 Checker 的病根（仍然成立，全部要摘除） + +1. **Checker 失败的 turn 不写入 DataEngine 历史**——`graph.py` `execution_check_node` 里 `if checker_success:` 才 `record_step`。 +2. **notes 目录全量快照回滚**——Operator 每轮前全量快照，checker 失败整体恢复。 +3. **Prompt 整体切换**——存在 checker feedback 时换 `troubleshooter_template` + verification chat 多轮辩论。 +4. **僵尸 checker**——`ctx.checker_task` 单槽位覆盖不 cancel，副作用在任务体内。 + +**关键认知：病根是"回滚过去、切换模板、副作用失控"，不是异步验证本身。** + +### 1.2 目标架构与能力边界 + +系统承担**测试任务**，短暂状态只能在发生时刻附近取证，所以中途检查点必须存在，以**乐观异步**方式运行。架构分四层： + +- **配置层**：`中途检查` 与 `结尾终审` 两个独立开关（§6）。用户关闭的检查一律尊重；**未执行的断言报告为 unchecked，绝不计为通过**。 +- **计划层**：Planner 把检查项声明进 task_plan.md（`- verify:` / `- assert:`，带时机，§4）。计划文件是唯一权威载体（锚定子目标、随重写存活、受守护覆盖）；但**相关 prompt 注入全部条件化**（§4.4）——检查全关时 Planner 不生成 check 行、语法说明不进任何 prompt，零上下文污染。 +- **能力层**：Checker 享有与 Operator 等价的观察能力（历史、截图、当前屏幕），但工具权限独立——只读检索 + 受约束探测，零副作用，只出裁决（§5.4）。 +- **控制层**：Graph 识别检查点、spawn 任务、收割裁决、执行副作用、出口结算（§5.1-5.3）。 + +**能力边界（必须如实声明，写进 planner prompt 与最终报告语义）**： +- 本期的中途检查是**事后核验**：它能在事后发现流程违规并留下证据，**不能阻止**违规发生。"必须确认 A 之后才允许执行 B"这类**强制前置关卡本期不支持**——Planner 遇到此类用户要求时不得静默接受：断言照常生成（保留用户原语义），但计划与报告中标注"顺序核验为事后判定"。 +- 异步检查只能核验**已被采集的证据**。从未被记录的瞬时提示无法找回——瞬态断言的证据来源是执行历史中的前后截图；Planner 应优先把预期表述为可持久探测的状态。 + +**三条铁律**： +1. Operator 的历史与上下文 **append-only**——不换 prompt、不回滚 notes、不丢 step、不开平行对话。 +2. 验证影响未来（打回状态、注入 finding），**不修改过去**。 +3. **放行、裁决、任务收尾三者分离**：fail-open 只影响放行决策，裁决原值（inconclusive）永远入账；assert 失败不修复但必须产生机器可读的测试失败结果并传导到 SDK 收尾（§5.5）。 + +**裁决 append-only**：每次检查有独立 `attempt_id`，所有裁决（含被 supersede 的）永久入账；断言首次失败不被覆盖；报告呈现每条断言 passed / failed / inconclusive / unchecked 与完整裁决序列。 + +### 1.3 横切原则：Prompt 与工具按配置装配（No Dead Instructions） + +本次全部改动遵守同一条装配纪律，**任何 agent 的 prompt 段与工具注册都是"配置 × 当前场景"的函数**： + +1. 配置关闭的能力，不得在任何 prompt 中被提及，其工具不得注册——不存在"指令还在、机制已关"的死指令； +2. 反向同样成立：prompt 中出现的每条规则，必须对应一个当前激活的机制（如"删除会被恢复"只在守护生效时可说）； +3. 装配点集中在代码里的组件列表/工具注册表（PromptComponent 列表、工具 dict 构造函数），**不在 prompt 模板内部堆条件文本**； +4. 场景也参与装配：同一 agent 的不同入口（如 checker 的中途/终审）按各自的证据纪律装配不同的工具与指南段。 + +落点：Planner（§4.1/§4.2 条件语法段与生成指令）、Operator（§4.4 说明段、§3.1 feedback 组件按内容渲染）、Checker（§5.4 两入口装配表）、SDK 返回结构（§5.5 test summary 仅在存在 check 项时出现）。 + +--- + +## 2. 代码地图（改动涉及的锚点） + +Graph 拓扑（`artemis/graph/graph.py` `get_graph`，~622）： + +``` +START → planner → convergence +perception → operator → execution_check ─(execute_decisions)→ validator → summarizer → convergence + └(review_subgoals)→ convergence +convergence ─(continue)→ perception + └(end)→ END +``` + +注意时序：Operator 在自己的 turn 内通过 note 工具写计划（含完成标记）；该 turn 的设备动作在**其后的 validator 节点**才执行；`execution_check_node` 的 `record_step` 记录的是本 turn 的观察（pre 截图）与决策。因此"子目标完成"的证据 = 截至本 turn step 的历史（上一 turn 的 post 截图 + 本 turn 的 pre 截图）。 + +| 锚点 | 位置（按内容搜索确认） | 处置 | +|---|---|---| +| `execution_check_node` | `graph.py` ~88-289 | 重构（§3.2、§5.1、§5.2） | +| checker 中途触发 ①（chat_path 分支） | ~100-132 | 删除 | +| checker 单槽消费段 | ~134-153 | 删除 | +| planner validation 消费段 + 拒绝提前 return | ~155-211，**return 在 ~190** | 保留机制；拒绝分支补 record_step（§3.2） | +| record_step（`if checker_success:` 门控） | ~213-250 | 无条件化（§3.2） | +| notes 快照 commit/rollback 段 | ~252-280 | 删除 | +| checker 中途触发 ②（`new_completions`） | `_process_plan_write` 末段 ~499-506 | 改为排队 pending checkpoint（§5.1） | +| planner validation 漂移检测 | `_process_plan_write` `milestones_changed`（~452） | 保留；旁加确定性 check 行守护（§4.3） | +| `convergence_gate` | ~625 | 扩为三值路由（§5.3） | +| notes 快照创建 / troubleshooter 分支 / verification chat 机制 | `operator.py` | 删除（§3） | +| `CheckerFeedbackPromptComponent` | `operator/prompts.py` ~231 | 改写为读 state 的 append 注入组件 | +| checker 本体 | `agents/checker/checker.py` `run_async_check`（~80）、`CheckerResult`（~68） | 重写（§5.4） | +| context 字段 | `context.py`：`checker_task`、`disable_checker=True`（~108）、`checker_max_chat_rounds`（~110）、`disable_planner_validation=True`（~111，**注意默认关**）、`task_plan_snapshot` | §3、§6 | +| state 字段 | `graph/state.py`：`checker_success`、`initial_goal` | 保留；新增字段见 §5.2/§5.5 | +| plan 语法 | `utils/plan_grammar.py`：`CHECKBOX_LINE_RE`（~49）、`parse_plan`（~149）、`PlanSnapshot`（~100）、`subgoal_hash`（~56）、`milestones_changed`（~167）、`new_top_level_completions`（~172）、`PLAN_GRAMMAR_SPEC`（~199） | 扩展（§4.1） | +| DataEngine | `engine.py`：`record_step`（~491，存 pre/post 截图）、`get_step_number`（~922）、`get_agent_friendly_steps`（~1045）、`end_session(status)`（~300）；`storage.py` `get_steps`（~651） | 只加只读检索辅助 + 收尾状态核对（§5.5） | +| SDK 收尾 | `sdk/agent.py`：图完成后无条件 success 日志 + `end_session("completed")`（~718-721）、task.status 收敛（~1035-1040）、trace 命名 `_PASS/_FAIL`（~1061） | 接入 run outcome（§5.5） | +| verification 工具函数 | `utils/verification.py` | 删除 | +| 快照工具函数 | `utils/file.py` `create_snapshot`/`restore_snapshot` | 删前 grep 确认无其他调用方 | +| SDK 配置透传 | `sdk/types/agent.py`、`sdk/builders/agent_config_builder.py`、`sdk/agent.py` | §6 | + +--- + +## 3. WP1 — 摘除旧干扰源（先做） + +### 3.1 Prompt 不再切换 +- `operator.py` `_build_prompt`：删 `has_feedback` 分支，恒用 `main_template`。 +- `CheckerFeedbackPromptComponent` 改写为 `CheckFeedbackPromptComponent`：读 state 的 `check_feedback`（§5.2 写入），无内容不渲染，加入恒定组件列表。 +- 删 `_has_checker_feedback`、`_get_verification_chat_rounds`、`_get_reply_to_checker_tool` 及工具注册逻辑；`operator.json` `troubleshooter_template` 相关键清理（grep 确认无他处引用）。 + +### 3.2 每个 operator turn 都必须 record_step（含 planner 拒绝的 turn） +- 仅移除 `if checker_success:` 门控**不够**：planner 拒绝分支在 ~190 提前 `return`，同样跳过记录。 +- 做法：把现有 record_step 代码抽成节点内局部函数 `_record_turn(extra_metadata: dict) -> str | None`： + - 正常路径原位调用（extra 不变）； + - planner 拒绝分支在 `return` 前调用，extra 加 `{"planner_rejected": True}`（该 turn 的 terminal actions 被拦截未执行，记录如实反映决策与拦截）。 +- 保留"planner 拒绝的 turn 不清零 subagent 循环计数"语义（`checker_success` 今后仅由 planner 拒绝路径置 False，维持现有 `if checker_success: update["subagent_calls"] = []` 写法）。 + +### 3.3 删除 notes 快照/回滚 +- `operator.py` `create_snapshot` 段、`ctx.task_plan_snapshot` 赋值：删。 +- `execution_check_node` "Handle optimistic execution rollback/commit" 整段（~252-280）：删。 +- `context.py` `task_plan_snapshot`：删。`utils/file.py` 快照函数 grep 无调用方后删。 +- planner validation 拒绝时的 task_plan.md 单文件回滚（`ctx.task_plan_content_before`）**保留**。 + +### 3.4 删除单槽位与 chat 机制 +- `execution_check_node`：删 chat_path 触发段与 `await ctx.checker_task` 消费段。 +- `context.py`：删 `checker_task`、`checker_max_chat_rounds`。`planner_task` 保留。 +- `_process_plan_write` `new_completions` 触发段：暂时整段删除（§5.1 重建）。 +- `utils/verification.py`、`checker_status.json` 写入、相关 import：删。 + +WP1 完成后系统处于"无任何语义验证"的干净中间态，可独立运行与测试。 + +--- + +## 4. WP2 — 计划语法、Planner 与检查标准保护 + +### 4.1 plan grammar 扩展（`utils/plan_grammar.py`） +- 新语法：顶层子目标下缩进附属行，关键字带可选时机后缀： + ``` + - [ ] 在时钟应用创建 7:30 AM 闹钟 + - verify: 闹钟列表应出现 7:30 AM 且开关为开 + - assert: 创建后应弹出"已设定闹钟"提示 + - [ ] … + （任务级，可置于计划末尾游离段） + - assert@end: 全程不应出现应用崩溃对话框 + ``` +- 解析为： + ```python + @dataclass(frozen=True) + class CheckItem: + kind: Literal["verify", "assert"] + when: Literal["on_complete", "at_end"] # 无后缀=on_complete；@end=at_end + text: str + parent_key: str | None # 挂靠子目标的 key；任务级为 None + ``` + `when` 语义：`on_complete` 在锚定子目标完成时刻用**当时证据**判定；`at_end` 只在出口用**最终状态**判定。**禁止**在出口用当前状态重判 `on_complete` 项，也禁止在中途提前执行 `at_end` 项。 +- 新增 `CHECK_LINE_RE`；先确认 `parse_plan` 对非 checkbox 缩进行的容忍度（`CHECKBOX_LINE_RE` 只匹配 checkbox 行），确保 check 行**不影响** `milestones_changed`/`subgoal_hash`/`new_top_level_completions` 现有行为。 +- `PlanSnapshot` 新增 `check_items_of(item)`、`task_level_check_items`、`all_check_items`；新增模块函数 `check_items_changed(before, after)`（比较 check 行多重集合）。 +- **语法说明条件化**：`PLAN_GRAMMAR_SPEC` 从模块常量拆为"基础语法 + check 扩展"两段，新增 `render_plan_grammar_spec(include_checks: bool) -> str`。check 扩展段内容：两关键字语义（verify=完成度，失败触发修复；assert=测试断言，失败如实记录不修复）、`@end` 后缀、§1.2 能力边界声明（事后核验、不支持强制前置关卡、瞬态断言的证据约束）。三个现有注入点改用该函数：`planner.py` ~105/~178、`operator/prompts.py` ~206；`include_checks = not（两开关全关）`。全关时任何 prompt 里都不出现 check 语法。 + +### 4.2 Planner 生成检查项（`planner.json`） +- **生成指令是 planner prompt 的条件段**：检查项相关的全部指令（下述各条 + §4.1 的 check 语法扩展）只在 `include_checks` 为真时注入。两开关全关 → Planner 的 prompt 与产出和无此功能时完全一致，不生成任何 check 行。 +- 用户目标里的显式预期 → `- assert:` 行，逐字保留用户语义；属于全程/最终状态的预期用 `@end`。 +- `- verify:` 行**只为确需验收的关键节点**生成——判断标准：不可逆操作完成点、对最终目标有直接贡献的里程碑。**不是**每个子目标一条；克制是明确要求（成本与 check 项数量成正比，见 §6）。 +- 不可验证的子目标不硬造断言；瞬态预期优先改写为可持久探测的状态断言。 +- 用户提出"必须先确认 A 才能做 B"类强序要求时：断言照常生成，但按 §1.2 标注"事后核验"，不得静默承诺前置拦截。 + +### 4.3 检查标准保护（确定性守护，不依赖 LLM） +- **前提事实**：planner validation 默认关闭（`disable_planner_validation=True`），不能作为 check 行不可删的唯一保障。 +- **确定性守护**（`_process_plan_write` 内，Operator 来源的写入）：写入后若 `check_items_changed(before, after)` 检出 check 行被删除或改写，Graph **把缺失的 check 行合并回写**进新计划（锚定其 `parent_key`；父子目标整个被删时，该行转为任务级 `@end` 项）。合并回写是纯文本操作、同步完成，不打断 Operator；Operator 新增 check 行允许保留。Planner 节点自身的计划写入不受此守护（Planner 有权修订检查标准）。 +- planner validation **开启时**：check 行差异同样纳入其审计范围（prompt 增加"检查标准被弱化应拒绝"要点）。机制本体（CAS 回滚、ratchet baseline）不动。 + +### 4.4 Operator 可选说明段（`operator/prompts.py`） +- 新增 `CheckItemsExplainerPromptComponent`，加入 `main_template` 恒定组件列表，**仅当当前 task_plan.md 解析出 check 行时渲染**（以计划实际内容为准，不看开关——续跑的旧计划带 check 行时同样需要解释），否则输出空串。 +- 内容（一段，短）： + - `- verify:` 是该子目标的验收标准，执行时可以参考它确认做到位，但**判定由独立 Checker 执行**，不要自行宣布通过或替 checker 记录结论； + - `- assert:` 是测试断言，**不需要为它做额外动作**，更不得为满足断言去构造/伪造状态——断言失败是合法的测试结果； + - check 行**不可删除或改写**（删除会被系统自动恢复）；重写计划时原样保留，新增允许； + - 正常按语法打完成标记即可，检查在后台异步进行，不阻塞执行。 +- 这与 §4.1 注入 operator 的语法扩展分工：语法扩展讲"这些行是合法语法"，本组件讲"你该怎么对待它们"；前者受开关控制，后者受计划内容控制。 + +### 5.1 触发：排队与 spawn 分离（证据锚点修正） +- **`_process_plan_write` 只排队**：`new_top_level_completions` 检出新完成项后，对每个带 `on_complete` check 项的完成项，追加 `ctx.pending_checkpoints: list[PendingCheckpoint]`（字段：`checkpoint_id=subgoal_hash`、check 项、完成时计划全文、触发时间戳）。**不在此处 spawn**——此刻本 turn 的 step 尚未记录，`get_step_number` 读到的是上一轮，锚点会错位。 +- **`execution_check_node` 在 record_step 之后 spawn**：对每个 pending checkpoint 生成 `attempt_id`（`{checkpoint_id}#{递增序号}`），构造证据锚： + ```python + @dataclass(frozen=True) + class EvidenceAnchor: + anchor_step_id: str # 刚记录的本 turn step（含 pre 截图；上一 turn 的 post 截图已在库） + trigger_ts: float + plan_text: str # 完成时刻的计划全文 + ``` + 然后 `ctx.checkpoint_tasks[checkpoint_id] = (attempt_id, asyncio.create_task(run_checkpoint_check(...)))`。 +- **supersede 不丢裁决**：spawn 前若同 `checkpoint_id` 已有旧条目——旧任务 `done()` → **先收割入账**（§5.2）再替换；仍在运行 → `cancel()` 并 `await`（吞 `CancelledError`，完成资源清理），账本记一条 `superseded` 记录（attempt_id、时间），再 spawn 新 attempt。 +- 每个新完成项都触发（修复旧"只验 `new_completions[-1]`"bug）。中途开关关闭时不排队不 spawn（相关项出口按 §5.3 处理）。`user_stop_requested` 后不再 spawn 任何新检查。 +- **并发上限**：`max_concurrent_checkpoints`（默认 3），超出的 pending 留在队列，下轮 execution_check 再 spawn。 + +### 5.2 收割（`execution_check_node` 内，非阻塞） +- 每轮遍历 `ctx.checkpoint_tasks`，**只收割 `task.done()` 的**（绝不 await 未完成的），收割后从 dict 移除： + 1. **入账**：所有裁决 append-only 写入 DataEngine 账本（session 元数据或专用 jsonl：attempt_id、checkpoint_id、item_text、kind、when、status、evidence、anchor_step_id、时间）。这是"首次失败永久保留"的唯一载体，任何后续结果只能追加。 + 2. **状态副作用的适用性校验**：仅当该 attempt 仍是此 checkpoint 的**当前 attempt**、且锚定子目标在当前计划中文本未变时，才允许改执行状态；过期/失配的裁决只入账不作用。 + 3. `verify` FAIL（适用时）→ 子目标 `[x]`→`[/]`（`subgoal_hash` 匹配，直接写文件不走 wrapped note tool）+ finding 追加进 state `check_feedback`；**每 checkpoint 修复配额 `checkpoint_max_repairs`（默认 2）**：超配额不再打回，裁决保持 failed，留待出口如实呈现。`user_stop_requested` 后不再打回。 + 4. `assert` FAIL → 只入账，不打回不注入（可配 `assert_failure_policy: "continue" | "halt"`，halt=置停止标记走 §5.3 收尾）。 + 5. cancelled 记 `superseded`；异常/超时：verify 记 `inconclusive` + warning（**裁决不改写为 passed**，仅放行决策按通过处理），assert 记 `inconclusive`。 +- **单次检查超时**：`checkpoint_timeout`（默认 180s，spawn 时用 `asyncio.wait_for` 包裹）→ 超时按异常处理。 +- 副作用只发生在收割点与出口结算，任务体内零副作用。 + +### 5.3 出口：结算与终审分离 +- `convergence_gate` 扩为 `Literal["continue", "exit_settlement", "end"]`： + - `ctx.data_engine` 为 None → `"end"`； + - 任务到达终态（`all_top_level_done`，或 continuous+user_stop，或 halt 标记）→ 只要**存在任何 check 项/未决任务/账本记录，或结尾终审开启**，一律 `"exit_settlement"`；两者皆无才 `"end"`。 + - **空计划（`not has_top_level`）且结尾终审开启 → 仍走 `"exit_settlement"`**：没有计划项不等于无可验之物，终审对象是用户原始目标。 +- 新增节点 `exit_settlement_node`，两阶段，**阶段一无条件执行，阶段二受结尾开关控制**： + 1. **结算（barrier）**：收齐 `checkpoint_tasks` 全部未决任务并按 §5.2 入账——这是全流程唯一有意的阻塞等待，带**加总超时** `settlement_timeout`（默认 120s）：超时 cancel 剩余任务、对应项记 `unchecked(timeout)`。已启动的检查必须在退出前有账，不论结尾开关。 + 2. **终审（`disable_final_check=False` 时）**：运行 `run_final_check`，对照 **`state.initial_goal`（用户原始目标，不是计划）** + 全部 check 项： + - `at_end` 项：用最终状态判定； + - `on_complete` 项：**引用账本裁决**（当前 attempt 的 PASS/FAIL 直接采信，FAIL 过的 assert 保持 FAIL）；从未检查过的（中途关闭/配额限制），基于锚点附近的**历史证据**补判，证据不足记 `unchecked`——禁止用当前屏幕重判过程断言； + - 终审自身超时/异常：fail-open 放行，账记 `inconclusive`。 + 3. 路由：终审通过或关闭 → `END`；verify 类未达成（assert 失败**不算**）→ 打回 unmet 项 + `check_feedback` 注入 → `perception`；`final_check_attempts` 达上限（默认 3）或 user_stop/halt → `END`。 +- 收尾状态见 §5.5；continuous+user_stop 时 FAIL 不回循环。 + +### 5.4 Checker 本体（`agents/checker/checker.py` 重写） +- 两个入口共用旧 `run_async_check` 的工具循环骨架（多轮循环、迭代上限 `checker_max_iterations`、结构化兜底，可整体沿用）： + - `run_checkpoint_check(check_items, anchor: EvidenceAnchor, goal, ...)` + - `run_final_check(goal, plan_text, ledger, ...)` +- **输入**：用户原始目标、计划、待验项、证据锚、`get_agent_friendly_steps()` 摘要历史。 +- **工具表按入口 × 配置装配（§1.3；工具表是唯一权限来源，全部只读）**： + + | 工具 | `run_checkpoint_check` | `run_final_check` | 配置开关 | + |---|---|---|---| + | `get_step_detail(step_no)` / `get_step_screenshot(step_no, which: pre\|post)`（`storage.get_steps` + step 目录，新增只读辅助） | ✓ | ✓ | 恒定 | + | `read_note`（只读） | ✓ | ✓ | 恒定 | + | `probe_device(kind, params)`（固定探测类型 + 程序生成 argv，见下） | ✓ | ✓ | `disable_device_probes`（默认 False）；关闭则**不注册且 prompt 零提及** | + | 实时屏幕读取（`UnifiedMobileController.get_screen_data()` + OCR，旧 checker 有现成代码） | **✗ 不装配** | ✓ | 恒定（仅 final 入口） | + + 中途检查点**不装配实时屏幕**是刻意的证据纪律：检查点的证据是锚点历史与持久状态探测，当前屏幕早已随 Operator 前进，装配它只会诱导用错证据——纪律靠工具表落实，不靠 prompt 劝告。 + **任何入口都不注册**：设备 action、write_note、子代理。 +- **`checker.json` 拆为组件段，按待验内容与入口装配**（不在单一模板里堆条件文本）： + + | prompt 段 | 装配条件 | + |---|---| + | 基础裁决规则（逐字引用、证据具体化、inconclusive 语义） | 恒定 | + | verify 语义段（完成度判定、fail-open 放行规则、suggestion 要求） | 待验项含 verify | + | assert 语义段（测试断言、失败即结果、证据缺失记 inconclusive 不记 failed） | 待验项含 assert | + | 锚点取证指南（瞬态/顺序断言必须查锚点附近历史） | `run_checkpoint_check` 入口 | + | 终审指南（对照原始目标、引用账本、at_end 用最终状态、禁止重判过程断言） | `run_final_check` 入口 | + | probe 使用指南（持久状态优先探测、探测类型速查） | `probe_device` 已注册 | +- **`probe_device` 设计（不做前缀白名单）**：`dumpsys` 等含状态修改子命令（如 `dumpsys battery set level 100` 可改电量模拟），且前缀判断挡不住参数拼接。改为枚举探测表，argv 列表由代码构造、不经 shell 字符串拼接，参数正则校验（拒绝空白符与 shell 元字符）： + ```python + PROBES: dict[str, ProbeSpec] = { + "alarms": argv=["dumpsys", "alarm"], # 无参数 + "battery": argv=["dumpsys", "battery"], # 无参数（禁 set/unplug 等子命令） + "foreground": argv=["dumpsys", "activity", "activities"], + "notifications":argv=["dumpsys", "notification"], + "setting": argv=["settings", "get", , ], # ns∈{system,secure,global}；key: ^[A-Za-z0-9._-]+$ + "content": argv=["content", "query", "--uri", ], # uri: ^content://[A-Za-z0-9./_-]+$ + "packages": argv=["pm", "list", "packages"], + "prop": argv=["getprop", ], # key: ^[A-Za-z0-9._-]+$ + } + ``` + 未列举的探测类型一律拒绝；扩表需改代码，不接受运行时自由命令。底层复用 `run_adb_command` 的执行通道但以 argv 形式传递。 +- **输出**（structured output）： + ```python + class CheckVerdict(BaseModel): + item_text: str # check 行原文，逐字 + kind: Literal["verify", "assert"] + status: Literal["passed", "failed", "inconclusive"] + evidence: str # 具体观察，禁止空泛 + suggestion: str # 仅 verify failed 需要 + + class CheckReport(BaseModel): + verdicts: list[CheckVerdict] + unmet_subgoals: list[str] # 仅 final check：逐字引用计划项 + ``` + 放行决策（passed 布尔）由 **node 侧**从 verdicts 计算（verify 全 passed/inconclusive → 放行），不由模型直接输出——保证"放行≠裁决改值"。 +- **各段共同要点**：failed 必须附具体证据（探测输出或截图元素），空泛否决 node 侧降级 inconclusive；证据从未被采集时输出 inconclusive 并说明"证据缺失"而非 failed；引用计划项逐字。 + +### 5.5 任务收尾与 SDK 传导（放行 / 测试结果 / 收尾状态三轴分离） +- Graph 在 END 前汇总 **run outcome**（写入 state 新字段 `run_outcome` 与 DataEngine session 元数据）： + ```python + class RunOutcome(BaseModel): + task_status: Literal["completed", "partial", "blocked"] # 目标达成轴 + tests: TestSummary # passed/failed/inconclusive/unchecked 计数 + failed 明细 + ``` + - verify 未达成且预算耗尽 → `blocked`（或 `partial`）； + - **assert 有任何 FAIL → `tests.failed > 0`，即使 `task_status="completed"`**——任务做完与测试通过是两回事。 +- **SDK 链路必改**（`sdk/agent.py`）： + - 图完成处（~718-721）：不再无条件打 success 日志 / `end_session("completed")`——读 `run_outcome`：`blocked` → `end_session("failed")`（确认 `data_engine.end_session` 支持的状态集，必要时扩）；`completed` 但 `tests.failed>0` → `end_session("completed")` 且 session 元数据带 test summary，日志明确打出"任务完成、N 条断言失败"； + - task.status 收敛处（~1035-1040）与 trace 命名处（~1061 `_PASS/_FAIL`）：断言失败的运行 trace 命名用 `_TESTFAIL`（或并入 `_FAIL`，二选一但必须与 `_PASS` 区分）； + - `mobile_run_task`/`mobile_manage_task` 返回结构透出 test summary，调用方无需翻报告正文即可拿到机器可读结果。 +- Outputter 报告呈现完整裁决序列与三态统计；BLOCKED/partial 收尾时最后一次 finding 入元数据。 + +--- + +## 6. WP5 — 配置与 SDK 透传 + +- `ExecutionSetup`（`context.py`）： + - `disable_midway_checks: bool = False`、`disable_final_check: bool = False`（默认开。成本与计划中 check 项数量成正比——§4.2 已要求 Planner 只在关键节点生成 verify，这是成本上界的来源，**不存在"零成本"**）； + - `disable_checker` 保留为兼容别名：True 时两者同关； + - `final_check_max_attempts: int = 3`、`checkpoint_max_repairs: int = 2`、`max_concurrent_checkpoints: int = 3`、`checkpoint_timeout: float = 180.0`、`settlement_timeout: float = 120.0`、`assert_failure_policy: Literal["continue","halt"] = "continue"`、`disable_device_probes: bool = False`； + - `disable_planner_validation`、`checker_max_iterations` 语义不变。 +- SDK 透传链：`sdk/types/agent.py`、`sdk/builders/agent_config_builder.py`（`with_disable_checker` 保留 + 新开关映射）、`sdk/agent.py`。 +- 四组合行为： + | 中途 | 结尾终审 | 行为 | + |---|---|---| + | 关 | 关 | 不 spawn、不终审；**已启动的任务仍出口结算**（本组合下无任务）；全部 check 项报 unchecked | + | 关 | 开 | 出口结算（空）+ 终审：`at_end` 判最终状态，`on_complete` 凭历史证据补判，不足记 unchecked | + | 开 | 关 | 检查点照跑；出口**只结算不终审**（阶段一无条件），账本与 test summary 照常产出 | + | 开 | 开 | 检查点 + 结算 + 终审 | + +--- + +## 7. 明确不做（Out of scope，勿顺手扩大） + +- **强制前置关卡**（"确认 A 后才允许执行 B"的中途阻塞等待）——本期只提供事后核验；出口 barrier 保证的是**退出前完成裁决结算**，不保证执行顺序约束。此边界必须如实声明（§1.2），不得静默接受此类需求。 +- "动作前"检查时机（前置状态是 Operator 感知职责）。 +- 逐动作依赖分析。 +- 多验证器投票、版本化 VerificationKey、shadow 遥测。 +- Validator 拆分、统一 FailureEvent/RecoveryCoordinator、`_check_infinite_loop` 降级。 + +--- + +## 8. 测试 + +现有测试（先跑通再动手，改完逐个修）： +- `tests/unit/graph/test_graph.py` —— execution_check_node / convergence_gate 断言大改。 +- `tests/unit/agents/test_plan_mutation_guardrails.py` —— planner validation 主体不动；补 check 行确定性守护用例（§4.3）。 +- `tests/unit/utils/test_plan_grammar.py` —— verify/assert/@end 解析、不影响 milestone hash。 +- 全仓 grep `verification_chat`、`checker_status`、`task_plan_snapshot`、`create_snapshot`、`troubleshooter`、`ctx.checker_task`、`checker_max_chat_rounds`：零残留。 + +新增（建议 `tests/unit/graph/test_checkpoints.py` + `test_exit_settlement.py`）： +1. 排队/spawn 分离：`_process_plan_write` 只排队；execution_check 在 record_step 后 spawn，anchor_step_id 等于本 turn 刚记录的 step。 +2. 多个同轮完成项全部排队；无 check 项不排队；中途开关关不排队；user_stop 后不 spawn。 +3. supersede：旧任务 done 未收割 → 先入账再替换（**失败裁决不丢**）；运行中 → cancel 且 await 完成，账本有 superseded 记录。 +4. 收割只取 done；适用性校验：过期 attempt / 子目标文本已变 → 只入账不打回。 +5. verify FAIL 打回 + 注入；超 `checkpoint_max_repairs` 后不再打回、裁决保持 failed。 +6. assert FAIL 只入账；`halt` 策略触发收尾；账本 append-only（先 FAIL 后 PASS 两条都在）。 +7. 超时：单任务超 `checkpoint_timeout` → inconclusive；结算超 `settlement_timeout` → 剩余 cancel + unchecked(timeout)。 +8. 出口路由：结算与终审解耦——终审关、有未决任务 → 仍进 exit_settlement 且只走阶段一；空计划 + 终审开 → 进终审；全关全无 → 直接 end。 +9. `when` 语义：`at_end` 不在中途执行；`on_complete` 在终审引用账本、无账本凭历史补判、证据不足 unchecked（构造"当前状态已满足但历史证据缺失"的用例，断言**不**判 passed）。 +10. record_step 无条件：正常 turn 与 planner 拒绝 turn 都有记录，后者带 `planner_rejected` 标记。 +11. check 行守护：Operator 写入删除 check 行 → 合并回写；父目标被删 → 转任务级 `@end`；Planner 写入可修订；planner validation 关闭时守护独立生效。 +12. `probe_device`：枚举表内放行、表外拒绝、参数带空白/元字符拒绝、`battery set` 类子命令无法表达。 +13. fail-open 三轴分离：verify 异常 → 放行但账记 inconclusive（**不是 passed**）。 +14. run outcome / SDK：assert 失败 → `tests.failed>0`、trace 名非 `_PASS`、`end_session` 元数据含 summary；verify 预算耗尽 → `blocked` + `end_session("failed")`。 +15. continuous+user_stop+FAIL → 不回循环。 +16. 四开关组合矩阵（§6 表逐行）。 +17. prompt 条件化：两开关全关 → `render_plan_grammar_spec` 输出不含 check 语法、planner prompt 无生成指令；`CheckItemsExplainerPromptComponent` 仅在计划含 check 行时渲染（含"开关关但旧计划带 check 行仍渲染"的用例）。 +18. checker 装配：`run_checkpoint_check` 的工具表**无实时屏幕**、`run_final_check` 有；`disable_device_probes=True` → `probe_device` 未注册且 prompt 无探测指南；待验项全 verify → prompt 无 assert 段（反之亦然）；checkpoint 入口无终审指南段。 + +运行：`python -m pytest tests/unit/graph tests/unit/agents tests/unit/utils -x`（需要假 LLM 设 `ARTEMIS_FAKE_LLM=1`）。 + +--- + +## 9. 验收标准 + +1. Operator 的 prompt 模板、notes、DataEngine 历史在任何验证结果下不被回滚/切换/丢弃；**每个 operator turn（含 planner 拒绝的）都有 step 记录**；中途路径上不存在对未完成验证任务的等待。 +2. verification chat、`checker_status.json`、notes 全量快照、单槽 `ctx.checker_task` 从代码库消失。 +3. 检查项由 Planner 声明（verify/assert × on_complete/at_end），Operator 删除/弱化 check 行会被**确定性守护**恢复，不依赖 planner validation 开关。 +3a. **§1.3 装配纪律全局成立**：检查全关时 planner/operator prompt 与无此功能时一致（零污染）；计划含 check 行时 Operator 收到行为说明段；checker 的 prompt 段与工具表按入口 × 配置装配（checkpoint 入口无实时屏幕、probe 关闭则工具不注册且 prompt 零提及、无 assert 项则无 assert 段）；不存在指向已关闭机制的死指令。 +4. 检查点在 record_step 之后 spawn，证据锚指向正确的 step；裁决全部 append-only 入账，supersede 不丢已完成裁决；检查有单次超时、并发上限、修复配额。 +5. 出口结算无条件执行（含加总超时与 unchecked 标记），终审受结尾开关控制；空计划不绕过已开启的终审；终审对照用户原始目标；`at_end`/`on_complete` 判定时机不错位。 +6. 放行、裁决、收尾三轴分离：fail-open 不改写裁决值；assert 失败产生机器可读 test summary 并传导至 `sdk/agent.py` 收尾（日志、end_session、trace 命名、返回结构）。 +7. `probe_device` 为枚举探测 + 程序生成 argv，无自由命令通道。 +8. 计划/报告如实声明"事后核验"边界，强序需求不被静默接受。 +9. planner validation（入口卡点）除新增 check 行审计外与现状一致。 +10. §8 测试全绿。 + +--- + +## 10. 建议执行顺序 + +WP1（3.2 → 3.3 → 3.1 → 3.4，摘干扰源，得到干净中间态）→ WP2（§4 语法 + Planner + 守护，独立可测）→ §5.4 checker 本体与 probe 表（可单测）→ §5.1 触发 → §5.2 收割 → §5.3 出口 → §5.5 收尾传导 → WP5 配置透传 → 全量测试。 + +每步完成后系统都应可运行：WP1 后=无验证；WP2 后=计划里有 check 项但无人消费（守护已生效，无害）；§5.4 后=checker 可单测；接线后=全功能。 diff --git a/docs/plans/context-visibility-redesign.md b/docs/plans/context-visibility-redesign.md new file mode 100644 index 0000000..3007e83 --- /dev/null +++ b/docs/plans/context-visibility-redesign.md @@ -0,0 +1,586 @@ +# ARTEMIS 上下文与记忆重构：显式状态、按 agent 可见域、统一记忆内核 + +状态：提案 v2（2026-08-31，经"不损伤"审计细化，待终审） + +范围：State / ArtemisContext 的类型化与生命周期分域；感知数据通道；Flash 压缩机制上提为通用记忆内核。 + +底线与目标：**不损伤为最低要求**——每一处删除/搬迁都在附录 B 台账中核销其全部消费者， +任何信息流不得静默中断；**优雅处理所有信息为目标**——每份数据有唯一属主、显式通道、 +声明的可见域，重复实现收敛为单一入口。 + +姊妹文档：`docs/design/pro-context-memory-redesign.md`（2026-08-25）深度覆盖了 Pro 侧 +"压缩什么、怎么压"（视觉摘要、历史分块、召回工具、token 预算）。本提案覆盖它没有处理的 +另一半——**数据放在哪、谁能看见什么、通道是否显式**——并在 §6 把两份文档的交叠处收敛为 +同一个记忆运行时。 + +--- + +## 1. 调查结论：现状与证据 + +### 1.1 实际存在四个上下文平面，只有一个有主人 + +| 平面 | 载体 | 生命周期 | 现状 | +|---|---|---|---| +| P1 图通道 | `State`（pydantic + `extra="allow"` + reducer） | 跨节点、跨回合 | 声明字段与实际使用严重脱节（见 1.2） | +| P2 就地突变 | 同一个 State 实例被节点/工具直接赋值 | **节点内**（经 `InjectedState` 同实例） | 无声明的隐式契约；经核验其全部消费都是节点内的（见 5.3） | +| P3 ctx 动态属性 | `ArtemisContext`（`extra="allow"`） | 进程/运行期 | `latest_perception_data` 等动态挂载，无 schema | +| P4 持久存储 | DataEngine（步骤/截图/摘要）+ notes 文件 | 持久 | **唯一健康的平面**：单一写入路径，多消费者 | + +Pro 的真实架构其实很好：Operator 每回合从 P4 **重新编译**全量 prompt +（`operator.py:624` → `build_plan_and_history`），不靠累积消息。问题不在架构方向， +而在 P1–P3 三个平面无 schema、无属主、互相兜圈子。 + +### 1.2 `extra="allow"` 已经吞掉的真 bug（本次调查实证） + +**B1 幽灵键 `ui_tree`：4 个读取点，0 个写入点。** +`operator.py:995`、`validator/tool_declarations.py:477`、`mcp/action_executor.py:401`、 +`tools/mobile/exec_tools.py:548` 均 `getattr(state, "ui_tree", None)`，但全仓无人写入 +（真实数据在 `latest_ui_hierarchy`）。后果：`utils/coordinates.py:185-215` 的 +"从层级树找主滚动容器"分支**永远走不到**，smart swipe 静默降级。 + +**B2 异步 Planner 驳回反馈被静默丢弃。** +`graph.py:195` 把驳回理由写进 `validator_messages` 通道，但该通道**没有任何节点读取** +（`operator.py:388` 和 `tool_wrapper.py:175` 只是当场拆 `Command.update` 信封，从不读 +State 通道）。后果：计划被回滚了，Operator 下一回合看到回滚后的计划，却永远看不到 +驳回理由。同时 `validator_messages` 经 `add_messages` 单调累积整轮所有 ToolMessage， +无人消费、无人裁剪。 + +**B3 `ctx.latest_perception_data` 孤儿写。** +`perception.py:187` 每轮把整张截图 b64 挂上 ctx，注释声称"给 Checker 零延迟复用"； +但工作区的 checker 重写已改为自己抓屏（`checker.py:620-652`）+ 从 DataEngine 取历史 +证据。全仓**零读取点**，纯内存驻留。 + +**B4 `indexed_elements` 只写实例不回通道。** +`operator.py:649-650` 就地写两个字段，但 `operator.py:766` 的 update 只回传 +`indexed_points`，漏了 `indexed_elements`。审计确认跨节点消费实际不存在（validator 侧 +自行重推导，`action_executor.py:173-174`），但两键不对称是持续的误导源。 + +**B5 更多幽灵/半死键与坏路径。** 全表见附录 B。要点： +约 9 个只初始化/只读不写的死键；`graph_runner.py:48` 的 +`getattr(self.ctx, "artemis_context", None)` 恒 None → 构造出丢失 data_engine 的空 +ArtemisContext；且 `reactive_runner.py` 的 `State(messages=[])` 缺必填字段构造即抛 +——engine 双 runner 的真机路径今日均必炸；`_active_driver` 未声明为 `PrivateAttr`。 + +（v2.1 更正：初版审计将 `operator_agent.py:63` 的 `task_goal` 与 +`failure_analyzer_agent.py:48-50` 的 `failure_context` 列为图 State 幽灵键，实为 +engine/BaseAgent 栈自己的 state 协议——`ExecutionContextState.task_goal` 存在 +（core/state.py:48），`.get()` 有 `isinstance(state, dict)` 防护。两处均无需修改。） + +**B6 感知数据同屏最多 4 份拷贝、抓屏三处重复实现。** +同一帧 b64 同时在 `ctx.latest_perception_data`（死）、`state.operator_raw_data` +（走图通道且随 `astream` values 流每超步复制，`sdk/agent.py:694-696`）、operator 局部、 +prompt data-URL。抓屏管线在 perception_node（settle+OCR+融合）、validator 动作后轮询 +（`validator.py:117-140`）、checker 终检（`checker.py:620-652`）三处各写一遍， +planner 首帧还有第四处自拍（`planner.py:211-236`，注意 planner 在图中先于 perception +运行，这是常规路径不是边角）。 + +### 1.3 Flash 与 Pro 的压缩机制现状 + +| | Flash | Pro | +|---|---|---| +| 上下文模型 | 累积消息 + 每轮幂等原地压缩（`compress_flash_messages`） | 每回合从 DataEngine 重编译（`build_plan_and_history`） | +| 步骤摘要 | `VisualStepSummarizer`：前后截图 + 动作红标叠加，客观中立措辞契约，异步无阻塞，无界重试 | `SummarizerNode`：**纯文本**（不加载 pre/post 图），3 次退避重试，无 flush 收尾语义 | +| 摘要落点 | `data_engine.update_step_summary`（同一个 API！） | `data_engine.update_step_summary` | +| 未就绪降级 | 摘要 pending → 保留原图（lossless pending） | 无 summary → detailed 渲染（同构原则的另一形态，`task_tree.py:917-921`） | +| 历史可见性策略 | 无（30 轮上限内全量） | 三种互斥策略 + 6 个调用点各自硬编码 kwargs | + +关键事实：**两条路径的摘要已经写进同一个 DataEngine 字段**。但两者的摘要语义结构性 +不同（见 6.1），统一的只能是运行时，不是语义。 + +--- + +## 2. 设计原则 + +1. **上下文是编译出来的，不是累积出来的。** Pro 已经如此；把它升格为正式模型， + Flash 的消息列表只是同一份记忆的另一种渲染。 +2. **一份情景记忆，多种视图。** DataEngine 是唯一事实源；每个 agent 看到的是按其 + 角色声明的"最高价值密度视图"。agent 的智能上限由输入质量决定。 +3. **可见域靠声明与类型约束。** `extra="forbid"` 拦截幽灵**写**与非法构造；幽灵**读** + （`getattr` 带默认值不会因 forbid 报错）由节点 READS/WRITES manifest 的测试期 + 代理拦截。两者缺一不可。 +4. **重物不过通道。** 截图 b64 不进 LangGraph 通道；通道里只放引用，重物放进有属主、 + 有生命周期的 store。 +5. **就地突变不消灭，而是正名。** P2 平面的全部用途经审计都是"节点内工具通信"—— + 给它一个声明的、节点局部的载体（TurnWorkspace），而不是假装它不存在。 +6. **后台增强不可成为依赖。**（继承姊妹文档 §4.6）摘要未就绪必有无损降级。 +7. **不损伤：删除必先核销。** 每个字段/属性/通道的处置都对照附录 B 台账，列明全部 + 消费者与同步改动点；无消费者证据不删。 + +--- + +## 3. 方案 A：State 显式化与按 agent 可见域 + +### 3.1 State 收敛为四组显式字段，`extra="forbid"` + +```python +class State(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") + + # ── 控制面（跨回合、路由与信号）────────────────────── + initial_goal: str + injected_instruction: str | None # take_last + user_stop_requested: bool # sticky_or + checker_success: bool | None # take_last + operator_feedback: list[str] | None # take_last；见 3.3 + exit_settlement_route: str | None # take_last + run_outcome: dict | None # take_last + + # ── 回合产物（上一回合的决策与结果）─────────────────── + current_step_id: str | None + structured_decisions: str | None + operator_raw_thinking: str | None + operator_native_thinking: str | None + short_term_memory: str | None # 活跃字段；移除权属于姊妹文档 §3.3 + last_execution_result: dict | None + subagent_calls: list[str] + operator_tool_limit_exceeded: bool | None + + # ── 感知引用（重物不进通道，见方案 C）───────────────── + perception: PerceptionRef | None # take_last + + # ── 节点局部工作区（声明但非通道语义，见 5.3）───────── + workspace: TurnWorkspace | None = None # 不进 update、不跨节点 +``` + +**处置总表见附录 B。** 与 v1 的差别：`short_term_memory` 保留（它是活跃读写对， +operator.py:743-765 写 → prompts.py:331 读；删除属姊妹文档 Phase 1 的事，本方案 +不越权）；`indexed_*` 不再声称"并入冻结快照"（见 5.3）；`messages` 删除需同步改 +三处消费代码（`sdk/agent.py:1182-1187` 兜底输出、`sdk/agent.py:1321-1325` stderr +打印、`engine/graph_runner.py:57` 入参——三处今日均为空转/坏路径，删除无信息损失， +但必须同 PR 内改掉）。 + +`asanitize_update`（`state.py:152-159` 空壳直通）删除；其在 14 处工具里的调用 +随信封迁移（3.3）一并消失。 + +### 3.2 可见域：节点声明读/写集，测试期强制 + +每个节点类声明清单，运行期零开销，测试期强制： + +```python +class OperatorNode: + READS = frozenset({"initial_goal", "perception", "operator_feedback", + "short_term_memory", "subagent_calls", "current_step_id", + "operator_tool_limit_exceeded", "injected_instruction", + "structured_decisions"}) + WRITES = frozenset({"structured_decisions", "operator_raw_thinking", + "operator_native_thinking", "short_term_memory", + "current_step_id", "subagent_calls", + "operator_tool_limit_exceeded"}) +``` + +- 新增 `artemis/graph/visibility.py`：`StateView(state, reads)` 代理，未声明字段访问 + 即抛 `VisibilityError`；`check_update(update, writes)` 校验返回键。 +- 仅在测试与 `ARTEMIS_STRICT_STATE=1` 下启用；生产路径直接传 State。 +- 附录 B 的实证读写清单即初始 manifest；上线先按现状放宽，再逐步收紧。 +- **幽灵读的防线在这里**，不在 `extra="forbid"`（forbid 管不住 `getattr(x, k, None)`）。 + +### 3.3 工具信封与反馈通道拆分（修 B2） + +`validator_messages` 承担两个不相干职责，拆开： + +**1）工具返回信封 → 显式返回类型。** +审计确认全部 14 个工具写入点（`exec_tools.py` 5 处、`scratchpad.py` 6 处、 +`launch_app.py:191`、`ocr.py:185`、`video_tool.py:161`、`object_detector.py:210,227`） +的 `Command.update` **只含**这一个键（以 `exec_tools.py:215-221` 为样本核验）， +迁移是纯机械的：工具直接返回 `ToolOutcome(content, status)`， +`tool_wrapper.get_tool_result_content`（`tool_wrapper.py:175-176`）与 operator 循环 +（`operator.py:388-391`）的拆信封逻辑同步简化。`graph.py:526` 的计划驳回信封同理 +（它经 `_process_plan_write` 返回给 operator 循环当场消费，属节点内信封）。 +通道及 `add_messages` 无界累积随之消失。 + +**2）跨回合反馈 → 统一 `operator_feedback` 通道。** +`check_feedback` 更名并扩容为 `operator_feedback`：来源包括 checker harvest +（`graph.py:198,230`）、exit settlement 弹回（`graph.py:367`）、以及**新接入的** +异步 Planner 驳回（`graph.py:195` 改写到这里，条目带 `[planner]` 前缀，与现有 +`[final check]`/`[verify failed]` 前缀惯例一致）。渲染组件更名 +`FeedbackPromptComponent`，提示语从"来自独立 Checker"改为按条目前缀中性表述。 +现有读者同步改名：operator prompt（`prompts.py:243`）与 diagnoser +（`diagnoser.py:178`）——diagnoser 因此也能看到 planner 驳回上下文，属增益。 + +--- + +## 4. 方案 B：ArtemisContext 分解为组合根 + +ctx 保留为唯一入口（避免全仓签名大改），字段按属主聚类成显式子对象， +`extra="forbid"`： + +```python +class ArtemisContext(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") + + # 基础设施（启动后只读） + device: DeviceContext + llm_config: LLMConfig | None + model_router: Any | None + agent_config: Any + adb_client / ui_adb_client / actuator / action_session / mcp_* + + # 运行服务（有生命周期、有属主） + execution_setup: ExecutionSetup | None + data_engine: DataEngine | None + perception: PerceptionStore # 方案 C，取代 latest_perception_data + step_memory: StepMemoryService | None # 方案 D，统一摘要运行时 + background: BackgroundTaskGroup # 吸收 background_tasks/jobs + 宽限期 + + # 编排状态（从散字段聚成对象） + checkpoints: CheckpointLedger # 吸收 pending_checkpoints/checkpoint_tasks/ + # attempt_seq/repairs/final_check_attempts/assert_halt + plan_governor: PlanGovernor # 吸收 task_plan_content_before/ + # last_validated_plan/pending_validated_plan/planner_task + + run_outcome: dict | None = None # 显式化（今日为 sdk/agent.py:721 动态挂载） + package_cache: dict[str, str | None] + _active_driver: Any | None = PrivateAttr(default=None) +``` + +`CheckpointLedger` 与 `PlanGovernor` 不是新逻辑，只是把 `graph/checkpoints.py` 和 +`graph/graph.py::_process_plan_write` 已在操作的字段簇搬进属主类型。 +`ExecutionSetup` 审计后可直接上 `forbid`（唯一运行期赋值 `app_lock_status` 为已声明 +字段，`sdk/agent.py:997`）。`graph_runner.py:48` 的 `artemis_context` 恒 None 坏路径 +在 Phase 0 修复（显式传入或显式报错，禁止静默构造空 ctx）。 + +--- + +## 5. 方案 C：感知走显式通道（PerceptionStore + TurnWorkspace） + +### 5.1 数据模型 + +```python +class PerceptionSnapshot(BaseModel): + """一帧感知的产物。frozen：每次捕获生成新快照，绝不就地改。""" + model_config = ConfigDict(frozen=True, extra="forbid") + + step_id: str | None # 关联的 DataEngine step（轮询帧可为 None） + image_name: str # sha256，DataEngine 图片主键 + screenshot_path: str | None + screenshot_b64: str # 重物：只存在于 store，不进图通道 + width: int + height: int + captured_at: float + kind: Literal["decision", "post_action", "probe"] + # 完整管线产物（raw 帧为 None） + raw_xml: list[dict] | None + fused_elements: list[dict] | None # OCR+XML 融合 + ocr_results: list | None + minimal_list: str | None # 派生物在此算一次，消费者不再各自重算 + +class PerceptionRef(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + step_id: str | None + image_name: str + width: int + height: int + +class PerceptionStore: + """ctx.perception。环形缓存最近 N=4 帧；更早的按 image_name/step_id 从 + DataEngine 回捞重建（b64 懒加载）。""" + async def capture_full(self, *, settle: bool, kind: str) -> PerceptionSnapshot: ... + async def capture_raw(self, *, kind: str) -> PerceptionSnapshot: ... + def put(self, snap: PerceptionSnapshot) -> PerceptionRef: ... + def latest(self, *, full_only: bool = False) -> PerceptionSnapshot | None: ... + def get(self, ref: PerceptionRef | str) -> PerceptionSnapshot | None: ... +``` + +**抓屏统一收口**：现有三处重复管线 + planner 自拍全部改走 store—— + +| 现调用点 | 改为 | 备注 | +|---|---|---| +| `perception_node`（settle+OCR+融合，`perception.py:116-168`） | `capture_full(settle=..., kind="decision")` | settle 启发式与 OCR/融合逻辑原样搬入 store，行为不变 | +| validator 动作后轮询（`validator.py:117-140`） | 轮询保持轻量私有；**最终帧** `capture_raw(kind="post_action")` 入 store | 轮询中间帧不入 store，避免噪音；最终帧成为可引用证据（今日被就地写进 `state.latest_screenshot` 后即失名） | +| checker 终检自拍（`checker.py:620-652`） | `capture_full(kind="probe")` | 其 OCR+裁剪逻辑与 perception 重复，删除 | +| planner 首帧自拍（`planner.py:211-236`） | `capture_full(kind="decision")` | planner 先于 perception 运行是常规路径，必须显式支持 | + +由此**每一帧都自动进入环形缓存与 DataEngine**，异步消费者（Checker 检查点）持 +`PerceptionRef`/`image_name` 而非活对象——缓存被挤掉也能从 DataEngine 重建， +天然解决"异步任务引用过期感知"的悬挂问题。 + +### 5.2 通道改造 + +- `perception_node`：`capture_full` → `put()` → update 只含 + `{"perception": ref, "injected_instruction": ..., "user_stop_requested": ...}`。 + 删除 `ctx.latest_perception_data`（B3）与 `operator_raw_data` 过通道（B6： + b64 不再随 `astream` values 每超步复制）。 +- 消费者一律 `ctx.perception.get(state.perception)` 或 `latest()`： + - **Operator**：快照直取 `minimal_list`/`fused_elements`，不再自算； + - **Validator 内工具**（object_detection 兜底 `object_detection_tool.py:93` 今日读 + validator 就地写的 `latest_screenshot`）：改读 `latest()`——validator 每次动作后 + `capture_raw` 入 store，时序语义与今日一致； + - **Diagnoser/Committee/Explorer/FA**：同一入口，删除各自的文件重读与 getattr 链； + - **Outputter**：今日读最终 State 的 `operator_raw_data`（最后决策帧）。改为 + `latest()`（最后 post_action 帧，比决策帧更新，属增益）；若需保持逐字节兼容， + 改读 `get(final_state.perception)` 即决策帧——落地时二选一并记录； + - 幽灵键 `ui_tree` 的 4 个读取点改读 `snapshot.fused_elements`（B1 终态修复； + Phase 0 先直改 `latest_ui_hierarchy` 止血）。 + +### 5.3 TurnWorkspace：给 P2 平面正名（修 B4 的正确姿势） + +审计结论：`indexed_points`/`indexed_elements` 是**回合内可变工作集**—— +explorer 增量 append 候选（`explorer_tool.py:311-315`），validator/FA 在动作后 +整体重推导（`action_executor.py:173-174`）；且全部消费都发生在**产生它的节点内部** +（operator 在返回前就把索引翻译成坐标；validator 侧自给自足）。跨节点传递从未 +被需要——B4 之所以没被发现，正因为如此。 + +因此它们**不属于冻结快照，也不属于图通道**，属于节点局部工作区： + +```python +class TurnWorkspace(BaseModel): + """节点局部的可变工作集。经 InjectedState 对同节点工具可见； + 不进 update、不跨节点、每节点从快照重播种。""" + model_config = ConfigDict(extra="forbid") + indexed_points: list[list[int]] = [] + indexed_elements: list[dict] = [] + + @classmethod + def seed_from(cls, snap: PerceptionSnapshot) -> "TurnWorkspace": ... + def reseed(self, snap: PerceptionSnapshot) -> None: ... # validator/FA 重观察 + def extend_candidates(self, cands: list[dict]) -> None: ... # explorer 增量 +``` + +- `state.workspace` 为声明字段但**永不出现在任何节点的 update 里**（visibility + manifest 强制），每个需要它的节点入口自行 `seed_from(store.get(state.perception))`。 +- 隐式契约（"恰好同一实例"）就此变成显式契约（"声明的节点局部容器 + 三个具名操作"）。 +- `state.indexed_points`/`indexed_elements`/就地写 `latest_screenshot`/`operator_raw_thinking` + 等全部 P2 突变点随之收编（thinking 就地写是给工具内 `record_step` 读的，同为 + 节点内通信，归入 workspace 或显式传参，落地时按触点少者取舍）。 + +--- + +## 6. 方案 D：记忆内核统一（Flash 压缩上提） + +### 6.1 一个运行时，两份摘要契约 + +两个摘要器的差异是**结构性的**，不能合并语义： + +- Flash 的摘要是**证据替身**——消息流保留了模型全部推理原文，压缩器只摘图， + 摘要因此必须是纯客观的视觉转换描述，掺入意图即是污染； +- Pro 的摘要是**整步胶囊**——`build_plan_and_history` 丢弃旧步骤的全部思考， + 一行摘要+精确动作+Validator 结果就是该步的全部遗留，因此必须压缩意图/策略/ + 进度（这正是姊妹文档 §8.2 坚持 `summarizer.md` 为语义唯一权威的原因）。 + +因此统一的是**机制层**，分离的是**语义层**。新建 +`artemis/memory/step_memory.py`： + +- **共享运行时 `StepMemoryService`**（合并两边的调度骨架）： + - 键：`step_id`（Flash 的 `tool_call_id` 保留为别名映射；`dispatch` 已有 + `data_engine_step_id` 参数，接线即可）； + - 调度：零阻塞 dispatch + **有界**重试 + 显式降级态（替换 Flash 的无界重试与 + Pro 的 3 次退避；此为有意的行为变更，见 §8）+ `flush(timeout)` 收尾语义 + （Pro 目前 run 结束时未完成摘要行为未定义，直接补上）； + - 落点：`data_engine.update_step_summary`；状态机（pending/ready/failed）与 + 版本化按姊妹文档 §7.1；lossless-pending 原则由消费侧共同遵守； + - 素材来源：pre/post 图直接以 `PerceptionRef` 引用（方案 C 的红利：不再在 + dispatch 参数里搬运 bytes，`summarizer.py:92-100` 的 `_step_inputs` 图字节 + 缓存与手动置 None 回收逻辑消失）。 +- **两份摘要契约（lens），各 profile 只跑自己的**： + - `VisualTransitionLens`（Flash）：输入 pre/post 截图（动作红标叠加）+ 动作 + + Controller 结果；产出客观视觉转换，`flash_summarizer.md` 禁判定词契约不变； + - `StepCapsuleLens`（Pro）：输入按姊妹文档 §8.1（决策图 + 可选后图 + 动作 + + Validator 结果 + FA 记录 + 当步思考 + 少量近期摘要）；产出意图/策略/进度 + 胶囊，`summarizer.md` 为语义权威，视觉仅作接地、仍禁自行判定成败 + （姊妹文档 §4.5 事实优先级）。 + - 单 profile 单 lens，`update_step_summary` 单字段无冲突；未来若 Pro 需同时保留 + 视觉转换记录，按姊妹文档 §7.1 版本化元数据加 lens 标记，不在本期范围。 +- Pro 图中 `SummarizerNode` 退化为 `dispatch(StepCapsuleLens)` 一行调用； + Flash `runner.py` 同理换成 `dispatch(VisualTransitionLens)`。 + +### 6.2 两个渲染器，一张策略表 + +渲染器保持各自形态（合理的方言差异，不强行同构）： + +- `compress_flash_messages`：消息列表原地压缩，摘要后端换成共享运行时； +- `build_plan_and_history`：编译式渲染，无改动即消费到视觉接地摘要。 + +把 6 个调用点散落的硬编码 kwargs 收拢为**每 agent 一条声明**，新建 +`artemis/memory/context_policy.py`： + +```python +CONTEXT_POLICIES: dict[str, ContextPolicy] = { + "operator": ContextPolicy(strategy="strict_milestone", last_n_detailed=1, recent_window=3, chronological_last=True), + "planner": ContextPolicy(strategy="strict_milestone", last_n_detailed=2, recent_window=5, chronological_last=True), + "failure_analyzer": ContextPolicy(strategy="strict_milestone", last_n_detailed=1, recent_window=3, for_failure_analyzer=True), + "checker": ContextPolicy(strategy="milestone_whitelist"), + "diagnoser": ContextPolicy(strategy="milestone_whitelist", window_from_config="history_window_steps"), + "committee": ContextPolicy(strategy="milestone_whitelist", last_n_detailed=3), + "outputter": ContextPolicy(strategy="full", last_n_detailed=0), + "summarizer": ContextPolicy(strategy="sliding_window", last_n_detailed=1, min_summaries=10), +} +``` + +这张表与 3.2 的 READS/WRITES manifest 互为镜像：一个声明"看 State 的哪些字段"， +一个声明"看历史的哪个投影"。合起来就是"按 agent 划分可见域"的完整答案， +也是今后调优各 agent 输入质量的唯一改动点。收拢时**逐调用点对拍**：策略表渲染 +输出与现 kwargs 输出必须一致（对拍测试进 CI）。 + +### 6.3 与姊妹文档的配置收敛 + +姊妹文档 §14 主张 Pro 单独 `agent.pro.memory`、不复用 Flash `step_summarizer`。 +本方案在语义层同意（lens 分离），仅在**运行时层**共享： + +```jsonc +"agent": { + "memory": { // 共享运行时：重试/flush/并发 + 策略表覆盖 + "runtime": { "max_concurrency": 1, "retry_limit": 3, "flush_timeout_s": 30 }, + "policies": { "operator": { "recent_window": 3 } } + }, + "flash": { "step_summarizer": { /* VisualTransitionLens：模型、开关（现有键保持兼容） */ } }, + "pro": { "memory": { /* StepCapsuleLens 模型/输入项 + token 预算等，per 姊妹文档 §14 */ } } +} +``` + +姊妹文档的 L3 历史分块、`recall_history`、token 预算不变，全部建立在同一个 +`StepMemoryService` 产物之上。 + +--- + +## 7. 分阶段落地（已按依赖重排，消除 v1 的顺序隐患） + +**Phase 0 — 修 bug（不动架构，可独立合入，零行为风险）** ✅ 已实施 2026-08-31 +1. B1 止血：4 处 `ui_tree` → `latest_ui_hierarchy`（终态在 Phase 2 改快照）。✅ +2. B2：`graph.py` 驳回反馈改走现有 `check_feedback` 通道（条目带 `[planner]` + 前缀，弃写无人读的 validator_messages），`CheckFeedbackPromptComponent` + 提示语按前缀中性化（标题保持 "Verification Findings" 不变，兼容既有断言）。✅ +3. B3：删 `ctx.latest_perception_data` 写入与过期注释。✅ +4. B4 止血：operator update 补回 `indexed_elements`（消除两键不对称；终态在 + Phase 2 收编进 workspace）。✅ +5. B5：`_active_driver` → `PrivateAttr`；新增 `State.initial()` 工厂统一初始状态 + 构造（sdk `_get_graph_state` 与 engine 双 runner 三处收拢单源）； + `ExecutionContext.artemis_context` 显式声明；`graph_runner`/`reactive_runner` + 真机路径缺 ctx 时显式 RuntimeError（替代静默构造空 ctx 后期炸）。 + （task_goal / failure_context 两项经复核为误报，见 §1.2 v2.1 更正。）✅ + +验证：目标子集 99/99 通过；全量 `tests/unit + tests/tools` 1072 通过、 +5 失败均为真实 LLM 联网测试的 API key 环境问题（与改动无关）。 + +**Phase 1 — State 显式化（仅动经台账核销的死键与信封）** ✅ 已实施 2026-08-31 + +实施要点（与下述计划的差异）：信封迁移最终形态为**工具直接返回 ToolMessage** +（承载 content/status/tool_call_id，未传 state 时仍返回纯字符串），未引入独立的 +ToolOutcome 类型——ToolMessage 已是 LangChain 原生形态，语义完全覆盖； +`_process_plan_write` 的拒绝路径返回 status="success" 的 ToolMessage 以逐字节保持 +旧信封行为（SystemMessage 无 status 属性 → 旧路径视为 success）。visibility +manifest 已接线至全部 7 个节点入口（`strict_state`，默认关闭零开销）， +契约测试 tests/unit/graph/test_state_contract.py 11 项覆盖往返/forbid/代理/清单。 +验证：tests/unit 全量 1067 通过 0 失败；tests/tools 仅余 5 个既有 API key +环境失败（与 Phase 0 前完全一致）。测试改写含 test_operator.py 12 处 +asanitize 断言 → 返回值断言、15 个工具测试文件 Command→ToolMessage。 +- 删真死键：`messages`（同步改 `sdk/agent.py:1182-1187`、`:1321-1325`、 + `graph_runner.py:57`、`_get_graph_state`）、`remaining_steps`、`focused_app_info`、 + `device_date`、`subgoal_plan`、`operator_tactical_plan`、`complete_subgoals_by_ids` + （同步改 `operator.py:767` 与 `_get_graph_state`）、`current_agent`（scratchpad + 两处读改固定 `"operator"` 或显式传参）、`operator_replan_reason`（committee 去引用）。 +- **不删**：`short_term_memory`（活跃；移除并入姊妹文档 Phase 1 落地时执行）、 + 感知相关键（归宿在 Phase 2）。 +- 信封迁移：14 个工具写入点 → `ToolOutcome` 直返；`operator.py:388-391` 与 + `tool_wrapper.py:175-176` 拆信封逻辑简化；删 `validator_messages` 通道与 + `asanitize_update`。 +- `check_feedback` → `operator_feedback` 更名（operator prompt + diagnoser 两读者 + 同步）。 +- `extra="forbid"` 上 State + `State(**values)` 往返测试（`sdk/agent.py:696` 路径）； + visibility manifest + StateView 测试强制（先按附录 B 现状放宽，再收紧）。 + +**Phase 2 — PerceptionStore + TurnWorkspace** +- store/快照/引用/环形缓存；四个抓屏点收口（§5.1 表）；消费者逐个切换； +- 删 `operator_raw_data`、`latest_ui_hierarchy`、`latest_screenshot`、 + `indexed_points`、`indexed_elements`（State 通道侧）；新增 `perception` ref 与 + `workspace`； +- outputter 帧选择决策（post_action vs 决策帧）落地并记录。 + +**Phase 3 — ctx 分解** +`CheckpointLedger` / `PlanGovernor` / `BackgroundTaskGroup`；`run_outcome` 显式化； +`extra="forbid"` 上 ArtemisContext / DeviceContext / ExecutionSetup / AppLaunchResult。 + +**Phase 4 — 记忆内核** +`StepMemoryService` 统一运行时、双 lens 分立语义（Pro 侧 `StepCapsuleLens` 即 +姊妹文档 Phase 2 的实现载体）；策略表收拢（逐点对拍）；配置收敛。之后按姊妹文档 +继续 L3 分块与 recall。 + +每阶段独立可回滚。Phase 0/1 以台账+现有单测护航；Phase 2 起加对拍测试。 + +--- + +## 8. 验收 + +**不损伤（最低要求）** +- 附录 B 台账全表核销：每个字段/属性的每个消费者都有对应改动与测试；CI 中 + visibility manifest 测试保证幽灵读为零。 +- `State(**values)` 往返（`astream` values 流）在 `forbid` 下全绿。 +- Flash：成功路径下压缩器输出与现版**逐字节对拍一致**；重试上界与 flush 属有意 + 变更，单独测试覆盖（失败注入下有界退出 + 降级态可见）。 +- Pro：`build_plan_and_history` 策略表收拢后与现 6 处调用点输出逐点对拍一致。 +- 性能红线沿用姊妹文档 §18（P50/P95 输入与延迟回归 ≤3–5%，关键路径零摘要等待）。 + +**优雅处理所有信息（目标）** +- 全部 `extra="forbid"`；抓屏管线唯一实现；图通道序列化不再含任何 b64 + （对照：当前 `operator_raw_data` 每超步随 values 流复制）。 +- Planner 驳回后，Operator 与 Diagnoser 的下一 prompt 可见驳回理由（新增回归测试）。 +- validator 的 post-action 帧从"就地写路径后失名"变为可引用证据（store + DataEngine）。 +- Pro 长任务（>30 步）历史出现视觉接地摘要；每个 agent 的输入面由 manifest + + ContextPolicy 两张表完整声明。 + +--- + +## 附录 A：agent → State 读/写实证清单（2026-08-31 工作区） + +（visibility manifest 初始值；来源为本次全仓扫描。） + +| 节点 | 读 | 写 | +|---|---|---| +| perception | structured_decisions | latest_screenshot, latest_ui_hierarchy, operator_raw_data, injected_instruction, user_stop_requested | +| planner | latest_screenshot, initial_goal | （无通道写；就地突变 latest_screenshot） | +| operator | operator_raw_data, latest_ui_hierarchy, indexed_elements, subagent_calls, current_step_id, initial_goal, check_feedback, short_term_memory, operator_tool_limit_exceeded, injected_instruction | structured_decisions, operator_raw_thinking, operator_native_thinking, short_term_memory, indexed_points, complete_subgoals_by_ids, current_step_id, subagent_calls, operator_tool_limit_exceeded | +| execution_check | operator_raw_data, structured_decisions, operator_*_thinking, user_stop_requested, initial_goal | checker_success, validator_messages, structured_decisions, current_step_id, check_feedback, subagent_calls | +| validator | structured_decisions, latest_screenshot, current_step_id, operator_raw_data, operator_*_thinking, indexed_points, indexed_elements | last_execution_result（+就地突变 latest_screenshot, indexed_*） | +| summarizer | current_step_id, structured_decisions, operator_*_thinking, last_execution_result | （无；异步写 DataEngine） | +| exit_settlement | user_stop_requested, initial_goal | exit_settlement_route, check_feedback, run_outcome | +| checker | （不读 State） | （不写 State；ctx.assert_halt + ledger） | +| diagnoser | initial_goal, check_feedback, latest_screenshot, latest_ui_hierarchy | （无） | +| outputter | initial_goal, operator_raw_data | （无） | +| explorer 工具 | operator_raw_data, latest_screenshot, latest_ui_hierarchy, indexed_* | （就地 append indexed_*） | +| failure_analyzer | latest_ui_hierarchy, initial_goal | （就地重推导 indexed_*） | +| committee 工具 | latest_screenshot, initial_goal | （无） | +| video/object_detector/log/history_analyzer | （不读或仅透传；object_detection 兜底读 latest_screenshot） | validator_messages（信封用法） | + +## 附录 B：信息处置台账（"不损伤"核销表） + +### B.1 State 字段 + +| 字段 | 消费者（证据） | 处置 | 阶段 | 同步改动点 | +|---|---|---|---|---| +| initial_goal | 多节点读 | 保留 | — | — | +| injected_instruction | perception 写 / operator prompt 读 | 保留 | — | — | +| user_stop_requested | perception 写 / gate、checkpoints、plan 守卫读 | 保留 | — | — | +| checker_success | execution_check 写 / 两处 gate 读 | 保留 | — | — | +| check_feedback | graph.py:198,230,367 写 / prompts.py:243、diagnoser.py:178 读 | 更名 operator_feedback，扩容纳 planner 驳回 | P0(接入)+P1(更名) | 写 3 处、读 2 处、组件提示语 | +| run_outcome | graph.py:367 写 / sdk/agent.py:720 读 | 保留；ctx 侧动态挂载显式化 | P3 | sdk/agent.py:721 | +| exit_settlement_route | exit_settlement 写 / gate 读 | 保留 | — | — | +| current_step_id / structured_decisions / operator_*_thinking / last_execution_result / subagent_calls / operator_tool_limit_exceeded | 活跃读写对（附录 A） | 保留 | — | — | +| short_term_memory | operator.py:743-765 写 / prompts.py:331 读 | **保留**；移除权属姊妹文档 §3.3 | — | — | +| messages | 写：无（Pro）；读：sdk/agent.py:1182(空转)、:1321(空转)；graph_runner.py:57 写(坏路径) | 删除 | P1 | 读 2 处、写 1 处、_get_graph_state | +| validator_messages | 14 工具信封写 + graph.py:195 跨回合写；通道 0 读 | 信封→ToolOutcome；:195→operator_feedback；删通道 | P0(:195)+P1(信封) | 14 工具 + tool_wrapper:175 + operator:388 + graph:526,595 | +| remaining_steps | 仅 init（sdk:1201）；上限实际由 recursion_limit 承载（sdk:674） | 删除 | P1 | _get_graph_state | +| focused_app_info / device_date | 仅 init（sdk:1197-1198） | 删除 | P1 | _get_graph_state | +| subgoal_plan / operator_tactical_plan | 仅 init / 零引用 | 删除 | P1 | _get_graph_state | +| complete_subgoals_by_ids | operator:767 写 []、init、0 读 | 删除 | P1 | operator:767、_get_graph_state | +| current_agent | scratchpad:166,260 等读（恒回退 "operator"），0 写 | 删除 | P1 | scratchpad 固定值或显式传参 | +| operator_replan_reason | committee_tool:203 读（恒 None），0 写 | 删除 | P1 | committee 去引用 | +| latest_ui_hierarchy | perception 写 / operator:614、diagnoser:193、FA:419、explorer:1595 读 | → snapshot.fused_elements | P2 | 4 读者切 store | +| latest_screenshot | perception/planner/validator/flash 写 / validator:98、diagnoser:186、committee:138、explorer:268、object_detection:93 读 | → snapshot + capture 收口 | P2 | §5.1 表全部触点 | +| operator_raw_data | perception 写 / operator:610、validator:618、explorer:290、outputter:139 读 | → PerceptionRef + store | P2 | 4 读者切 store；outputter 帧选择决策 | +| indexed_points / indexed_elements | 节点内写读（operator/validator/FA/explorer/action_executor，附录 A） | → TurnWorkspace（节点局部） | P2 | §5.3 全部触点 | +| （幽灵）ui_tree | 4 读 0 写 | P0 止血→latest_ui_hierarchy ✅；P2 终态 | P0/P2 | operator:995、tool_declarations:477、action_executor:401、exec_tools:548 | +| （更正）failure_context | engine/BaseAgent 栈自有协议，`.get()` 有 isinstance 防护 | 误报，不动 | — | — | +| （更正）task_goal | `ExecutionContextState.task_goal` 存在（core/state.py:48） | 误报，不动 | — | — | +| （幽灵）metadata | sdk/types/task.py:223 对 finalize 快照 dict.get（恒 None） | 无害；随 finalize 清理顺手删 | P1 | 1 处 | + +### B.2 ArtemisContext 属性 + +| 属性 | 消费者 | 处置 | 阶段 | +|---|---|---|---| +| latest_perception_data | 0 读（HEAD 版 checker 曾读，工作区已删） | 删除写入 + 注释 | P0 | +| run_outcome（动态） | sdk:1107 读 → trace 后缀 | 显式字段 | P3 | +| _active_driver | drivers/factory:75-78、tools/base:104、mock:55、4 处测试 | → PrivateAttr | P0 | +| pending_checkpoints / checkpoint_tasks / checkpoint_attempt_seq / checkpoint_repairs / final_check_attempts / assert_halt | graph/checkpoints.py 族 | → CheckpointLedger | P3 | +| task_plan_content_before / last_validated_plan / pending_validated_plan / planner_task | graph.py:_process_plan_write / execution_check | → PlanGovernor | P3 | +| background_tasks / background_jobs / grace_period | ctx.__aexit__、summarizer:66、diagnoser:234 | → BackgroundTaskGroup | P3 | +| 其余声明字段 | 基础设施 | 保留（分组） | P3 | +| （engine 侧）ExecutionContext.artemis_context | graph_runner:48、reactive_runner:57 读，0 写 | 显式接线或显式报错 | P0 | diff --git a/mcp_server/README.md b/mcp_server/README.md index a8469ab..e994757 100644 --- a/mcp_server/README.md +++ b/mcp_server/README.md @@ -69,7 +69,7 @@ When you run `uv run artemis mcp --install all` (or target a specific IDE like ` ## 🛠️ MCP Tools Overview * **`mobile_run_task`**: Asynchronously launches an autonomous mobile automation task (`Flash` or `Pro` model) with optional `device_serial` targeting. -* **`mobile_manage_task`**: Manages task lifecycle (`status`, `stop`, `inject_instruction`), returning task state and assigned `device_serial`. +* **`mobile_manage_task`**: Manages task lifecycle (`status`, `stop`, `inject_instruction`), returning task state and assigned `device_serial`. Pass `release_loop=True` with `inject_instruction` to gracefully end a `[Loop:continuous]` monitoring task — this explicit signal (not "please stop" wording) is what unlocks the loop milestone's completion. * **`mobile_get_device_state`**: Real-time observer (`screenshot` or OCR+XML `hierarchy`) with optional `device_serial`. * **`mobile_inspect_trace`**: Granular trace inspection, visual action overlays, agent reasoning, and `device_serial` tracking. diff --git a/mcp_server/tools/task_manager.py b/mcp_server/tools/task_manager.py index 02883ae..6f80fe6 100644 --- a/mcp_server/tools/task_manager.py +++ b/mcp_server/tools/task_manager.py @@ -192,7 +192,10 @@ def _mark_liveness_failure(trace_id: str, status_data: dict[str, Any]) -> None: @mcp.tool() def mobile_manage_task( - action: str, trace_id: str, instruction: str | None = None + action: str, + trace_id: str, + instruction: str | None = None, + release_loop: bool = False, ) -> dict[str, Any]: """Manages the lifecycle and retrieves the status of a background mobile automation task. @@ -204,23 +207,35 @@ def mobile_manage_task( ### Actions - **'status'**: Returns `trace_id`, `status` ('running'/'completed'/'failed'/ 'cancelled'), `device_serial` (which phone owns this task in multi-device - setups), `task_desc`, `model`, `elapsed_seconds`, and `progress` (Flash: + setups), `task_desc`, `model`, `elapsed_seconds`, a `test_summary` (when + the run declared verification check items: machine-readable + passed/failed/inconclusive/unchecked counts plus failed-item details — + no need to parse report prose), and `progress` (Flash: current turn, latest thought/action — intervene if turns climb without progress or the thought indicates it is stuck; Pro: the active task plan — check it still aligns with your goal). - **'inject_instruction'**: Injects real-time guidance mid-flight when the subagent errs, stalls, or loops. Applied at the start of the next planning turn (Pro) or the next reactive loop (Flash). Requires - `instruction`. + `instruction` (unless `release_loop=True`). To end a continuous + monitoring task ([Loop:continuous] milestone) gracefully, you MUST pass + `release_loop=True` — this is the only signal that unlocks the + milestone's completion; natural-language "please stop" phrasing in + `instruction` is NOT interpreted as a stop signal. - **'stop'**: Forcefully terminates the subagent, immediately halting device interactions and releasing the device. Use when the task is done, - irreparably broken, or running out of control. + irreparably broken, or running out of control. Prefer + `inject_instruction` with `release_loop=True` when a monitoring task + should wind down cleanly instead of being killed. Args: action: `"status"`, `"inject_instruction"`, or `"stop"`. trace_id: The task's session identifier from `mobile_run_task`. - instruction: Guidance string; required for `inject_instruction`, omit - otherwise. + instruction: Guidance string; required for `inject_instruction` + (optional when `release_loop=True`), omit otherwise. + release_loop: With `inject_instruction`: explicit user stop signal + that authorizes the subagent to complete continuous monitoring + loops and finish the task gracefully. """ status_data = trace_store.read_status(trace_id) if not status_data: @@ -298,6 +313,25 @@ def mobile_manage_task( elif current_status == "completed": response["result"] = status_data.get("result") + # Machine-readable test summary (written by exit settlement): callers + # get assertion results without parsing report prose. + for candidate in ( + os.path.join(trace_store.TRACES_DIR, trace_id, "run_outcome.json"), + os.path.join(trace_dir, "run_outcome.json"), + ): + if os.path.exists(candidate): + try: + with open(candidate, encoding="utf-8") as f: + run_outcome = json.load(f) + tests = run_outcome.get("tests") or {} + response["test_summary"] = { + "task_status": run_outcome.get("task_status"), + **tests, + } + except Exception: + pass + break + if current_status in ("running", "pending") and is_alive: progress: dict[str, Any] = {} @@ -491,6 +525,11 @@ def mobile_manage_task( } elif action == "inject_instruction": + if not instruction and release_loop: + instruction = ( + "The user has requested to stop the continuous monitoring loop." + " Wrap up gracefully and complete the task." + ) if not instruction: return { "trace_id": trace_id, @@ -518,16 +557,18 @@ def mobile_manage_task( json.dump( { "instruction": instruction, + "release_loop": bool(release_loop), "timestamp": time.time(), "status": "pending", }, f, indent=2, ) + suffix = " (with release_loop stop signal)" if release_loop else "" return { "trace_id": trace_id, "status": current_status, - "message": f"Successfully injected instruction: '{instruction}'", + "message": f"Successfully injected instruction{suffix}: '{instruction}'", } except Exception as e: return { diff --git a/tests/fixtures/action_surfaces/action_server_manifest.json b/tests/fixtures/action_surfaces/action_server_manifest.json new file mode 100644 index 0000000..968aa5d --- /dev/null +++ b/tests/fixtures/action_surfaces/action_server_manifest.json @@ -0,0 +1,1443 @@ +{ + "click": { + "description": "Tap at a 0-1000 normalized [x, y] coordinate.", + "inputSchema": { + "properties": { + "target": { + "items": { + "type": "integer" + }, + "title": "Target", + "type": "array" + }, + "times": { + "default": 1, + "title": "Times", + "type": "integer" + }, + "delay_ms": { + "default": 100, + "title": "Delay Ms", + "type": "integer" + } + }, + "required": [ + "target" + ], + "title": "clickArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } + }, + "click_sequence": { + "description": "Tap a series of 0-1000 normalized [x, y] points in one atomic burst.", + "inputSchema": { + "properties": { + "sequence": { + "items": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "title": "Sequence", + "type": "array" + }, + "delay_ms": { + "default": 50, + "title": "Delay Ms", + "type": "integer" + } + }, + "required": [ + "sequence" + ], + "title": "click_sequenceArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } + }, + "long_press": { + "description": "Long-press at a 0-1000 normalized [x, y] coordinate.", + "inputSchema": { + "properties": { + "target": { + "items": { + "type": "integer" + }, + "title": "Target", + "type": "array" + }, + "duration_ms": { + "default": 1000, + "title": "Duration Ms", + "type": "integer" + } + }, + "required": [ + "target" + ], + "title": "long_pressArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } + }, + "input_text": { + "description": "Type text, optionally focusing a 0-1000 normalized [x, y] target first.", + "inputSchema": { + "properties": { + "text": { + "title": "Text", + "type": "string" + }, + "target": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Target" + }, + "clear_exist": { + "default": true, + "title": "Clear Exist", + "type": "boolean" + } + }, + "required": [ + "text" + ], + "title": "input_textArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } + }, + "swipe": { + "description": "Swipe between two 0-1000 normalized [x, y] points.", + "inputSchema": { + "properties": { + "start": { + "items": { + "type": "integer" + }, + "title": "Start", + "type": "array" + }, + "end": { + "items": { + "type": "integer" + }, + "title": "End", + "type": "array" + }, + "duration_ms": { + "default": 800, + "title": "Duration Ms", + "type": "integer" + } + }, + "required": [ + "start", + "end" + ], + "title": "swipeArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } + }, + "press_key": { + "description": "Press a device key (home, back, enter, delete, tab, search, menu, app_switch).", + "inputSchema": { + "properties": { + "key": { + "title": "Key", + "type": "string" + } + }, + "required": [ + "key" + ], + "title": "press_keyArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } + }, + "manage_app": { + "description": "Launch or stop an app by human-readable name or package.", + "inputSchema": { + "properties": { + "action": { + "title": "Action", + "type": "string" + }, + "app_name": { + "title": "App Name", + "type": "string" + } + }, + "required": [ + "action", + "app_name" + ], + "title": "manage_appArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } + }, + "wait_for_delay": { + "description": "Wait for a fixed number of milliseconds.", + "inputSchema": { + "properties": { + "time_in_ms": { + "title": "Time In Ms", + "type": "integer" + } + }, + "required": [ + "time_in_ms" + ], + "title": "wait_for_delayArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } + }, + "wait_for_text": { + "description": "Wait for text to appear on or disappear from the screen.", + "inputSchema": { + "properties": { + "text": { + "title": "Text", + "type": "string" + }, + "wait_state": { + "default": "appear", + "title": "Wait State", + "type": "string" + }, + "timeout_ms": { + "default": 5000, + "title": "Timeout Ms", + "type": "integer" + } + }, + "required": [ + "text" + ], + "title": "wait_for_textArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } + }, + "open_link": { + "description": "Open a URL on the device.", + "inputSchema": { + "properties": { + "url": { + "title": "Url", + "type": "string" + } + }, + "required": [ + "url" + ], + "title": "open_linkArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } + }, + "erase_one_char": { + "description": "Erase a single character in the focused field.", + "inputSchema": { + "properties": {}, + "title": "erase_one_charArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } + }, + "focus_and_clear_text": { + "description": "Focus the field at a 0-1000 normalized [x, y] and clear its text.", + "inputSchema": { + "properties": { + "target": { + "items": { + "type": "integer" + }, + "title": "Target", + "type": "array" + } + }, + "required": [ + "target" + ], + "title": "focus_and_clear_textArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Outcome of one device action, independent of any transport or agent.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode" + }, + "action": { + "title": "Action", + "type": "string" + }, + "message": { + "title": "Message", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Detail" + }, + "normalized_coordinates": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Normalized Coordinates" + }, + "duration_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Ms" + } + }, + "required": [ + "ok", + "code", + "action", + "message" + ], + "title": "ActionResult", + "type": "object" + } + }, + "observe_screen": { + "description": "Capture and index the current screen (adapter-internal).", + "inputSchema": { + "properties": { + "settle_ms": { + "default": 400, + "title": "Settle Ms", + "type": "integer" + }, + "include_image": { + "default": false, + "title": "Include Image", + "type": "boolean" + } + }, + "title": "observe_screenArguments", + "type": "object" + }, + "outputSchema": { + "$defs": { + "ActionCode": { + "description": "Machine-readable outcome classification for a device action.", + "enum": [ + "OK", + "INVALID_ARGS", + "TARGET_NOT_FOUND", + "DEVICE_ERROR", + "PACKAGE_NOT_FOUND", + "TIMEOUT", + "UNSUPPORTED" + ], + "title": "ActionCode", + "type": "string" + } + }, + "description": "Post-action observation: persisted screenshot plus the parsed element list.", + "properties": { + "ok": { + "title": "Ok", + "type": "boolean" + }, + "code": { + "$ref": "#/$defs/ActionCode", + "default": "OK" + }, + "message": { + "default": "", + "title": "Message", + "type": "string" + }, + "screenshot_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Screenshot Path" + }, + "image_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Image Name" + }, + "width": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Width" + }, + "height": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Height" + }, + "elements_text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Elements Text" + }, + "elements": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Elements", + "type": "array" + }, + "hierarchy_ok": { + "default": true, + "title": "Hierarchy Ok", + "type": "boolean" + } + }, + "required": [ + "ok" + ], + "title": "ObserveResult", + "type": "object" + } + }, + "take_screenshot": { + "description": "Return the current screen as base64 (adapter-internal, no persistence).", + "inputSchema": { + "properties": { + "timeout_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Timeout Ms" + } + }, + "title": "take_screenshotArguments", + "type": "object" + }, + "outputSchema": null + }, + "get_ui_hierarchy": { + "description": "Return the raw UI element hierarchy (adapter-internal).", + "inputSchema": { + "properties": { + "timeout_ms": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Timeout Ms" + } + }, + "title": "get_ui_hierarchyArguments", + "type": "object" + }, + "outputSchema": null + } +} diff --git a/tests/fixtures/action_surfaces/operator_shells.json b/tests/fixtures/action_surfaces/operator_shells.json new file mode 100644 index 0000000..86b61e1 --- /dev/null +++ b/tests/fixtures/action_surfaces/operator_shells.json @@ -0,0 +1,303 @@ +{ + "click": { + "type": "function", + "function": { + "name": "click", + "description": "[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates).", + "parameters": { + "properties": { + "target": { + "anyOf": [ + { + "type": "integer" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + } + ], + "description": "Click target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." + }, + "times": { + "default": 1, + "description": "Number of consecutive clicks on this target. Use this for double-clicks or multi-clicks (e.g. 7 to enter developer mode). Default is 1.", + "type": "integer" + }, + "delay_ms": { + "default": 100, + "description": "Delay in milliseconds between consecutive clicks. Default is 100.", + "type": "integer" + } + }, + "required": [ + "target" + ], + "type": "object" + } + } + }, + "input_text": { + "type": "function", + "function": { + "name": "input_text", + "description": "[ACTION] Type text into the target input field (supports replacing whole text or appending to the end, and multi-line strings with '\\n').", + "parameters": { + "properties": { + "text": { + "description": "The text content to input. Supports multi-line content with '\\n'.", + "type": "string" + }, + "target": { + "anyOf": [ + { + "type": "integer" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + } + ], + "description": "Input target field. Can be an input box element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." + }, + "clear_exist": { + "default": true, + "description": "Whether to clear existing text before typing. True (default): clear/replace entire text. False: append at the end of existing content.", + "type": "boolean" + } + }, + "required": [ + "text", + "target" + ], + "type": "object" + } + } + }, + "swipe": { + "type": "function", + "function": { + "name": "swipe", + "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen.\n\n• Directional Scrolling ('direction'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end'): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.\n\nArgs:\n direction: Smart directional scrolling ('up', 'down', 'left', 'right'). Automatically computes safe swipe vectors, retaining 40% visual overlap: 'up' (reveal content below), 'down' (reveal content above), 'left', 'right'.\n start: Start normalized coordinates [start_x, start_y] in 0-1000 scale.\n end: End normalized coordinates [end_x, end_y] in 0-1000 scale.\n target: Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within.\n gesture: Backward-compatible parameter: direction string OR custom coordinates list [start_x, start_y, end_x, end_y] in 0-1000 scale.\n duration: Optional gesture duration in milliseconds (default 800).", + "parameters": { + "properties": { + "direction": { + "anyOf": [ + { + "enum": [ + "up", + "down", + "left", + "right" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below), 'down' (drags top-to-bottom, scrolling up to reveal content above), 'left' (drags right-to-left, scrolling right), 'right' (drags left-to-right, scrolling left)." + }, + "start": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." + }, + "end": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "End normalized coordinates [end_x, end_y] in 0-1000 scale for precise, local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)." + }, + "target": { + "anyOf": [ + { + "type": "integer" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within." + }, + "gesture": { + "anyOf": [ + { + "enum": [ + "up", + "down", + "left", + "right" + ], + "type": "string" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." + }, + "duration": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional swipe/drag duration in milliseconds (default 800). For drag-and-drop, list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars), set duration >= 1000 (e.g. 1500). If omitted for directional swipe, duration is computed automatically." + } + }, + "type": "object" + } + } + }, + "press_key": { + "type": "function", + "function": { + "name": "press_key", + "description": "[ACTION] Press a physical or virtual system button (e.g.\n\nENTER, BACK, HOME, APP_SWITCH).", + "parameters": { + "properties": { + "key": { + "description": "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH).", + "enum": [ + "ENTER", + "BACK", + "HOME", + "APP_SWITCH" + ], + "type": "string" + } + }, + "required": [ + "key" + ], + "type": "object" + } + } + }, + "manage_app": { + "type": "function", + "function": { + "name": "manage_app", + "description": "[ACTION] Launch or force stop a specified application.", + "parameters": { + "properties": { + "action": { + "description": "The action type.", + "enum": [ + "launch", + "stop" + ], + "type": "string" + }, + "app_name": { + "description": "Display name or package name of the application.", + "type": "string" + } + }, + "required": [ + "action", + "app_name" + ], + "type": "object" + } + } + }, + "wait_for_delay": { + "type": "function", + "function": { + "name": "wait_for_delay", + "description": "[ACTION] Pause execution and wait for a specified duration in milliseconds.\n\nUse this whenever you need time to elapse—whether for UI loading, animations,\nscreen transitions, or longer scheduled delays and intervals specified in the task.", + "parameters": { + "properties": { + "time_in_ms": { + "description": "The exact duration to wait in milliseconds. Accurately convert the required time duration into milliseconds based on your objective or plan (e.g., 2000 for 2s, 5000 for 5s, 60000 for 1 minute, 180000 for 3 minutes, 300000 for 5 minutes).", + "type": "integer" + } + }, + "required": [ + "time_in_ms" + ], + "type": "object" + } + } + }, + "long_press": { + "type": "function", + "function": { + "name": "long_press", + "description": "[ACTION] Long press on the target location on the screen (supports element index or absolute normalized coordinates).", + "parameters": { + "properties": { + "target": { + "anyOf": [ + { + "type": "integer" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + } + ], + "description": "Long press target. Can be an element index number (int, e.g. 3) OR normalized coordinates (list of 2 integers, e.g. [500, 600])." + }, + "duration": { + "default": 1000, + "description": "Long press duration in milliseconds (default 1000).", + "type": "integer" + } + }, + "required": [ + "target" + ], + "type": "object" + } + } + } +} diff --git a/tests/fixtures/action_surfaces/tool_declarations.json b/tests/fixtures/action_surfaces/tool_declarations.json new file mode 100644 index 0000000..394ef97 --- /dev/null +++ b/tests/fixtures/action_surfaces/tool_declarations.json @@ -0,0 +1,412 @@ +{ + "CLICK_TOOL": { + "type": "function", + "function": { + "name": "click", + "description": "[ACTION] Tap/click on the target coordinate on the screen. The screen after click will be returned automatically. For buttons, checkboxes, tabs, icons, items in a list, tap ON the element. For text fields / search bars / input boxes, tap INSIDE the input box to focus it.", + "parameters": { + "type": "object", + "properties": { + "target": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "Normalized coordinates [x, y] in 0-1000 scale." + }, + "times": { + "type": "integer", + "description": "Number of taps to perform (default 1). Set to 2 for double click." + }, + "delay_ms": { + "type": "integer", + "description": "Delay between taps in milliseconds (default 100)." + } + }, + "required": [ + "target" + ] + } + }, + "name": "click", + "description": "[ACTION] Tap/click on the target coordinate on the screen. The screen after click will be returned automatically. For buttons, checkboxes, tabs, icons, items in a list, tap ON the element. For text fields / search bars / input boxes, tap INSIDE the input box to focus it.", + "parameters": { + "type": "object", + "properties": { + "target": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "Normalized coordinates [x, y] in 0-1000 scale." + }, + "times": { + "type": "integer", + "description": "Number of taps to perform (default 1). Set to 2 for double click." + }, + "delay_ms": { + "type": "integer", + "description": "Delay between taps in milliseconds (default 100)." + } + }, + "required": [ + "target" + ] + } + }, + "CLICK_SEQUENCE_TOOL": { + "type": "function", + "function": { + "name": "click_sequence", + "description": "[ACTION] Executes a sequence of taps one by one in order on the specified targets (e.g. [[500, 280], [885, 362]]). The screen will be returned ONLY after all clicks in the sequence have completed.", + "parameters": { + "type": "object", + "properties": { + "sequence": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280]), or a single integer element index." + }, + "description": "List of targets to tap in sequence, e.g. [[500, 280], [885, 362]]." + }, + "delay_ms": { + "type": "integer", + "description": "Delay between consecutive taps in milliseconds (default 50ms)." + } + }, + "required": [ + "sequence" + ] + } + }, + "name": "click_sequence", + "description": "[ACTION] Executes a sequence of taps one by one in order on the specified targets (e.g. [[500, 280], [885, 362]]). The screen will be returned ONLY after all clicks in the sequence have completed.", + "parameters": { + "type": "object", + "properties": { + "sequence": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "Normalized coordinates [x, y] in 0-1000 scale (e.g., [500, 280]), or a single integer element index." + }, + "description": "List of targets to tap in sequence, e.g. [[500, 280], [885, 362]]." + }, + "delay_ms": { + "type": "integer", + "description": "Delay between consecutive taps in milliseconds (default 50ms)." + } + }, + "required": [ + "sequence" + ] + } + }, + "LONG_PRESS_TOOL": { + "type": "function", + "function": { + "name": "long_press", + "description": "[ACTION] Long press on a target coordinate on the screen. The screen after long pressing will be returned automatically.", + "parameters": { + "type": "object", + "properties": { + "target": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "Normalized coordinates [x, y] in 0-1000 scale." + }, + "duration_ms": { + "type": "integer", + "description": "Duration of press in milliseconds (default 1000ms)." + } + }, + "required": [ + "target" + ] + } + }, + "name": "long_press", + "description": "[ACTION] Long press on a target coordinate on the screen. The screen after long pressing will be returned automatically.", + "parameters": { + "type": "object", + "properties": { + "target": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "Normalized coordinates [x, y] in 0-1000 scale." + }, + "duration_ms": { + "type": "integer", + "description": "Duration of press in milliseconds (default 1000ms)." + } + }, + "required": [ + "target" + ] + } + }, + "INPUT_TEXT_TOOL": { + "type": "function", + "function": { + "name": "input_text", + "description": "[ACTION] Type text into an input field on the screen. The screen after typing will be returned automatically. Automatically taps inside the input box at target [x, y] to focus before typing.", + "parameters": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "Text to type into the focused input field." + }, + "target": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "Coordinates [x, y] of the input box in 0-1000 scale." + }, + "clear_exist": { + "type": "boolean", + "description": "Whether to clear existing text in the input box before typing (default True)." + } + }, + "required": [ + "text", + "target" + ] + } + }, + "name": "input_text", + "description": "[ACTION] Type text into an input field on the screen. The screen after typing will be returned automatically. Automatically taps inside the input box at target [x, y] to focus before typing.", + "parameters": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "Text to type into the focused input field." + }, + "target": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "Coordinates [x, y] of the input box in 0-1000 scale." + }, + "clear_exist": { + "type": "boolean", + "description": "Whether to clear existing text in the input box before typing (default True)." + } + }, + "required": [ + "text", + "target" + ] + } + }, + "SWIPE_TOOL": { + "type": "function", + "function": { + "name": "swipe", + "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen. The screen and UI hierarchy after swipe will be returned automatically.\n\n• Directional Scrolling ('direction' or 'action'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end' or coordinates list): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.", + "parameters": { + "type": "object", + "properties": { + "direction": { + "type": "string", + "enum": [ + "up", + "down", + "left", + "right" + ], + "description": "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below), 'down' (drags top-to-bottom, scrolling up to reveal content above), 'left' (drags right-to-left, scrolling right), 'right' (drags left-to-right, scrolling left)." + }, + "start": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale." + }, + "end": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "End normalized coordinates [end_x, end_y] in 0-1000 scale." + }, + "target": { + "description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within." + }, + "action": { + "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." + }, + "duration": { + "type": "integer", + "description": "Optional swipe/drag duration in milliseconds (default 800). For drag-and-drop, list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars), set duration >= 1000 (e.g. 1500). If omitted for directional swipe, duration is computed automatically." + } + } + } + }, + "name": "swipe", + "description": "[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the screen. The screen and UI hierarchy after swipe will be returned automatically.\n\n• Directional Scrolling ('direction' or 'action'): Recommended for general browsing and standard page scrolling in most scenarios. Automatically computes safe swipe vectors and adaptive duration, retains a ~40% visual overlap anchor for zero-omission traversal, and prevents inertial flings. Supports scoping to a sub-container via 'target'. If it fails on certain custom layouts, fall back to specifying exact coordinates ('start' and 'end') directly.\n• Precise Coordinate Gestures ('start', 'end' or coordinates list): Best for local, fine-grained interactions such as adjusting sliders/SeekBars (e.g., volume, brightness, progress bars), drag-and-drop / list reordering, or as a reliable fallback when directional scrolling fails on specific containers. Always drag slightly PAST the target position to overcome touch slop and reliably trigger the update. When setting a slider to Maximum (100%) or Minimum (0%), swipe fully to the extreme boundary.", + "parameters": { + "type": "object", + "properties": { + "direction": { + "type": "string", + "enum": [ + "up", + "down", + "left", + "right" + ], + "description": "Direction for scrolling and swiping: 'up' (drags bottom-to-top, scrolling down to reveal content below), 'down' (drags top-to-bottom, scrolling up to reveal content above), 'left' (drags right-to-left, scrolling right), 'right' (drags left-to-right, scrolling left)." + }, + "start": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "Start normalized coordinates [start_x, start_y] in 0-1000 scale." + }, + "end": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "End normalized coordinates [end_x, end_y] in 0-1000 scale." + }, + "target": { + "description": "Optional target element index (e.g. 2) or container bounds [left, top, right, bottom] to scope the directional swipe within." + }, + "action": { + "description": "Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right') OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale." + }, + "duration": { + "type": "integer", + "description": "Optional swipe/drag duration in milliseconds (default 800). For drag-and-drop, list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars), set duration >= 1000 (e.g. 1500). If omitted for directional swipe, duration is computed automatically." + } + } + } + }, + "PRESS_KEY_TOOL": { + "type": "function", + "function": { + "name": "press_key", + "description": "[ACTION] Press a physical or virtual system button. The screen after pressing will be returned automatically.", + "parameters": { + "type": "object", + "properties": { + "key": { + "type": "string", + "description": "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH)." + } + }, + "required": [ + "key" + ] + } + }, + "name": "press_key", + "description": "[ACTION] Press a physical or virtual system button. The screen after pressing will be returned automatically.", + "parameters": { + "type": "object", + "properties": { + "key": { + "type": "string", + "description": "Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH)." + } + }, + "required": [ + "key" + ] + } + }, + "MANAGE_APP_TOOL": { + "type": "function", + "function": { + "name": "manage_app", + "description": "[ACTION] Launch or force stop a specified application. The screen after launch/stop will be returned automatically.", + "parameters": { + "type": "object", + "properties": { + "action": { + "type": "string", + "description": "The action type ('launch' or 'stop')." + }, + "app_name": { + "type": "string", + "description": "Display name or package name of the application." + } + }, + "required": [ + "action", + "app_name" + ] + } + }, + "name": "manage_app", + "description": "[ACTION] Launch or force stop a specified application. The screen after launch/stop will be returned automatically.", + "parameters": { + "type": "object", + "properties": { + "action": { + "type": "string", + "description": "The action type ('launch' or 'stop')." + }, + "app_name": { + "type": "string", + "description": "Display name or package name of the application." + } + }, + "required": [ + "action", + "app_name" + ] + } + }, + "WAIT_FOR_DELAY_TOOL": { + "type": "function", + "function": { + "name": "wait_for_delay", + "description": "[ACTION] Pause execution and wait for a specified duration in milliseconds. The screen after pause will be returned automatically.", + "parameters": { + "type": "object", + "properties": { + "time_in_ms": { + "type": "integer", + "description": "The exact duration to pause in milliseconds (e.g., 2000 for 2s, 60000 for 1 min, 180000 for 3 mins, 300000 for 5 mins). Convert any required waiting duration into milliseconds." + } + }, + "required": [ + "time_in_ms" + ] + } + }, + "name": "wait_for_delay", + "description": "[ACTION] Pause execution and wait for a specified duration in milliseconds. The screen after pause will be returned automatically.", + "parameters": { + "type": "object", + "properties": { + "time_in_ms": { + "type": "integer", + "description": "The exact duration to pause in milliseconds (e.g., 2000 for 2s, 60000 for 1 min, 180000 for 3 mins, 300000 for 5 mins). Convert any required waiting duration into milliseconds." + } + }, + "required": [ + "time_in_ms" + ] + } + } +} diff --git a/tests/integration/test_failure_analyzer_standalone_youtube.py b/tests/integration/test_failure_analyzer_standalone_youtube.py index 8888443..2148112 100644 --- a/tests/integration/test_failure_analyzer_standalone_youtube.py +++ b/tests/integration/test_failure_analyzer_standalone_youtube.py @@ -79,7 +79,6 @@ async def main(): # Set up State state = State( - messages=[], initial_goal="Play a video in full screen on YouTube in Chrome", latest_ui_hierarchy=screen_data.elements, latest_screenshot=str( @@ -87,11 +86,7 @@ async def main(): data_engine.get_or_create_image(base64.b64decode(screenshot_b64)) ) ), - focused_app_info="", - device_date="", structured_decisions="", - complete_subgoals_by_ids=[], - validator_messages=[], ) # 3. Instantiate and run Failure Analyzer directly diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py index 6499fd6..3c849cd 100644 --- a/tests/tools/conftest.py +++ b/tests/tools/conftest.py @@ -51,6 +51,9 @@ def artemis_context(inputs_dir, mock_adb_client): ctx.execution_setup = MagicMock() ctx.execution_setup.traces_path = str(inputs_dir) + # A bare MagicMock trace_name yields a repr with '<>' characters, which is an + # invalid path component on Windows once it flows into trace dir construction. + ctx.execution_setup.trace_name = "trace_output" # Initialize real DataEngine if the fixture database exists db_path = inputs_dir / "data_engine.db" diff --git a/tests/tools/operator/test_operator_tools.py b/tests/tools/operator/test_operator_tools.py index d5c2cce..8fe4092 100644 --- a/tests/tools/operator/test_operator_tools.py +++ b/tests/tools/operator/test_operator_tools.py @@ -13,56 +13,27 @@ # limitations under the License. from langchain_core.tools import BaseTool -from artemis.agents.operator.operator import OperatorNode import pytest +from artemis.mcp.action_specs import OPERATOR_SHELL_ORDER, operator_shell_tool + @pytest.mark.asyncio -async def test_operator_tools(artemis_context, mock_state): - """Test all tools exposed by the OperatorNode.""" - node = OperatorNode(ctx=artemis_context) +async def test_operator_shell_tools(): + """Every manifest-generated Operator shell is a declaration-only LangChain tool.""" + sample_args = { + "click": {"target": 1, "times": 1, "delay_ms": 100}, + "input_text": {"text": "hello", "target": 1, "clear_exist": True}, + "swipe": {"gesture": "up"}, + "press_key": {"key": "ENTER"}, + "manage_app": {"action": "launch", "app_name": "Settings"}, + "wait_for_delay": {"time_in_ms": 1000}, + "long_press": {"target": 1, "duration": 1000}, + } + assert set(sample_args) == set(OPERATOR_SHELL_ORDER) - click_tool = node._get_click_tool() - assert isinstance(click_tool, BaseTool) - res = click_tool.invoke({"target": 1, "times": 1, "delay_ms": 100}) - assert res == "Action Recorded" - - input_text_tool = node._get_input_text_tool() - assert isinstance(input_text_tool, BaseTool) - res = input_text_tool.invoke({"text": "hello", "target": 1, "clear_exist": True}) - assert res == "Action Recorded" - - swipe_tool = node._get_swipe_tool() - assert isinstance(swipe_tool, BaseTool) - res = swipe_tool.invoke({"gesture": "up"}) - assert res == "Action Recorded" - - press_key_tool = node._get_press_key_tool() - assert isinstance(press_key_tool, BaseTool) - res = press_key_tool.invoke({"key": "ENTER"}) - assert res == "Action Recorded" - - manage_app_tool = node._get_manage_app_tool() - assert isinstance(manage_app_tool, BaseTool) - res = manage_app_tool.invoke({"action": "launch", "app_name": "Settings"}) - assert res == "Action Recorded" - - wait_for_delay_tool = node._get_wait_for_delay_tool() - assert isinstance(wait_for_delay_tool, BaseTool) - res = wait_for_delay_tool.invoke({"time_in_ms": 1000}) - assert res == "Action Recorded" - - long_press_tool = node._get_long_press_tool() - assert isinstance(long_press_tool, BaseTool) - res = long_press_tool.invoke({"target": 1, "duration": 1000}) - assert res == "Action Recorded" - - wait_for_text_tool = node._get_wait_for_text_tool() - assert isinstance(wait_for_text_tool, BaseTool) - res = wait_for_text_tool.invoke({"text": "Success", "state": "visible", "timeout_ms": 5000}) - assert res == "Action Recorded" - - reply_tool = node._get_reply_to_checker_tool() - assert isinstance(reply_tool, BaseTool) - res = reply_tool.invoke({"reasoning": "This is a test reply."}) - assert isinstance(res, str) + for name in OPERATOR_SHELL_ORDER: + tool = operator_shell_tool(name) + assert isinstance(tool, BaseTool) + assert tool.name == name + assert tool.invoke(sample_args[name]) == "Action Recorded" diff --git a/tests/tools/video_analyzer/test_video_analyzer_tools.py b/tests/tools/video_analyzer/test_video_analyzer_tools.py index 85c015d..0507a98 100644 --- a/tests/tools/video_analyzer/test_video_analyzer_tools.py +++ b/tests/tools/video_analyzer/test_video_analyzer_tools.py @@ -20,9 +20,13 @@ from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer @pytest.mark.asyncio -async def test_video_analyzer_tools(artemis_context, mock_state, inputs_dir): +async def test_video_analyzer_tools(artemis_context, mock_state, inputs_dir, tmp_path): artemis_context.device.mobile_platform = "android" artemis_context.adb_client = artemis_context.ui_adb_client + # A leaked MagicMock video_analyzer config coerces every numeric setting to + # 1 (float(MagicMock()) == 1.0), e.g. a 1s model-call timeout. None makes + # VideoAnalyzer fall back to its real defaults. + artemis_context.agent_config.video_analyzer = None agent = VideoAnalyzer(ctx=artemis_context) @@ -70,6 +74,9 @@ async def test_video_analyzer_tools(artemis_context, mock_state, inputs_dir): ) artemis_context._genai_client = agent.client + import shutil + import uuid + mock_session = MagicMock() mock_session.local_video_path = Path(inputs_dir) / "recording.mp4" mock_session.start_time = 0.0 @@ -78,17 +85,40 @@ async def test_video_analyzer_tools(artemis_context, mock_state, inputs_dir): mock_session.process.returncode = None mock_session.android_video_segments = [] mock_session.android_segment_index = 0 + # Real-typed fields consumed by UnifiedMobileController.extract_segment_metadata + # and the pydantic VideoRecordingResult; leaked MagicMock attributes fail + # validation or produce invalid paths. + mock_session.android_segment_records = [] + mock_session.android_segment_started_at = None + mock_session.video_id = uuid.uuid4() + mock_session.generation = 0 + mock_session.sealed_until = None from unittest.mock import AsyncMock sample_video = Path(inputs_dir) / "recording.mp4" mock_driver = MagicMock() mock_driver.stop_video_recording = AsyncMock(return_value=sample_video) + # A bare MagicMock attribute is truthy, which would trip the mock-driver + # short-circuit in extract_segment_metadata and return a nonexistent path. + mock_driver.is_mock = False artemis_context._active_driver = mock_driver - with patch( - "artemis.controllers.android_controller.get_active_session", - return_value=mock_session, + # Screen-recording segments carry no audio track (render_timeline_clip maps + # only the video stream), so real audio extraction can never succeed here. + # Substitute the fixture mp3 as the extracted audio for the audio sub-agent. + audio_copy = tmp_path / "audio_recording.mp3" + shutil.copyfile(Path(inputs_dir) / "audio_recording.mp3", audio_copy) + + with ( + patch( + "artemis.controllers.unified_controller.get_active_session", + return_value=mock_session, + ), + patch( + "artemis.agents.video_analyzer.video_analyzer.extract_audio_from_video", + AsyncMock(return_value=audio_copy), + ), ): result = await agent.exec_extract_segment_metadata(start_time=0.0, end_time=1.0) assert isinstance(result, str) diff --git a/tests/unit/agents/test_check_prompt_assembly.py b/tests/unit/agents/test_check_prompt_assembly.py new file mode 100644 index 0000000..fe7e3c7 --- /dev/null +++ b/tests/unit/agents/test_check_prompt_assembly.py @@ -0,0 +1,176 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""§1.3 assembly discipline: prompts are functions of configuration x scenario. +Both check gates off => zero context pollution; a plan carrying check lines +gets the behavior explainer regardless of switches.""" + +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from artemis.agents.operator.prompts import ( + CheckItemsExplainerPromptComponent, + FeedbackPromptComponent, + PromptBuilder, + TemplatePromptComponent, +) +from artemis.agents.planner.planner import build_planner_system_blocks +from artemis.context import ArtemisContext, ExecutionSetup +from artemis.graph.state import State + +PLANNER_JSON = Path(__file__).resolve().parents[3] / "artemis/agents/planner/planner.json" +OPERATOR_JSON = Path(__file__).resolve().parents[3] / "artemis/agents/operator/operator.json" + + +def _planner_data(): + return json.loads(PLANNER_JSON.read_text(encoding="utf-8")) + + +def test_planner_blocks_without_checks_are_byte_identical_to_legacy(): + data = _planner_data() + baseline = list(data["modes"]["initial_plan"]["system"]) + assert build_planner_system_blocks(data, "initial_plan", include_checks=False) == baseline + # And the disabled assembly never references the check blocks + rendered = "\n\n".join( + data["blocks"][b] + for b in build_planner_system_blocks(data, "initial_plan", include_checks=False) + ) + assert "- verify:" not in rendered + assert "- assert:" not in rendered + + +def test_planner_blocks_with_checks_mount_generation_and_audit(): + data = _planner_data() + initial = build_planner_system_blocks(data, "initial_plan", include_checks=True) + assert initial[-1] == "check_generation" + validator = build_planner_system_blocks(data, "validator", include_checks=True) + assert validator[-1] == "check_audit" + + generation = data["blocks"]["check_generation"] + # Restraint, honesty about post-hoc auditing, and no silent ordering promises + assert "restraint" in generation.lower() or "Do NOT attach" in generation + assert "post-hoc" in generation.lower() + assert "Never silently promise" in generation + + audit = data["blocks"]["check_audit"] + assert "REJECT" in audit + + +def _mock_ctx(tmp_path=None, plan: str | None = None, **setup_kwargs): + ctx = MagicMock(spec=ArtemisContext) + ctx.execution_setup = ExecutionSetup(**setup_kwargs) if setup_kwargs is not None else None + if plan is not None and tmp_path is not None: + notes = tmp_path / "notes" + notes.mkdir(parents=True, exist_ok=True) + (notes / "task_plan.md").write_text(plan, encoding="utf-8") + ctx.data_engine = MagicMock() + ctx.data_engine.base_dir = tmp_path + else: + ctx.data_engine = None + return ctx + + +def _state(**overrides): + state = MagicMock(spec=State) + state.operator_feedback = None + for k, v in overrides.items(): + setattr(state, k, v) + return state + + +@pytest.mark.asyncio +async def test_explainer_renders_iff_plan_has_check_lines(tmp_path): + component = CheckItemsExplainerPromptComponent() + + # Plan without check lines -> nothing rendered + builder = PromptBuilder() + ctx = _mock_ctx(tmp_path, plan="- [ ] G\n") + await component(builder, _state(), ctx) + assert builder.human_parts == [] + + # Plan with check lines -> explainer rendered + builder2 = PromptBuilder() + ctx2 = _mock_ctx(tmp_path, plan="- [ ] G\n - verify: V\n") + await component(builder2, _state(), ctx2) + joined = "\n".join(p for p in builder2.human_parts if isinstance(p, str)) + assert "check lines" in joined + assert "independent Checker" in joined + assert "automatically restored" in joined + + +@pytest.mark.asyncio +async def test_explainer_is_content_driven_not_switch_driven(tmp_path): + """A resumed old plan with check lines still gets the explanation even when + every check switch is off.""" + component = CheckItemsExplainerPromptComponent() + builder = PromptBuilder() + ctx = _mock_ctx(tmp_path, plan="- [ ] G\n - assert: A\n", disable_checker=True) + await component(builder, _state(), ctx) + joined = "\n".join(p for p in builder.human_parts if isinstance(p, str)) + assert "check lines" in joined + + +async def _render_main_template(**setup_kwargs) -> str: + component = TemplatePromptComponent() + builder = PromptBuilder() + ctx = _mock_ctx(**setup_kwargs) + ctx.actuator = None + state = _state(initial_goal="G") + prompts = json.loads(OPERATOR_JSON.read_text(encoding="utf-8")) + await component(builder, state, ctx, prompts=prompts, plan_and_history="") + return "".join(builder.system_parts) + (builder.human_footer or "") + + +@pytest.mark.asyncio +async def test_main_template_diagnosis_trigger_gated_on_verification(): + """The rejection/finding diagnosis trigger is a dead instruction unless a + mechanism that can produce rejections or findings is active (§1.3).""" + # Factory default: checks off (master alias) AND planner validation off. + default = await _render_main_template(disable_checker=True, disable_planner_validation=True) + assert "Verification Finding" not in default + assert "Ambiguous Validation Rejection" not in default + + # Checks on -> the trigger is real and must be documented. + with_checks = await _render_main_template( + disable_checker=False, disable_planner_validation=True + ) + assert "Ambiguous Validation Rejection or Verification Finding" in with_checks + + # Planner validation alone can also produce rejections. + with_validation = await _render_main_template( + disable_checker=True, disable_planner_validation=False + ) + assert "Ambiguous Validation Rejection or Verification Finding" in with_validation + + +@pytest.mark.asyncio +async def test_check_feedback_component_appends_findings_only_when_present(): + component = FeedbackPromptComponent() + + builder = PromptBuilder() + await component(builder, _state(operator_feedback=None), _mock_ctx()) + assert builder.human_parts == [] + + builder2 = PromptBuilder() + await component( + builder2, + _state(operator_feedback=["[verify failed] 'X': missing"]), + _mock_ctx(), + ) + joined = "\n".join(p for p in builder2.human_parts if isinstance(p, str)) + assert "Verification Findings" in joined + assert "[verify failed] 'X': missing" in joined diff --git a/tests/unit/agents/test_checker.py b/tests/unit/agents/test_checker.py index 11d964c..5e30c1f 100644 --- a/tests/unit/agents/test_checker.py +++ b/tests/unit/agents/test_checker.py @@ -14,85 +14,252 @@ from unittest.mock import AsyncMock, MagicMock, patch -from artemis.agents.checker.checker import CheckerResult, run_async_check -from artemis.context import ArtemisContext import pytest +from artemis.agents.checker.checker import ( + CheckReport, + CheckVerdict, + _normalize_report, + assemble_checker_prompt_segments, + build_checker_tools, + build_probe_argv, + run_checkpoint_check, + run_final_check, + verdicts_allow_release, +) +from artemis.context import ArtemisContext, ExecutionSetup +from artemis.graph.checkpoints import EvidenceAnchor +from artemis.utils.plan_grammar import CheckItem + + +def _ci(kind="verify", when="on_complete", text="expected state", parent="p"): + return CheckItem(kind=kind, when=when, text=text, parent_key=parent) + + +def _mock_ctx(**setup_kwargs): + ctx = MagicMock(spec=ArtemisContext) + ctx.execution_setup = ExecutionSetup(**setup_kwargs) + ctx.data_engine = MagicMock() + ctx.data_engine.base_dir = "unused" + ctx.data_engine.get_agent_friendly_steps.return_value = [] + ctx.data_engine.get_step_number.return_value = 7 + return ctx + + +# --- probe_device: enumerated table, programmatic argv (§8 item 12) ------------------ + + +def test_probe_argv_enumerated_kinds_allowed(): + assert build_probe_argv("alarms") == ["dumpsys", "alarm"] + assert build_probe_argv("battery") == ["dumpsys", "battery"] + assert build_probe_argv("foreground") == ["dumpsys", "activity", "activities"] + assert build_probe_argv("packages") == ["pm", "list", "packages"] + assert build_probe_argv("setting", {"namespace": "system", "key": "screen_brightness"}) == [ + "settings", + "get", + "system", + "screen_brightness", + ] + assert build_probe_argv("content", {"uri": "content://settings/system"}) == [ + "content", + "query", + "--uri", + "content://settings/system", + ] + assert build_probe_argv("prop", {"key": "ro.build.version.sdk"}) == [ + "getprop", + "ro.build.version.sdk", + ] + + +def test_probe_argv_rejects_unknown_kind(): + with pytest.raises(ValueError): + build_probe_argv("shell") + with pytest.raises(ValueError): + build_probe_argv("dumpsys") + + +def test_probe_argv_rejects_whitespace_and_metachars(): + for bad in ("a key", "key;rm", "key|x", "key$", "key`x`", "key&&y", "a\nb"): + with pytest.raises(ValueError): + build_probe_argv("prop", {"key": bad}) + with pytest.raises(ValueError): + build_probe_argv("setting", {"namespace": "system", "key": "a b"}) + with pytest.raises(ValueError): + build_probe_argv("content", {"uri": "content://a; rm -rf /"}) + + +def test_probe_argv_battery_set_inexpressible(): + """Mutating dumpsys subcommands cannot be expressed: no-parameter probes + reject every parameter.""" + with pytest.raises(ValueError): + build_probe_argv("battery", {"extra": "set level 100"}) + with pytest.raises(ValueError): + build_probe_argv("alarms", {"args": "anything"}) + # And the setting namespace is a closed set + with pytest.raises(ValueError): + build_probe_argv("setting", {"namespace": "battery", "key": "level"}) + + +# --- Release decision is node-side and never rewrites verdicts ----------------------- + + +def test_verdicts_allow_release_semantics(): + ok = CheckReport( + verdicts=[ + CheckVerdict(item_text="a", kind="verify", status="passed", evidence="e"), + CheckVerdict(item_text="b", kind="verify", status="inconclusive", evidence="e"), + CheckVerdict(item_text="c", kind="assert", status="failed", evidence="e"), + ] + ) + # Assert failures never block release + assert verdicts_allow_release(ok) + + blocked = CheckReport( + verdicts=[ + CheckVerdict(item_text="a", kind="verify", status="failed", evidence="e"), + ] + ) + assert not verdicts_allow_release(blocked) + + +def test_normalize_report_downgrades_vague_failures_and_fills_missing(): + items = [_ci(kind="verify", text="v1"), _ci(kind="assert", text="a1")] + report = CheckReport( + verdicts=[CheckVerdict(item_text="v1", kind="verify", status="failed", evidence=" ")] + ) + normalized = _normalize_report(report, items) + by_text = {(v.kind, v.item_text): v for v in normalized.verdicts} + # Vague failed -> inconclusive (verdict value hygiene, not release logic) + assert by_text[("verify", "v1")].status == "inconclusive" + # Missing item gets an inconclusive verdict, never a silent pass + assert by_text[("assert", "a1")].status == "inconclusive" + + +# --- Assembly: prompt segments x entry x content x config (§8 item 18) --------------- + + +def test_prompt_segments_verify_only_has_no_assert_section(): + prompts = { + "base_rules": "BASE", + "verify_semantics": "VERIFY-SEG", + "assert_semantics": "ASSERT-SEG", + "anchor_guide": "ANCHOR-SEG", + "final_guide": "FINAL-SEG", + "probe_guide": "PROBE-SEG", + } + text = assemble_checker_prompt_segments( + "checkpoint", [_ci(kind="verify")], probe_tool_registered=True, prompts=prompts + ) + assert "VERIFY-SEG" in text + assert "ASSERT-SEG" not in text + assert "ANCHOR-SEG" in text + assert "FINAL-SEG" not in text + assert "PROBE-SEG" in text + + text2 = assemble_checker_prompt_segments( + "final", [_ci(kind="assert")], probe_tool_registered=False, prompts=prompts + ) + assert "ASSERT-SEG" in text2 + assert "VERIFY-SEG" not in text2 + assert "FINAL-SEG" in text2 + assert "ANCHOR-SEG" not in text2 + # Probe disabled -> zero mention of probing anywhere in the prompt + assert "PROBE-SEG" not in text2 + + +def test_tool_table_probe_gate(): + ctx = _mock_ctx(disable_device_probes=False) + names = {t.name for t in build_checker_tools(ctx, "checkpoint")} + assert "probe_device" in names + assert "read_note" in names + assert "get_step_detail" in names + assert "get_step_screenshot" in names + # Never any device action, note write, or sub-agent + assert not names & { + "save_note", + "update_note", + "append_note", + "click", + "swipe", + "ask_diagnoser", + "ask_explorer", + "run_adb_command", + } + + ctx2 = _mock_ctx(disable_device_probes=True) + names2 = {t.name for t in build_checker_tools(ctx2, "checkpoint")} + assert "probe_device" not in names2 + @pytest.mark.asyncio -async def test_run_async_check(tmp_path): - notes_dir = tmp_path / "notes" - notes_dir.mkdir(parents=True, exist_ok=True) - - # Put a dummy task plan file - (notes_dir / "task_plan.md").write_text("Plan") - - # Put a dummy verification_chat file - import hashlib - - subgoal_text = "Open WhatsApp" - s_hash = hashlib.md5(subgoal_text.encode("utf-8")).hexdigest() - (notes_dir / f"verification_chat_{s_hash}.json").write_text("[]") - - mock_ctx = MagicMock(spec=ArtemisContext) - mock_ctx.data_engine = MagicMock() - mock_ctx.data_engine.base_dir = str(tmp_path) - mock_ctx.data_engine.get_agent_friendly_steps.return_value = [] - mock_ctx.execution_setup = None - - # Mock controller for screen data - mock_controller = MagicMock() - mock_controller.take_screenshot = AsyncMock(return_value="dummy_screenshot_base64") - - from artemis.controllers.device_controller import ScreenDataResponse - - mock_screen_data = ScreenDataResponse( - base64="dummy_screenshot_base64", - elements=[], - width=1080, - height=2400, - platform="android", +async def test_checkpoint_entry_never_touches_live_screen(): + """Evidence discipline: the checkpoint entry must not capture the current + screen — its evidence is anchored history plus persistent probes.""" + ctx = _mock_ctx() + report = CheckReport( + verdicts=[CheckVerdict(item_text="x", kind="verify", status="passed", evidence="e")] ) - mock_controller.get_screen_data = AsyncMock(return_value=mock_screen_data) - # Mock LLM response with with_structured_output - mock_llm = MagicMock() - mock_response = MagicMock() - mock_response.content = "Task completed successfully." - mock_response.tool_calls = [] - - async def mock_astream(*args, **kwargs): - yield mock_response - - mock_llm.astream.side_effect = mock_astream - mock_llm.bind_tools.return_value = mock_llm - - mock_structured_llm = MagicMock() - mock_structured_llm.ainvoke = AsyncMock( - return_value=CheckerResult(success=True, reason="Task completed successfully.") - ) - mock_llm.with_structured_output.return_value = mock_structured_llm + structured = MagicMock() + structured.ainvoke = AsyncMock(return_value=report) + llm = MagicMock() + llm.with_structured_output.return_value = structured + response = MagicMock() + response.tool_calls = [] with ( + patch("artemis.agents.checker.checker.get_llm", return_value=llm), patch( - "artemis.agents.checker.checker.UnifiedMobileController", - return_value=mock_controller, + "artemis.agents.checker.checker.acomplete", + new=AsyncMock(return_value=response), ), - patch("artemis.agents.checker.checker.get_llm", return_value=mock_llm), patch( - "artemis.utils.task_tree.build_plan_and_history", - return_value="Task tree", - ), + "artemis.agents.checker.checker._capture_final_screen", + new=AsyncMock(return_value=(None, "SHOULD NOT BE CALLED")), + ) as capture_mock, ): - result = await run_async_check( - mock_ctx, - subgoal_text=subgoal_text, - subgoal_hash=s_hash, - raw_perception_data={"screenshot_b64": "dummy", "width": 1080, "height": 2400}, - latest_ui_hierarchy=[], + result = await run_checkpoint_check( + ctx, + check_items=[_ci(text="x")], + anchor=EvidenceAnchor(anchor_step_id="sid", trigger_ts=0.0, plan_text="- [x] G"), + goal="the goal", + subgoal_text="G", ) - assert result["status"] == "success" - assert result["reason"] == "Task completed successfully." - assert mock_llm.astream.called - assert mock_llm.with_structured_output.called + capture_mock.assert_not_awaited() + assert result.verdicts[0].status == "passed" + + +@pytest.mark.asyncio +async def test_final_entry_captures_live_screen(): + ctx = _mock_ctx() + report = CheckReport(verdicts=[]) + + structured = MagicMock() + structured.ainvoke = AsyncMock(return_value=report) + llm = MagicMock() + llm.with_structured_output.return_value = structured + response = MagicMock() + response.tool_calls = [] + + with ( + patch("artemis.agents.checker.checker.get_llm", return_value=llm), + patch( + "artemis.agents.checker.checker.acomplete", + new=AsyncMock(return_value=response), + ), + patch( + "artemis.agents.checker.checker._capture_final_screen", + new=AsyncMock(return_value=("b64img", "elements")), + ) as capture_mock, + ): + await run_final_check( + ctx, + goal="the goal", + plan_text="- [x] G", + ledger=[], + check_items=[_ci(kind="assert", when="at_end", text="no crash", parent=None)], + ) + + capture_mock.assert_awaited_once() diff --git a/tests/unit/agents/test_diagnoser.py b/tests/unit/agents/test_diagnoser.py index 34617de..ab1f1f9 100644 --- a/tests/unit/agents/test_diagnoser.py +++ b/tests/unit/agents/test_diagnoser.py @@ -14,9 +14,7 @@ from unittest.mock import AsyncMock, Mock, patch from langchain_core.messages import AIMessage, ToolMessage -from langgraph.types import Command from artemis.agents.diagnoser.diagnoser import Diagnoser -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.graph.state import State import pytest @@ -148,14 +146,9 @@ async def test_diagnostic_agent_tool_calls( "purpose": None, "state": None, } - mock_command = Command( - update={ - VALIDATOR_MESSAGES_KEY: [ - ToolMessage(content="Video shows UI loading.", tool_call_id="call_2") - ] - } + mock_video_tool.ainvoke.return_value = ToolMessage( + content="Video shows UI loading.", tool_call_id="call_2" ) - mock_video_tool.ainvoke.return_value = mock_command mock_get_video_tool.return_value = mock_video_tool agent = Diagnoser(mock_context) @@ -273,12 +266,6 @@ async def test_diagnoser_background_job_lifecycle( mock_llm.astream.side_effect = mock_astream mock_get_llm.return_value = mock_llm - # Mock asanitize_update on mock_state - async def mock_asanitize_update(ctx, update, agent): - return update - - mock_state.asanitize_update = mock_asanitize_update - # Instantiate the Diagnoser agent = Diagnoser(ctx) diff --git a/tests/unit/agents/test_operator.py b/tests/unit/agents/test_operator.py index 2f956d5..cc7924a 100644 --- a/tests/unit/agents/test_operator.py +++ b/tests/unit/agents/test_operator.py @@ -30,10 +30,7 @@ async def test_operator_node_fast_path(): mock_state = MagicMock() mock_state.subagent_calls = [] - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) - mock_state.subgoal_plan = [] mock_state.initial_goal = "Test goal" - mock_state.operator_tactical_plan = [] # Mock controller and data mock_controller = MagicMock() @@ -62,18 +59,15 @@ async def test_operator_node_fast_path(): # Patch dependencies with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): node = OperatorNode(mock_ctx) - await node(mock_state) + node_update = await node(mock_state) # Verify state was updated with action - mock_state.asanitize_update.assert_called_once() - args, kwargs = mock_state.asanitize_update.call_args - update_dict = kwargs["update"] + update_dict = node_update assert "structured_decisions" in update_dict decisions = json.loads(update_dict["structured_decisions"]) assert decisions[0]["action"] == "tap" assert decisions[0]["coordinates"] == [54, 120] - assert update_dict["complete_subgoals_by_ids"] == [] @pytest.mark.asyncio @@ -241,8 +235,6 @@ async def test_operator_node_multiple_actions(): mock_state = MagicMock() mock_state.subagent_calls = [] - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) - mock_state.subgoal_plan = [] mock_state.initial_goal = "Test goal" mock_controller = MagicMock() @@ -267,11 +259,9 @@ async def test_operator_node_multiple_actions(): with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): node = OperatorNode(mock_ctx) - await node(mock_state) + node_update = await node(mock_state) - mock_state.asanitize_update.assert_called_once() - args, kwargs = mock_state.asanitize_update.call_args - update_dict = kwargs["update"] + update_dict = node_update assert "structured_decisions" in update_dict decisions = json.loads(update_dict["structured_decisions"]) @@ -296,7 +286,6 @@ async def test_operator_node_no_record_step(): mock_state = MagicMock() mock_state.subagent_calls = [] - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) mock_state.initial_goal = "Test goal" mock_controller = MagicMock() @@ -314,13 +303,11 @@ async def test_operator_node_no_record_step(): with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): node = OperatorNode(mock_ctx) - await node(mock_state) + node_update = await node(mock_state) mock_ctx.data_engine.record_step.assert_not_called() - mock_state.asanitize_update.assert_called_once() - args, kwargs = mock_state.asanitize_update.call_args - update_dict = kwargs["update"] + update_dict = node_update assert "structured_decisions" in update_dict @@ -336,7 +323,6 @@ async def test_operator_dynamic_prompt(): mock_state = MagicMock() mock_state.subagent_calls = [] mock_state.initial_goal = "Test goal" - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) class MockComponent(PromptComponent): async def __call__(self, builder, state, ctx, **kwargs): @@ -367,7 +353,7 @@ async def test_operator_dynamic_prompt(): with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): node = OperatorNode(mock_ctx, prompt_components=[MockComponent()]) - await node(mock_state) + node_update = await node(mock_state) @pytest.mark.asyncio @@ -443,7 +429,6 @@ async def test_operator_history_compression(): mock_state = MagicMock() mock_state.subagent_calls = [] mock_state.initial_goal = "Test goal" - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) mock_controller = MagicMock() mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=") @@ -465,10 +450,6 @@ async def test_operator_history_compression(): "Goals: {{ initial_goal }}\nPlan:\n{{ plan_and_history" " }}\n# CURRENT OBSERVATION" ), - "troubleshooter_template": ( - "Goals: {{ initial_goal }}\nPlan:\n{{ plan_and_history" - " }}\n# CURRENT OBSERVATION" - ), } ) elif "task_plan.md" in path_str: @@ -508,7 +489,7 @@ async def test_operator_history_compression(): patch("pathlib.Path.read_text", mock_read_text), ): node = OperatorNode(mock_ctx, last_n_detailed=5) - await node(mock_state) + node_update = await node(mock_state) @pytest.mark.asyncio @@ -528,7 +509,7 @@ async def test_operator_loop_detection(): node = OperatorNode(mock_ctx) with pytest.raises(RuntimeError) as exc_info: - await node(mock_state) + node_update = await node(mock_state) assert "Infinite loop detected" in str(exc_info.value) @@ -546,7 +527,6 @@ async def test_operator_tracks_subagent_calls(): mock_state = MagicMock() mock_state.subagent_calls = ["ask_explorer"] mock_state.initial_goal = "Test goal" - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) mock_controller = MagicMock() mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=") @@ -589,11 +569,9 @@ async def test_operator_tracks_subagent_calls(): ), ): node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool]) - await node(mock_state) + node_update = await node(mock_state) - mock_state.asanitize_update.assert_called_once() - args, kwargs = mock_state.asanitize_update.call_args - update_dict = kwargs["update"] + update_dict = node_update assert "subagent_calls" in update_dict assert update_dict["subagent_calls"] == [ @@ -615,7 +593,6 @@ async def test_operator_defer_action_for_gathering(): mock_state = MagicMock() mock_state.subagent_calls = [] - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) mock_state.initial_goal = "Test goal" mock_controller = MagicMock() @@ -684,15 +661,13 @@ async def test_operator_defer_action_for_gathering(): ), ): node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool]) - await node(mock_state) + node_update = await node(mock_state) # Subagent tool should have been executed mock_call_receiver.assert_called_once_with(query="check something", state=mock_state) # Verify final state has the action execution from the second response - mock_state.asanitize_update.assert_called_once() - args, kwargs = mock_state.asanitize_update.call_args - update_dict = kwargs["update"] + update_dict = node_update assert "structured_decisions" in update_dict decisions = json.loads(update_dict["structured_decisions"]) @@ -701,100 +676,6 @@ async def test_operator_defer_action_for_gathering(): assert decisions[0]["coordinates"] == [54, 120] -@pytest.mark.asyncio -async def test_operator_defer_reply_to_checker_for_gathering(): - from artemis.agents.operator.operator import OperatorNode - from artemis.context import ArtemisContext - from unittest.mock import AsyncMock, MagicMock, patch - - mock_ctx = MagicMock(spec=ArtemisContext) - mock_ctx.execution_setup = None - from pathlib import Path - - mock_ctx.data_engine = MagicMock() - mock_ctx.data_engine.base_dir = Path("/tmp/test_session") - mock_ctx.data_engine.get_agent_friendly_steps.return_value = [] - - mock_state = MagicMock() - mock_state.subagent_calls = [] - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) - mock_state.initial_goal = "Test goal" - - mock_llm = MagicMock() - - # First response calls ask_diagnoser and reply_to_checker - mock_response_1 = MagicMock() - mock_response_1.tool_calls = [ - {"name": "ask_diagnoser", "args": {"query": "test"}, "id": "call_diag"}, - { - "name": "reply_to_checker", - "args": {"reasoning": "I solved it"}, - "id": "call_reply", - }, - ] - - # Second response stops without action tools - mock_response_2 = MagicMock() - mock_response_2.tool_calls = [] - - responses = [mock_response_1, mock_response_2] - call_count = 0 - - async def mock_ainvoke(*args, **kwargs): - nonlocal call_count - if call_count < len(responses): - res = responses[call_count] - call_count += 1 - return res - - mock_llm.ainvoke.side_effect = mock_ainvoke - mock_llm.bind_tools.return_value = mock_llm - - from langgraph.prebuilt import InjectedState - from typing import Annotated - - mock_diagnoser_tool = MagicMock() - mock_diagnoser_tool.name = "ask_diagnoser" - mock_diagnoser_tool.args = {"state": None} - - mock_call_receiver = Mock() - - async def dummy_diagnoser_func(query: str, state: Annotated[MagicMock, InjectedState]): - mock_call_receiver(query=query, state=state) - return "Diagnostics completed" - - mock_diagnoser_tool.coroutine = dummy_diagnoser_func - mock_diagnoser_tool.func = None - - with ( - patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm), - patch( - "artemis.agents.operator.operator.trace_langchain_tool", - side_effect=lambda t, ctx: t, - ), - patch( - "artemis.agents.operator.operator.OperatorNode._has_checker_feedback", - return_value=True, - ), - patch( - "artemis.agents.operator.operator.OperatorNode._get_verification_chat_rounds", - return_value=(1, 1), - ), - ): - node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool]) - await node(mock_state) - - # Subagent was called - mock_call_receiver.assert_called_once() - - # Verify reply to checker was NOT executed / flagged - mock_state.asanitize_update.assert_called_once() - args, kwargs = mock_state.asanitize_update.call_args - update_dict = kwargs["update"] - - assert update_dict.get("operator_replied") is False - - @pytest.mark.asyncio async def test_wait_actions_translation(): from artemis.agents.operator.operator import OperatorNode @@ -907,7 +788,6 @@ async def test_operator_no_defer_for_note_updating(): mock_state = MagicMock() mock_state.subagent_calls = [] - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) mock_state.initial_goal = "Test goal" mock_controller = MagicMock() @@ -965,15 +845,13 @@ async def test_operator_no_defer_for_note_updating(): ), ): node = OperatorNode(mock_ctx, tools=[mock_update_note_tool]) - await node(mock_state) + node_update = await node(mock_state) # The update_note tool should have been executed immediately mock_call_receiver.assert_called_once_with(key="progress", content="step 1") # The click action should NOT be deferred, so it should be in structured_decisions - mock_state.asanitize_update.assert_called_once() - args, kwargs = mock_state.asanitize_update.call_args - update_dict = kwargs["update"] + update_dict = node_update assert "structured_decisions" in update_dict decisions = json.loads(update_dict["structured_decisions"]) @@ -995,7 +873,6 @@ async def test_operator_defer_for_note_reading(): mock_state = MagicMock() mock_state.subagent_calls = [] - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) mock_state.initial_goal = "Test goal" mock_controller = MagicMock() @@ -1058,7 +935,7 @@ async def test_operator_defer_for_note_reading(): ), ): node = OperatorNode(mock_ctx, tools=[mock_read_note_tool]) - await node(mock_state) + node_update = await node(mock_state) # The read_note tool should have been executed mock_call_receiver.assert_called_once_with(key="progress") @@ -1068,9 +945,7 @@ async def test_operator_defer_for_note_reading(): # Total call_count should be 2. assert call_count == 2 - mock_state.asanitize_update.assert_called_once() - args, kwargs = mock_state.asanitize_update.call_args - update_dict = kwargs["update"] + update_dict = node_update assert "structured_decisions" in update_dict decisions = json.loads(update_dict["structured_decisions"]) @@ -1092,7 +967,6 @@ async def test_operator_no_defer_for_note_appending(): mock_state = MagicMock() mock_state.subagent_calls = [] - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) mock_state.initial_goal = "Test goal" mock_controller = MagicMock() @@ -1149,15 +1023,13 @@ async def test_operator_no_defer_for_note_appending(): ), ): node = OperatorNode(mock_ctx, tools=[mock_append_note_tool]) - await node(mock_state) + node_update = await node(mock_state) # The append_note tool should have been executed immediately mock_call_receiver.assert_called_once_with(key="progress", content="step 1") # The click action should NOT be deferred, so it should be in structured_decisions - mock_state.asanitize_update.assert_called_once() - args, kwargs = mock_state.asanitize_update.call_args - update_dict = kwargs["update"] + update_dict = node_update assert "structured_decisions" in update_dict decisions = json.loads(update_dict["structured_decisions"]) @@ -1180,7 +1052,6 @@ async def test_operator_background_tasks_prompt_injection(): mock_state = MagicMock() mock_state.subagent_calls = [] mock_state.initial_goal = "Test goal" - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) mock_controller = MagicMock() mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=") @@ -1231,7 +1102,7 @@ async def test_operator_background_tasks_prompt_injection(): active_background_tasks=active_background_tasks, ) captured_messages.extend(messages) - return [], False, None, None, False + return [], None, None, False with ( patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm), @@ -1241,7 +1112,7 @@ async def test_operator_background_tasks_prompt_injection(): ), ): node = OperatorNode(mock_ctx) - await node(mock_state) + node_update = await node(mock_state) # Assert that the captured messages (prompts) contain the background tasks section full_content = "" @@ -1276,7 +1147,6 @@ async def test_operator_task_plan_warning_prompt_injection(): mock_state = MagicMock() mock_state.subagent_calls = [] mock_state.initial_goal = "Test goal" - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) mock_controller = MagicMock() mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=") @@ -1323,7 +1193,7 @@ async def test_operator_task_plan_warning_prompt_injection(): task_plan="", ) captured_messages_1.extend(messages) - return [], False, None, None, False + return [], None, None, False with ( patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm), @@ -1333,7 +1203,7 @@ async def test_operator_task_plan_warning_prompt_injection(): ), ): node = OperatorNode(mock_ctx) - await node(mock_state) + node_update = await node(mock_state) full_content_1 = "" for m in captured_messages_1: @@ -1376,7 +1246,7 @@ async def test_operator_task_plan_warning_prompt_injection(): task_plan="", ) captured_messages_2.extend(messages) - return [], False, None, None, False + return [], None, None, False with ( patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm), @@ -1386,7 +1256,7 @@ async def test_operator_task_plan_warning_prompt_injection(): ), ): node = OperatorNode(mock_ctx) - await node(mock_state) + node_update = await node(mock_state) full_content_2 = "" for m in captured_messages_2: @@ -1418,7 +1288,6 @@ async def test_operator_tool_limit_exceeded_warning(): # Set the flag to True to simulate tool call limit exceeded in previous turn mock_state.operator_tool_limit_exceeded = True mock_state.initial_goal = "Test goal" - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) mock_controller = MagicMock() mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=") @@ -1443,8 +1312,8 @@ async def test_operator_tool_limit_exceeded_warning(): task_plan="", ) captured_messages.extend(messages) - # Return 5 elements: action_result, requested_argue, raw_thinking, native_thinking, tool_limit_exceeded - return [], False, None, None, False + # Return 4 elements: action_result, raw_thinking, native_thinking, tool_limit_exceeded + return [], None, None, False with ( patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm), @@ -1454,7 +1323,7 @@ async def test_operator_tool_limit_exceeded_warning(): ), ): node = OperatorNode(mock_ctx) - await node(mock_state) + node_update = await node(mock_state) full_content = "" for m in captured_messages: @@ -1469,9 +1338,7 @@ async def test_operator_tool_limit_exceeded_warning(): "You did not execute any screen interaction actions in your last turn" in full_content ) - mock_state.asanitize_update.assert_called_once() - _, kwargs = mock_state.asanitize_update.call_args - assert kwargs["update"].get("operator_tool_limit_exceeded") is False + assert node_update.get("operator_tool_limit_exceeded") is False @pytest.mark.asyncio @@ -1488,7 +1355,6 @@ async def test_operator_fallback_function_call_parsing(): mock_state = MagicMock() mock_state.subagent_calls = [] - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) mock_state.initial_goal = "Test goal" mock_controller = MagicMock() @@ -1515,12 +1381,10 @@ async def test_operator_fallback_function_call_parsing(): with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): node = OperatorNode(mock_ctx) - await node(mock_state) + node_update = await node(mock_state) # Verify state was updated with parsed click action - mock_state.asanitize_update.assert_called_once() - _, kwargs = mock_state.asanitize_update.call_args - update_dict = kwargs["update"] + update_dict = node_update assert "structured_decisions" in update_dict decisions = json.loads(update_dict["structured_decisions"]) diff --git a/tests/unit/agents/test_operator_prompts.py b/tests/unit/agents/test_operator_prompts.py index f4dfbe6..eb379b6 100644 --- a/tests/unit/agents/test_operator_prompts.py +++ b/tests/unit/agents/test_operator_prompts.py @@ -29,15 +29,14 @@ def test_operator_prompt_contract_matches_note_runtime_semantics(): assert "memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`" not in prompt -def test_troubleshooter_prompt_has_mutually_exclusive_recovery_branches(): - prompt = apply_operator_prompt_contract(load_operator_prompts()["troubleshooter_template"]) - - assert "invoke `reply_to_checker` directly" in prompt - assert "This branch takes precedence over diagnosis" in prompt - assert "perform that correction directly" in prompt - assert "also takes precedence over generic diagnosis triggers" in prompt - assert "A Checker rejection alone is not a mandatory diagnosis trigger" in prompt - assert "**Validation/Checker Rejection**" not in prompt +def test_operator_prompt_is_single_template_without_checker_dialogue(): + """The prompt is never switched by verification results: main_template is + the only template, and no checker-dialogue machinery is advertised.""" + prompts = load_operator_prompts() + assert set(prompts) == {"main_template"} + prompt = apply_operator_prompt_contract(prompts["main_template"]) + assert "reply_to_checker" not in prompt + assert "Checker" not in prompt def test_operator_prompt_omits_environment_trust_and_explorer_directives(): diff --git a/tests/unit/agents/test_plan_mutation_guardrails.py b/tests/unit/agents/test_plan_mutation_guardrails.py index be041b1..5763b06 100644 --- a/tests/unit/agents/test_plan_mutation_guardrails.py +++ b/tests/unit/agents/test_plan_mutation_guardrails.py @@ -12,10 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -from unittest.mock import MagicMock -from artemis.graph.graph import check_plan_mutation_rejections +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from artemis.context import ArtemisContext, ExecutionSetup +from artemis.graph.graph import ( + check_plan_mutation_rejections, + check_unintended_rewrite, + wrap_note_tool, +) from artemis.graph.state import State +CONTINUOUS_PLAN = ( + "- [x] Open app\n" + "- [/] [Loop:continuous] Periodically monitor for new emails " + "(Interval: every 5 minutes)\n" + " - [x] Polling Check #1: Done" +) + def test_check_plan_mutation_incomplete_nested(): content_before = "- [/] Main task\n - [ ] Subtask 1" @@ -26,57 +41,52 @@ def test_check_plan_mutation_incomplete_nested(): def test_check_plan_mutation_delete_continuous_loop(): - content_before = ( - "- [x] Open app\n" - "- [/] [Loop] Periodically monitor for new emails " - "(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n" - " - [x] Polling Check #1: Done" - ) content_after = ( "- [x] Open app\n" "- [x] Monitor for new emails\n" " - [x] Baseline established\n" " - [x] Polling Check #1: Done" ) - res = check_plan_mutation_rejections(content_before, content_after) + res = check_plan_mutation_rejections(CONTINUOUS_PLAN, content_after) assert res is not None - assert "cannot delete an active [Loop]" in res + assert "cannot delete an active [Loop:continuous]" in res def test_check_plan_mutation_mark_continuous_loop_completed(): - content_before = ( - "- [x] Open app\n" - "- [/] [Loop] Periodically monitor for new emails " - "(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n" - " - [x] Polling Check #1: Done" - ) - content_after = ( - "- [x] Open app\n" - "- [x] [Loop] Periodically monitor for new emails " - "(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n" - " - [x] Polling Check #1: Done" - ) - res = check_plan_mutation_rejections(content_before, content_after) + content_after = CONTINUOUS_PLAN.replace("- [/] [Loop:continuous]", "- [x] [Loop:continuous]") + res = check_plan_mutation_rejections(CONTINUOUS_PLAN, content_after) assert res is not None - assert "cannot be unilaterally marked as completed [x]" in res + assert "cannot be" in res and "marked as completed [x]" in res def test_check_plan_mutation_user_stopped_allowed(): + content_after = CONTINUOUS_PLAN.replace("- [/] [Loop:continuous]", "- [x] [Loop:continuous]") + state = MagicMock(spec=State) + state.user_stop_requested = True + res = check_plan_mutation_rejections(CONTINUOUS_PLAN, content_after, state=state) + assert res is None + + +def test_check_plan_mutation_stop_wording_is_not_a_signal(): + """Natural-language 'stop' phrasing must NOT unlock the continuous loop — + only the explicit release_loop signal (user_stop_requested) does.""" + content_after = CONTINUOUS_PLAN.replace("- [/] [Loop:continuous]", "- [x] [Loop:continuous]") + state = MagicMock(spec=State) + state.user_stop_requested = False + state.injected_instruction = "Please stop the task now." + res = check_plan_mutation_rejections(CONTINUOUS_PLAN, content_after, state=state) + assert res is not None + + +def test_check_plan_mutation_bounded_loop_completion_allowed(): + """Plain [Loop] milestones are bounded: the model may complete them.""" content_before = ( "- [x] Open app\n" - "- [/] [Loop] Periodically monitor for new emails " - "(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n" - " - [x] Polling Check #1: Done" + "- [/] [Loop] Inspect candidate emails (Exit: all 5 candidates inspected)\n" + " - [x] Candidate #5: Done" ) - content_after = ( - "- [x] Open app\n" - "- [x] [Loop] Periodically monitor for new emails " - "(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n" - " - [x] Polling Check #1: Done" - ) - state = MagicMock(spec=State) - state.injected_instruction = "Please stop the task now." - res = check_plan_mutation_rejections(content_before, content_after, state=state) + content_after = content_before.replace("- [/] [Loop]", "- [x] [Loop]") + res = check_plan_mutation_rejections(content_before, content_after) assert res is None @@ -85,3 +95,209 @@ def test_check_plan_mutation_normal_bounded_task(): content_after = "- [x] Open Settings\n- [x] Toggle WiFi" res = check_plan_mutation_rejections(content_before, content_after) assert res is None + + +def test_check_unintended_rewrite_detects_hand_slip(): + content_before = "- [x] Open Settings app\n- [/] Toggle the WiFi switch" + # Full rewrite: milestone 1's status unchanged but text drifted + content_after = "- [x] Open the Settings application\n- [/] Toggle the WiFi switch" + res = check_unintended_rewrite(content_before, content_after) + assert res is not None + assert "Open Settings app" in res + assert "update_note" in res + + +def test_check_unintended_rewrite_allows_status_flips_and_appends(): + content_before = "- [x] Open Settings\n- [/] Toggle WiFi" + content_after = "- [x] Open Settings\n- [x] Toggle WiFi\n - [x] Confirmed enabled" + assert check_unintended_rewrite(content_before, content_after) is None + + +def test_check_unintended_rewrite_ignores_structural_replans(): + # Milestone count changes are declared replans, judged by planner validation + content_before = "- [x] Open Settings\n- [/] Toggle WiFi" + content_after = "- [x] Open Settings\n- [/] Toggle WiFi\n- [ ] Verify connectivity" + assert check_unintended_rewrite(content_before, content_after) is None + + +# --- §4.3 deterministic check-line guard (independent of planner validation) --------- + +GUARDED_PLAN = ( + "- [/] Create the alarm\n" + " - verify: alarm exists in the list\n" + " - assert: a toast appeared\n" + "- [ ] Next milestone\n" +) + + +def _guard_ctx(tmp_path, **setup_kwargs): + # Planner validation OFF by default: the guard must hold on its own. + setup_kwargs.setdefault("disable_planner_validation", True) + setup_kwargs.setdefault("disable_checker", False) + ctx = MagicMock(spec=ArtemisContext) + ctx.execution_setup = ExecutionSetup(**setup_kwargs) + ctx.data_engine = MagicMock() + ctx.data_engine.base_dir = str(tmp_path) + ctx.planner_task = None + ctx.last_validated_plan = None + ctx.pending_validated_plan = None + ctx.task_plan_content_before = None + ctx.pending_checkpoints = [] + ctx.checkpoint_tasks = {} + return ctx + + +def _guard_plan_path(tmp_path, content=GUARDED_PLAN): + notes_dir = tmp_path / "notes" + notes_dir.mkdir(parents=True, exist_ok=True) + path = notes_dir / "task_plan.md" + path.write_text(content, encoding="utf-8") + return path + + +def _save_invoke(path): + async def fake(tool, args, tool_call_id, state, record_trace=None): + path.write_text(args["content"], encoding="utf-8") + return "Success" + + return fake + + +def _save_tool(): + tool = MagicMock() + tool.name = "save_note" + tool.description = "save_note tool" + return tool + + +@pytest.mark.asyncio +async def test_operator_deleting_check_line_is_restored(tmp_path): + path = _guard_plan_path(tmp_path) + ctx = _guard_ctx(tmp_path) + + rewritten = "- [/] Create the alarm\n - assert: a toast appeared\n- [ ] Next milestone\n" + with patch( + "artemis.graph.graph.invoke_tool_with_injection", + side_effect=_save_invoke(path), + ): + wrapped = wrap_note_tool(ctx, _save_tool()) + await wrapped.ainvoke({"key": "task_plan", "content": rewritten, "tool_call_id": "t"}) + + content = path.read_text(encoding="utf-8") + assert "- verify: alarm exists in the list" in content + assert "- assert: a toast appeared" in content + # No planner validation was needed or spawned (guard is deterministic) + assert ctx.planner_task is None + + +@pytest.mark.asyncio +async def test_operator_rewording_check_line_restores_original_keeps_new(tmp_path): + path = _guard_plan_path(tmp_path) + ctx = _guard_ctx(tmp_path) + + rewritten = GUARDED_PLAN.replace( + " - verify: alarm exists in the list\n", + " - verify: alarm roughly configured\n", + ) + with patch( + "artemis.graph.graph.invoke_tool_with_injection", + side_effect=_save_invoke(path), + ): + wrapped = wrap_note_tool(ctx, _save_tool()) + await wrapped.ainvoke({"key": "task_plan", "content": rewritten, "tool_call_id": "t"}) + + content = path.read_text(encoding="utf-8") + # Original standard restored; the Operator's new line survives as an addition + assert "- verify: alarm exists in the list" in content + assert "- verify: alarm roughly configured" in content + + +@pytest.mark.asyncio +async def test_deleted_parent_turns_check_into_task_level_at_end(tmp_path): + path = _guard_plan_path(tmp_path) + ctx = _guard_ctx(tmp_path) + + # Whole parent subgoal removed + rewritten = "- [ ] Next milestone\n" + with patch( + "artemis.graph.graph.invoke_tool_with_injection", + side_effect=_save_invoke(path), + ): + wrapped = wrap_note_tool(ctx, _save_tool()) + await wrapped.ainvoke({"key": "task_plan", "content": rewritten, "tool_call_id": "t"}) + + content = path.read_text(encoding="utf-8") + assert "- verify@end: alarm exists in the list" in content + assert "- assert@end: a toast appeared" in content + + +@pytest.mark.asyncio +async def test_operator_adding_check_lines_is_allowed(tmp_path): + path = _guard_plan_path(tmp_path) + ctx = _guard_ctx(tmp_path) + + extended = GUARDED_PLAN + " - verify: extra criterion\n" + with patch( + "artemis.graph.graph.invoke_tool_with_injection", + side_effect=_save_invoke(path), + ): + wrapped = wrap_note_tool(ctx, _save_tool()) + result = await wrapped.ainvoke( + {"key": "task_plan", "content": extended, "tool_call_id": "t"} + ) + + assert result == "Success" + assert "- verify: extra criterion" in path.read_text(encoding="utf-8") + + +@pytest.mark.asyncio +async def test_guard_holds_with_planner_validation_disabled_and_checks_off(tmp_path): + """The guard is content-driven: it protects check lines even when every + check switch AND planner validation are disabled.""" + path = _guard_plan_path(tmp_path) + ctx = _guard_ctx(tmp_path, disable_checker=True, disable_planner_validation=True) + + with patch( + "artemis.graph.graph.invoke_tool_with_injection", + side_effect=_save_invoke(path), + ): + wrapped = wrap_note_tool(ctx, _save_tool()) + await wrapped.ainvoke( + { + "key": "task_plan", + "content": "- [/] Create the alarm\n- [ ] Next milestone\n", + "tool_call_id": "t", + } + ) + + content = path.read_text(encoding="utf-8") + assert "- verify: alarm exists in the list" in content + assert "- assert: a toast appeared" in content + + +@pytest.mark.asyncio +async def test_planner_node_writes_are_not_guarded(): + """The Planner keeps revision authority over check standards: its node is + wired with the UNWRAPPED note tools, so the guard never applies to it.""" + from artemis.context import DeviceContext, DevicePlatform + from artemis.graph.graph import get_graph + + device = DeviceContext( + host_platform="LINUX", + mobile_platform=DevicePlatform.ANDROID, + device_id="dummy", + device_width=1080, + device_height=2400, + ) + ctx = ArtemisContext(device=device, execution_setup=ExecutionSetup()) + graph = await get_graph(ctx) + + planner_tools = {t.name: t for t in graph.nodes["planner"].bound.afunc.tools} + operator_tools = {t.name: t for t in graph.nodes["operator"].bound.afunc.tools} + + # Operator's plan-writing tools go through the guard wrapper... + assert operator_tools["save_note"].coroutine.__name__ == "wrapped_note_tool" + assert operator_tools["update_note"].coroutine.__name__ == "wrapped_update_note" + # ...the Planner's do not. + assert getattr(planner_tools["save_note"].coroutine, "__name__", "") != "wrapped_note_tool" + assert getattr(planner_tools["update_note"].coroutine, "__name__", "") != "wrapped_update_note" diff --git a/tests/unit/agents/test_planner_node.py b/tests/unit/agents/test_planner_node.py index 38bfb35..2501529 100644 --- a/tests/unit/agents/test_planner_node.py +++ b/tests/unit/agents/test_planner_node.py @@ -32,9 +32,6 @@ class DummyState: self.operator_raw_data = operator_raw_data self.operator_replan_reason = operator_replan_reason - async def asanitize_update(self, ctx, update, agent): - return update - @pytest.fixture def mock_context(): diff --git a/tests/unit/agents/test_snapshot_rollback.py b/tests/unit/agents/test_snapshot_rollback.py deleted file mode 100644 index aefa5b0..0000000 --- a/tests/unit/agents/test_snapshot_rollback.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio -from unittest.mock import AsyncMock, MagicMock, patch - -from artemis.agents.operator.operator import OperatorNode -from artemis.context import ArtemisContext -from artemis.graph.graph import execution_check_node -from artemis.graph.state import State -import pytest - - -@pytest.mark.asyncio -async def test_snapshot_rollback_flow(tmp_path): - # Setup dummy directories - base_dir = tmp_path / "trace" - notes_dir = base_dir / "notes" - notes_dir.mkdir(parents=True) - - task_plan_path = notes_dir / "task_plan.md" - task_plan_path.write_text("Initial Plan", encoding="utf-8") - - # Mock Context - mock_ctx = MagicMock(spec=ArtemisContext) - mock_ctx.data_engine = MagicMock() - mock_ctx.data_engine.base_dir = base_dir - mock_ctx.execution_setup = None - - # Mock State - mock_state = MagicMock(spec=State) - mock_state.subagent_calls = [] - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) - mock_state.initial_goal = "Test goal" - mock_state.short_term_memory = "" - mock_state.operator_raw_data = { - "screenshot_b64": "YmFzZTY0", - "xml_hierarchy": "", - "ocr_results": [], - } - mock_state.current_step_id = "test-step-id" - - # Mock Controller and LLM to let Operator run minimally - mock_controller = MagicMock() - mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0") - mock_device_data = MagicMock() - mock_device_data.elements = [] - mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data) - - mock_llm = MagicMock() - mock_response = MagicMock() - mock_response.tool_calls = [] - - async def mock_astream(*args, **kwargs): - yield mock_response - - mock_llm.astream.side_effect = mock_astream - mock_llm.bind_tools.return_value = mock_llm - - # 1. Run Operator to trigger snapshot - with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): - node = OperatorNode(mock_ctx) - await node(mock_state) - - # Verify snapshot was created - snapshot_dir = notes_dir.with_name("notes_snapshot") - assert snapshot_dir.exists() - assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "Initial Plan" - assert mock_ctx.task_plan_snapshot == snapshot_dir - - # 2. Simulate Operator optimistic modification - task_plan_path.write_text("Modified Plan", encoding="utf-8") - - # 3. Run execution_check_node with failure to trigger rollback - future = asyncio.Future() - future.set_result({"status": "failed"}) - mock_ctx.checker_task = future - - await execution_check_node(mock_state, mock_ctx) - - # Verify rollback restored the file - assert task_plan_path.read_text(encoding="utf-8") == "Initial Plan" - - # 4. Run Operator again to create new snapshot (simulate next attempt) - # First modify it again to simulate operator working on restored plan - task_plan_path.write_text("Modified Plan 2", encoding="utf-8") - - with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): - await node(mock_state) - - assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "Modified Plan 2" - - # 5. Run execution_check_node with success to trigger commit (cleanup) - future_success = asyncio.Future() - future_success.set_result({"status": "success"}) - mock_ctx.checker_task = future_success - - await execution_check_node(mock_state, mock_ctx) - - # Verify snapshot was deleted - assert not snapshot_dir.exists() diff --git a/tests/unit/agents/test_validator.py b/tests/unit/agents/test_validator.py index 6895a03..38916f9 100644 --- a/tests/unit/agents/test_validator.py +++ b/tests/unit/agents/test_validator.py @@ -37,18 +37,13 @@ class DummyState: def __init__( self, structured_decisions, - validator_messages=[], current_step_id=None, latest_screenshot=None, ): self.structured_decisions = structured_decisions - self.validator_messages = validator_messages self.current_step_id = current_step_id self.latest_screenshot = latest_screenshot - async def asanitize_update(self, ctx, update, agent): - return update - class FakeActionSession: """Test double for artemis.mcp.action_session.ActionSession.""" @@ -146,6 +141,7 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot): # entirely. Its hazard class (leaked AnyIO cancel scopes) is now pinned by # tests/unit/mcp/test_action_session.py against the in-process ActionSession. + @pytest.mark.asyncio async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screenshot): """Test that ValidatorNode triggers failure analysis on error and handles diff --git a/tests/unit/data_engine/test_latency_optimizations_deep.py b/tests/unit/data_engine/test_latency_optimizations_deep.py index f8aa5d5..f55b74b 100644 --- a/tests/unit/data_engine/test_latency_optimizations_deep.py +++ b/tests/unit/data_engine/test_latency_optimizations_deep.py @@ -35,7 +35,6 @@ def mock_artemis_ctx(tmp_path): mock_execution_setup.traces_path = str(tmp_path) mock_ctx.execution_setup = mock_execution_setup mock_ctx.device = None - mock_ctx.task_plan_snapshot = None return mock_ctx @@ -225,14 +224,9 @@ async def test_perception_node_async_offloading(mock_artemis_ctx, tmp_path): mock_ocr.return_value = [{"text": "Home", "box": [100, 100, 200, 200]}] state = State( - messages=[HumanMessage(content="test")], initial_goal="test goal", - complete_subgoals_by_ids=[], - validator_messages=[], latest_ui_hierarchy=[], latest_screenshot=None, - focused_app_info=None, - device_date=None, structured_decisions=None, ) @@ -258,42 +252,6 @@ async def test_perception_node_async_offloading(mock_artemis_ctx, tmp_path): assert engine.get_image_path(expected_hash).exists() -@pytest.mark.asyncio -async def test_snapshot_copy_and_restore_offloading(mock_artemis_ctx, tmp_path): - """Verify operator create_snapshot and graph rollback/restore execute asynchronously without blocking.""" - from artemis.utils.file import create_snapshot, restore_snapshot - - notes_dir = tmp_path / "notes" - notes_dir.mkdir(parents=True, exist_ok=True) - (notes_dir / "task_plan.md").write_text("# Plan\n- Step 1", encoding="utf-8") - (notes_dir / "verification_chat_1.json").write_text('{"messages": ["hello"]}', encoding="utf-8") - - snapshot_dir = tmp_path / "notes_snapshot" - - # Test async offloading of create_snapshot (operator pattern) - await asyncio.to_thread(create_snapshot, notes_dir, snapshot_dir) - assert snapshot_dir.exists() - assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "# Plan\n- Step 1" - - # Modify live task_plan.md - (notes_dir / "task_plan.md").write_text("# Plan\n- Step 1 (Modified)", encoding="utf-8") - - # Test async offloading of restore_snapshot (graph pattern when checker fails) - def _restore_and_preserve(): - chat_contents = {} - for p in notes_dir.glob("verification_chat_*.json"): - chat_contents[p.name] = p.read_text(encoding="utf-8") - restore_snapshot(snapshot_dir, notes_dir) - for name, content in chat_contents.items(): - (notes_dir / name).write_text(content, encoding="utf-8") - - await asyncio.to_thread(_restore_and_preserve) - assert (notes_dir / "task_plan.md").read_text(encoding="utf-8") == "# Plan\n- Step 1" - assert (notes_dir / "verification_chat_1.json").read_text( - encoding="utf-8" - ) == '{"messages": ["hello"]}' - - @pytest.mark.asyncio async def test_ocr_api_persistent_http_client_singleton_and_tls_reuse(): """Verify get_http_client returns a persistent singleton httpx.AsyncClient that reuses TLS connections across OCR calls.""" @@ -411,15 +369,10 @@ async def test_validator_pre_execution_loop_reverted_to_exact_safety_contract( ] ) state = State( - messages=[], initial_goal="test", - complete_subgoals_by_ids=[], - validator_messages=[], latest_ui_hierarchy=[], latest_screenshot="dummy.png", structured_decisions=decisions, - focused_app_info=None, - device_date=None, ) with ( diff --git a/tests/unit/graph/test_checkpoints.py b/tests/unit/graph/test_checkpoints.py new file mode 100644 index 0000000..d3be08d --- /dev/null +++ b/tests/unit/graph/test_checkpoints.py @@ -0,0 +1,545 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checkpoint scheduling and harvest tests: queue/spawn separation, supersede +without verdict loss, applicability gating, repair quota, halt policy, +timeouts, unconditional record_step, and fail-open axis separation.""" + +import asyncio +import base64 +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from artemis.agents.checker.checker import CheckReport, CheckVerdict +from artemis.context import ArtemisContext, ExecutionSetup +from artemis.graph.checkpoints import ( + CheckpointRun, + PendingCheckpoint, + harvest_finished_checkpoints, + harvest_run, + queue_checkpoints, + read_ledger, + spawn_pending_checkpoints, +) +from artemis.graph.graph import execution_check_node, wrap_note_tool +from artemis.graph.state import State +from artemis.utils.plan_grammar import CheckItem, parse_plan, subgoal_hash + +GOAL_TEXT = "Create the alarm" +GOAL_KEY = subgoal_hash(GOAL_TEXT) + +PLAN_WITH_CHECKS = ( + f"- [x] {GOAL_TEXT}\n" + " - verify: the alarm list shows 7:30 AM\n" + " - assert: a toast appeared\n" + "- [ ] Next milestone\n" +) + + +def _make_ctx(tmp_path, **setup_kwargs): + setup_kwargs.setdefault("disable_checker", False) + ctx = MagicMock(spec=ArtemisContext) + ctx.execution_setup = ExecutionSetup(**setup_kwargs) + ctx.data_engine = MagicMock() + ctx.data_engine.base_dir = tmp_path + ctx.pending_checkpoints = [] + ctx.checkpoint_tasks = {} + ctx.checkpoint_attempt_seq = {} + ctx.checkpoint_repairs = {} + ctx.assert_halt = False + ctx.final_check_attempts = 0 + ctx.planner_task = None + ctx.last_validated_plan = None + ctx.pending_validated_plan = None + ctx.task_plan_content_before = None + return ctx + + +def _make_state(**overrides): + state = MagicMock(spec=State) + state.initial_goal = "the user goal" + state.user_stop_requested = False + state.operator_feedback = None + state.operator_raw_data = None + state.structured_decisions = None + defaults = { + "operator_raw_thinking": None, + "operator_native_thinking": None, + } + for k, v in {**defaults, **overrides}.items(): + setattr(state, k, v) + return state + + +def _write_plan(tmp_path, content=PLAN_WITH_CHECKS): + notes = tmp_path / "notes" + notes.mkdir(parents=True, exist_ok=True) + path = notes / "task_plan.md" + path.write_text(content, encoding="utf-8") + return path + + +def _pending(plan=PLAN_WITH_CHECKS, kinds=("verify", "assert")): + snapshot = parse_plan(plan) + item = next(i for i in snapshot.top_level if i.key == GOAL_KEY) + items = tuple( + ci for ci in snapshot.check_items_of(item) if ci.when == "on_complete" and ci.kind in kinds + ) + return PendingCheckpoint( + checkpoint_id=GOAL_KEY, + subgoal_text=GOAL_TEXT, + check_items=items, + plan_text=plan, + trigger_ts=1.0, + ) + + +def _done_run(ctx, report_or_exc, attempt_id=f"{GOAL_KEY}#1", pending=None): + loop = asyncio.get_event_loop() + fut = loop.create_future() + if isinstance(report_or_exc, Exception): + fut.set_exception(report_or_exc) + else: + fut.set_result(report_or_exc) + return CheckpointRun(attempt_id=attempt_id, task=fut, checkpoint=pending or _pending()) + + +def _verdict(kind="verify", status="failed", text=None, evidence="concrete evidence"): + default_text = "the alarm list shows 7:30 AM" if kind == "verify" else "a toast appeared" + return CheckVerdict( + item_text=text or default_text, + kind=kind, + status=status, + evidence=evidence, + suggestion="fix it" if (kind == "verify" and status == "failed") else "", + ) + + +# --- §8.1 / §8.2: queueing ------------------------------------------------------------ + + +def test_queue_checkpoints_enqueues_every_completion_with_items(tmp_path): + ctx = _make_ctx(tmp_path) + state = _make_state() + plan = ( + "- [x] A\n - verify: VA\n" + "- [x] B\n - verify: VB\n" + "- [x] C\n" # no check items -> never queued + ) + after = parse_plan(plan) + queue_checkpoints(ctx, state, after, ["A", "B", "C"], plan) + assert [p.checkpoint_id for p in ctx.pending_checkpoints] == [ + subgoal_hash("A"), + subgoal_hash("B"), + ] + # Only queued, never spawned here + assert ctx.checkpoint_tasks == {} + + +def test_queue_checkpoints_ignores_at_end_items(tmp_path): + ctx = _make_ctx(tmp_path) + plan = "- [x] A\n - assert@end: final only\n" + queue_checkpoints(ctx, _make_state(), parse_plan(plan), ["A"], plan) + assert ctx.pending_checkpoints == [] + + +def test_queue_checkpoints_respects_midway_gate_and_user_stop(tmp_path): + plan = "- [x] A\n - verify: VA\n" + after = parse_plan(plan) + + ctx = _make_ctx(tmp_path, disable_midway_checks=True) + queue_checkpoints(ctx, _make_state(), after, ["A"], plan) + assert ctx.pending_checkpoints == [] + + # Legacy master alias also gates + ctx2 = _make_ctx(tmp_path, disable_checker=True) + queue_checkpoints(ctx2, _make_state(), after, ["A"], plan) + assert ctx2.pending_checkpoints == [] + + ctx3 = _make_ctx(tmp_path) + queue_checkpoints(ctx3, _make_state(user_stop_requested=True), after, ["A"], plan) + assert ctx3.pending_checkpoints == [] + + +@pytest.mark.asyncio +async def test_process_plan_write_only_queues(tmp_path): + """The plan-write pipeline enqueues but never spawns: spawning waits for the + turn's step to be recorded.""" + plan_before = PLAN_WITH_CHECKS.replace(f"- [x] {GOAL_TEXT}", f"- [/] {GOAL_TEXT}") + task_plan_path = _write_plan(tmp_path, plan_before) + ctx = _make_ctx(tmp_path) + state = _make_state() + + async def fake_invoke(tool, args, tool_call_id, state, record_trace=None): + task_plan_path.write_text(args["content"], encoding="utf-8") + return "Success" + + mock_tool = MagicMock() + mock_tool.name = "save_note" + mock_tool.description = "save" + + with patch("artemis.graph.graph.invoke_tool_with_injection", side_effect=fake_invoke): + wrapped = wrap_note_tool(ctx, mock_tool) + await wrapped.ainvoke( + {"key": "task_plan", "content": PLAN_WITH_CHECKS, "tool_call_id": "t1"} + ) + + assert len(ctx.pending_checkpoints) == 1 + pc = ctx.pending_checkpoints[0] + assert pc.checkpoint_id == GOAL_KEY + assert [ci.kind for ci in pc.check_items] == ["verify", "assert"] + assert pc.plan_text == PLAN_WITH_CHECKS + assert ctx.checkpoint_tasks == {} + + +# --- §8.1 / §8.10: spawn after record_step; record_step unconditional ---------------- + + +def _raw_data(): + return { + "screenshot_b64": base64.b64encode(b"img").decode(), + "xml_hierarchy": [], + "ocr_results": [], + "width": 1080, + "height": 2400, + } + + +@pytest.mark.asyncio +async def test_execution_check_spawns_after_record_step_with_correct_anchor(tmp_path): + _write_plan(tmp_path) + ctx = _make_ctx(tmp_path) + ctx.data_engine.record_step.return_value = "step-uuid-42" + ctx.pending_checkpoints.append(_pending()) + state = _make_state(operator_raw_data=_raw_data()) + + captured = {} + + async def fake_check(ctx_arg, check_items, anchor, goal, subgoal_text): + captured["anchor"] = anchor + captured["goal"] = goal + return CheckReport(verdicts=[]) + + with ( + patch( + "artemis.agents.checker.checker.run_checkpoint_check", + side_effect=fake_check, + ), + patch("artemis.graph.graph._get_active_subgoal_hashes", return_value=("h", None)), + ): + update = await execution_check_node(state, ctx) + assert ctx.data_engine.record_step.called + assert update["current_step_id"] == "step-uuid-42" + assert GOAL_KEY in ctx.checkpoint_tasks + # Let the spawned task run + await ctx.checkpoint_tasks[GOAL_KEY].task + + assert captured["anchor"].anchor_step_id == "step-uuid-42" + assert captured["anchor"].plan_text == PLAN_WITH_CHECKS + assert captured["goal"] == "the user goal" + + +@pytest.mark.asyncio +async def test_execution_check_no_spawn_after_user_stop(tmp_path): + _write_plan(tmp_path) + ctx = _make_ctx(tmp_path) + ctx.data_engine.record_step.return_value = "sid" + ctx.pending_checkpoints.append(_pending()) + state = _make_state(operator_raw_data=_raw_data(), user_stop_requested=True) + + with patch("artemis.graph.graph._get_active_subgoal_hashes", return_value=("h", None)): + await execution_check_node(state, ctx) + + assert ctx.checkpoint_tasks == {} + # Step is still recorded regardless + assert ctx.data_engine.record_step.called + + +@pytest.mark.asyncio +async def test_record_step_on_planner_rejected_turn(tmp_path): + """Every operator turn leaves a step record — the planner-rejected turn is + recorded too, tagged planner_rejected.""" + _write_plan(tmp_path) + ctx = _make_ctx(tmp_path) + ctx.data_engine.record_step.return_value = "rejected-step" + rejection = asyncio.get_event_loop().create_future() + rejection.set_result({"status": "failed", "feedback": "bad plan"}) + ctx.planner_task = rejection + state = _make_state(operator_raw_data=_raw_data()) + + with patch("artemis.graph.graph._get_active_subgoal_hashes", return_value=("h", None)): + update = await execution_check_node(state, ctx) + + assert update["checker_success"] is False + assert update["current_step_id"] == "rejected-step" + extra = ctx.data_engine.record_step.call_args.kwargs["extra_metadata"] + assert extra.get("planner_rejected") is True + + +@pytest.mark.asyncio +async def test_concurrency_cap_leaves_excess_pending(tmp_path): + _write_plan(tmp_path) + ctx = _make_ctx(tmp_path, max_concurrent_checkpoints=1) + plans = [] + for name in ("A", "B"): + plan = f"- [x] {name}\n - verify: V{name}\n" + plans.append(plan) + snapshot = parse_plan(plan) + item = snapshot.top_level[0] + ctx.pending_checkpoints.append( + PendingCheckpoint( + checkpoint_id=item.key, + subgoal_text=name, + check_items=snapshot.check_items_of(item), + plan_text=plan, + trigger_ts=0.0, + ) + ) + + async def fake_check(*a, **k): + return CheckReport(verdicts=[]) + + with patch("artemis.agents.checker.checker.run_checkpoint_check", side_effect=fake_check): + await spawn_pending_checkpoints(ctx, _make_state(), "sid") + assert len(ctx.checkpoint_tasks) == 1 + assert len(ctx.pending_checkpoints) == 1 + for run in ctx.checkpoint_tasks.values(): + await run.task + + +# --- §8.3: supersede never loses verdicts -------------------------------------------- + + +@pytest.mark.asyncio +async def test_supersede_books_finished_unharvested_attempt_first(tmp_path): + _write_plan(tmp_path) + ctx = _make_ctx(tmp_path) + failed_report = CheckReport(verdicts=[_verdict("verify", "failed")]) + ctx.checkpoint_tasks[GOAL_KEY] = _done_run(ctx, failed_report) + ctx.checkpoint_attempt_seq[GOAL_KEY] = 1 + ctx.pending_checkpoints.append(_pending()) + + async def fake_check(*a, **k): + return CheckReport(verdicts=[]) + + with patch("artemis.agents.checker.checker.run_checkpoint_check", side_effect=fake_check): + await spawn_pending_checkpoints(ctx, _make_state(), "sid") + new_run = ctx.checkpoint_tasks[GOAL_KEY] + await new_run.task + + # The failed verdict was booked before replacement — never dropped + records = read_ledger(tmp_path) + failed = [r for r in records if r["status"] == "failed"] + assert len(failed) == 1 + assert failed[0]["attempt_id"] == f"{GOAL_KEY}#1" + # And a fresh attempt id was allocated + assert new_run.attempt_id == f"{GOAL_KEY}#2" + # Superseded harvest is ledger-only: no plan revert happened + assert "- [x]" in (tmp_path / "notes" / "task_plan.md").read_text(encoding="utf-8") + + +@pytest.mark.asyncio +async def test_supersede_cancels_running_attempt_and_books_superseded(tmp_path): + _write_plan(tmp_path) + ctx = _make_ctx(tmp_path) + + async def hang(): + await asyncio.sleep(3600) + + running = asyncio.create_task(hang()) + await asyncio.sleep(0) # let it start + ctx.checkpoint_tasks[GOAL_KEY] = CheckpointRun( + attempt_id=f"{GOAL_KEY}#1", task=running, checkpoint=_pending() + ) + ctx.checkpoint_attempt_seq[GOAL_KEY] = 1 + ctx.pending_checkpoints.append(_pending()) + + async def fake_check(*a, **k): + return CheckReport(verdicts=[]) + + with patch("artemis.agents.checker.checker.run_checkpoint_check", side_effect=fake_check): + await spawn_pending_checkpoints(ctx, _make_state(), "sid") + await ctx.checkpoint_tasks[GOAL_KEY].task + + assert running.cancelled() + superseded = [r for r in read_ledger(tmp_path) if r["status"] == "superseded"] + assert superseded and superseded[0]["attempt_id"] == f"{GOAL_KEY}#1" + + +# --- §8.4: harvest only takes done(); applicability gates side effects --------------- + + +@pytest.mark.asyncio +async def test_harvest_only_takes_done_tasks(tmp_path): + _write_plan(tmp_path) + ctx = _make_ctx(tmp_path) + + async def hang(): + await asyncio.sleep(3600) + + running = asyncio.create_task(hang()) + await asyncio.sleep(0) + ctx.checkpoint_tasks["running"] = CheckpointRun( + attempt_id="running#1", task=running, checkpoint=_pending() + ) + ctx.checkpoint_tasks[GOAL_KEY] = _done_run( + ctx, CheckReport(verdicts=[_verdict("verify", "passed")]) + ) + + findings = harvest_finished_checkpoints(ctx, _make_state()) + assert findings == [] + # The running attempt is untouched (never awaited, never removed) + assert "running" in ctx.checkpoint_tasks + assert GOAL_KEY not in ctx.checkpoint_tasks + assert not running.done() + running.cancel() + with pytest.raises(asyncio.CancelledError): + await running + + +@pytest.mark.asyncio +async def test_stale_verdict_is_ledger_only_when_subgoal_text_changed(tmp_path): + # Plan no longer contains the anchored subgoal text + _write_plan(tmp_path, "- [x] Create the alarm (rephrased)\n") + ctx = _make_ctx(tmp_path) + run = _done_run(ctx, CheckReport(verdicts=[_verdict("verify", "failed")])) + + findings = harvest_run(ctx, _make_state(), run, allow_side_effects=True) + + assert findings == [] # no repair injection + assert ctx.checkpoint_repairs == {} + records = read_ledger(tmp_path) + assert records and records[0]["status"] == "failed" # but still booked + + +# --- §8.5: verify FAIL repairs, bounded by quota ------------------------------------- + + +@pytest.mark.asyncio +async def test_verify_fail_reverts_subgoal_and_injects_finding(tmp_path): + plan_path = _write_plan(tmp_path) + ctx = _make_ctx(tmp_path) + run = _done_run(ctx, CheckReport(verdicts=[_verdict("verify", "failed")])) + + findings = harvest_run(ctx, _make_state(), run, allow_side_effects=True) + + content = plan_path.read_text(encoding="utf-8") + assert f"- [/] {GOAL_TEXT}" in content # [x] -> [/], forward-looking + assert any("verify failed" in f for f in findings) + assert any("fix it" in f for f in findings) + assert ctx.checkpoint_repairs[GOAL_KEY] == 1 + + +@pytest.mark.asyncio +async def test_verify_fail_beyond_repair_quota_keeps_failed_without_revert(tmp_path): + plan_path = _write_plan(tmp_path) + ctx = _make_ctx(tmp_path, checkpoint_max_repairs=2) + ctx.checkpoint_repairs[GOAL_KEY] = 2 + run = _done_run(ctx, CheckReport(verdicts=[_verdict("verify", "failed")])) + + findings = harvest_run(ctx, _make_state(), run, allow_side_effects=True) + + assert findings == [] + assert f"- [x] {GOAL_TEXT}" in plan_path.read_text(encoding="utf-8") + # The verdict is NOT rewritten: it stays failed in the ledger + assert read_ledger(tmp_path)[0]["status"] == "failed" + + +@pytest.mark.asyncio +async def test_verify_fail_after_user_stop_never_reverts(tmp_path): + plan_path = _write_plan(tmp_path) + ctx = _make_ctx(tmp_path) + run = _done_run(ctx, CheckReport(verdicts=[_verdict("verify", "failed")])) + + harvest_run(ctx, _make_state(user_stop_requested=True), run, allow_side_effects=True) + assert f"- [x] {GOAL_TEXT}" in plan_path.read_text(encoding="utf-8") + + +# --- §8.6: assert FAIL is a result, never a repair ----------------------------------- + + +@pytest.mark.asyncio +async def test_assert_fail_is_ledger_only(tmp_path): + plan_path = _write_plan(tmp_path) + ctx = _make_ctx(tmp_path) + run = _done_run(ctx, CheckReport(verdicts=[_verdict("assert", "failed")])) + + findings = harvest_run(ctx, _make_state(), run, allow_side_effects=True) + + assert findings == [] + assert f"- [x] {GOAL_TEXT}" in plan_path.read_text(encoding="utf-8") + assert ctx.assert_halt is False + assert read_ledger(tmp_path)[0]["status"] == "failed" + + +@pytest.mark.asyncio +async def test_assert_fail_halt_policy_latches_halt(tmp_path): + _write_plan(tmp_path) + ctx = _make_ctx(tmp_path, assert_failure_policy="halt") + run = _done_run(ctx, CheckReport(verdicts=[_verdict("assert", "failed")])) + + harvest_run(ctx, _make_state(), run, allow_side_effects=True) + assert ctx.assert_halt is True + + +@pytest.mark.asyncio +async def test_ledger_is_append_only_fail_then_pass_both_present(tmp_path): + _write_plan(tmp_path) + ctx = _make_ctx(tmp_path) + run1 = _done_run(ctx, CheckReport(verdicts=[_verdict("assert", "failed")])) + harvest_run(ctx, _make_state(), run1, allow_side_effects=True) + run2 = _done_run( + ctx, + CheckReport(verdicts=[_verdict("assert", "passed")]), + attempt_id=f"{GOAL_KEY}#2", + ) + harvest_run(ctx, _make_state(), run2, allow_side_effects=True) + + statuses = [r["status"] for r in read_ledger(tmp_path)] + assert statuses == ["failed", "passed"] # first failure permanently retained + + +# --- §8.7 / §8.13: timeouts and fail-open axis separation ---------------------------- + + +@pytest.mark.asyncio +async def test_timeout_records_inconclusive_not_passed(tmp_path): + _write_plan(tmp_path) + ctx = _make_ctx(tmp_path) + run = _done_run(ctx, TimeoutError()) + + findings = harvest_run(ctx, _make_state(), run, allow_side_effects=True) + + assert findings == [] # released (fail-open): no repair loop + records = read_ledger(tmp_path) + assert {r["status"] for r in records} == {"inconclusive"} + assert all(r["status"] != "passed" for r in records) + + +@pytest.mark.asyncio +async def test_exception_fail_open_releases_but_books_inconclusive(tmp_path): + plan_path = _write_plan(tmp_path) + ctx = _make_ctx(tmp_path) + run = _done_run(ctx, RuntimeError("checker blew up")) + + findings = harvest_run(ctx, _make_state(), run, allow_side_effects=True) + + # Release: no revert, no finding + assert findings == [] + assert f"- [x] {GOAL_TEXT}" in plan_path.read_text(encoding="utf-8") + # Verdict axis: inconclusive, never rewritten to passed + for r in read_ledger(tmp_path): + assert r["status"] == "inconclusive" + assert "checker blew up" in r["evidence"] diff --git a/tests/unit/graph/test_exit_settlement.py b/tests/unit/graph/test_exit_settlement.py new file mode 100644 index 0000000..2a4122f --- /dev/null +++ b/tests/unit/graph/test_exit_settlement.py @@ -0,0 +1,446 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exit settlement tests: gate routing, settlement/final-review decoupling, +settlement timeout, when-semantics at the exit, run outcome, and the four +switch-combination matrix.""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from artemis.agents.checker.checker import CheckReport, CheckVerdict +from artemis.context import ArtemisContext, ExecutionSetup +from artemis.graph.checkpoints import ( + CheckpointRun, + PendingCheckpoint, + append_ledger_record, + compute_run_outcome, + read_ledger, + resolve_item_status, + settle_all_checkpoints, +) +from artemis.graph.graph import ( + convergence_gate, + exit_settlement_gate, + exit_settlement_node, +) +from artemis.graph.state import State +from artemis.sdk.agent import attach_test_summary, resolve_trace_suffix +from artemis.utils.plan_grammar import CheckItem, parse_plan, subgoal_hash + +DONE_PLAN_WITH_CHECKS = ( + "- [x] Create alarm\n" + " - verify: alarm exists\n" + " - assert: toast appeared\n" + "- assert@end: no crash dialog\n" +) + +DONE_PLAN_NO_CHECKS = "- [x] Create alarm\n" + + +def _make_ctx(tmp_path, **setup_kwargs): + setup_kwargs.setdefault("disable_checker", False) + ctx = MagicMock(spec=ArtemisContext) + ctx.execution_setup = ExecutionSetup(**setup_kwargs) + ctx.data_engine = MagicMock() + ctx.data_engine.base_dir = tmp_path + ctx.pending_checkpoints = [] + ctx.checkpoint_tasks = {} + ctx.checkpoint_attempt_seq = {} + ctx.checkpoint_repairs = {} + ctx.assert_halt = False + ctx.final_check_attempts = 0 + return ctx + + +def _make_state(**overrides): + state = MagicMock(spec=State) + state.initial_goal = "the user goal" + state.user_stop_requested = False + state.checker_success = True + for k, v in overrides.items(): + setattr(state, k, v) + return state + + +def _write_plan(tmp_path, content): + notes = tmp_path / "notes" + notes.mkdir(parents=True, exist_ok=True) + (notes / "task_plan.md").write_text(content, encoding="utf-8") + + +def _pending_checkpoint(text="Create alarm"): + return PendingCheckpoint( + checkpoint_id=subgoal_hash(text), + subgoal_text=text, + check_items=( + CheckItem( + kind="verify", + when="on_complete", + text="alarm exists", + parent_key=subgoal_hash(text), + ), + ), + plan_text=f"- [x] {text}\n", + trigger_ts=0.0, + ) + + +def _hanging_run(cid="cid"): + async def hang(): + await asyncio.sleep(3600) + + task = asyncio.create_task(hang()) + return CheckpointRun(attempt_id=f"{cid}#1", task=task, checkpoint=_pending_checkpoint()) + + +# --- §8.8: gate routing — settlement decoupled from final review --------------------- + + +def test_gate_routes_settlement_when_tasks_pending_even_with_final_off(tmp_path): + _write_plan(tmp_path, DONE_PLAN_NO_CHECKS) + ctx = _make_ctx(tmp_path, disable_final_check=True) + ctx.checkpoint_tasks["cid"] = MagicMock() + assert convergence_gate(_make_state(), ctx) == "exit_settlement" + + +def test_gate_routes_empty_plan_to_final_review_when_enabled(tmp_path): + _write_plan(tmp_path, "just prose, no checkboxes\n") + ctx = _make_ctx(tmp_path) # final on (enabled master, default gates) + assert convergence_gate(_make_state(), ctx) == "exit_settlement" + + +def test_gate_ends_directly_when_nothing_check_related(tmp_path): + _write_plan(tmp_path, DONE_PLAN_NO_CHECKS) + ctx = _make_ctx(tmp_path, disable_checker=True) + assert convergence_gate(_make_state(), ctx) == "end" + + +def test_gate_check_items_force_settlement_even_all_switches_off(tmp_path): + """Combination row 1 (off/off): declared items still get settled (booked as + unchecked) instead of silently vanishing.""" + _write_plan(tmp_path, DONE_PLAN_WITH_CHECKS) + ctx = _make_ctx(tmp_path, disable_checker=True) + assert convergence_gate(_make_state(), ctx) == "exit_settlement" + + +def test_gate_incomplete_plan_continues(tmp_path): + _write_plan(tmp_path, "- [/] Create alarm\n - verify: alarm exists\n") + ctx = _make_ctx(tmp_path) + assert convergence_gate(_make_state(), ctx) == "continue" + + +def test_gate_assert_halt_terminates_mid_plan(tmp_path): + _write_plan(tmp_path, "- [/] Create alarm\n - assert: toast appeared\n") + ctx = _make_ctx(tmp_path) + ctx.assert_halt = True + assert convergence_gate(_make_state(), ctx) == "exit_settlement" + + +@pytest.mark.asyncio +async def test_settlement_runs_phase_one_only_when_final_disabled(tmp_path): + _write_plan(tmp_path, DONE_PLAN_WITH_CHECKS) + ctx = _make_ctx(tmp_path, disable_final_check=True) + report = CheckReport( + verdicts=[ + CheckVerdict(item_text="alarm exists", kind="verify", status="passed", evidence="e") + ] + ) + fut = asyncio.get_event_loop().create_future() + fut.set_result(report) + cid = subgoal_hash("Create alarm") + ctx.checkpoint_tasks[cid] = CheckpointRun( + attempt_id=f"{cid}#1", task=fut, checkpoint=_pending_checkpoint() + ) + + with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as final_mock: + update = await exit_settlement_node(_make_state(), ctx) + + final_mock.assert_not_awaited() + assert update["exit_settlement_route"] == "end" + # Phase one still booked the attempt and produced the summary + assert any(r["status"] == "passed" for r in read_ledger(tmp_path)) + assert update["run_outcome"]["tests"]["passed"] == 1 + assert ctx.checkpoint_tasks == {} + + +# --- §8.7: settlement timeout -------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_settlement_timeout_cancels_and_books_unchecked(tmp_path): + _write_plan(tmp_path, DONE_PLAN_NO_CHECKS) + ctx = _make_ctx(tmp_path, settlement_timeout=0.05) + run = _hanging_run() + ctx.checkpoint_tasks["cid"] = run + + await settle_all_checkpoints(ctx, _make_state()) + + assert run.task.cancelled() + records = read_ledger(tmp_path) + assert records and all(r["status"] == "unchecked" for r in records) + assert "settlement timeout" in records[0]["evidence"] + + +# --- §8.9: when-semantics at the exit ------------------------------------------------ + + +@pytest.mark.asyncio +async def test_final_check_receives_ledger_and_at_end_items(tmp_path): + _write_plan(tmp_path, DONE_PLAN_WITH_CHECKS) + ctx = _make_ctx(tmp_path) + append_ledger_record( + tmp_path, + { + "attempt_id": "x#1", + "checkpoint_id": "x", + "item_text": "toast appeared", + "kind": "assert", + "when": "on_complete", + "status": "failed", + "evidence": "no toast in anchor history", + }, + ) + + captured = {} + + async def fake_final(ctx_arg, goal, plan_text, ledger, check_items): + captured["goal"] = goal + captured["ledger"] = ledger + captured["items"] = list(check_items) + return CheckReport(verdicts=[]) + + with patch("artemis.graph.graph.run_final_check", side_effect=fake_final): + await exit_settlement_node(_make_state(), ctx) + + # Final review audits the user's original goal, citing the ledger + assert captured["goal"] == "the user goal" + assert any(r["status"] == "failed" for r in captured["ledger"]) + whens = {(ci.text, ci.when) for ci in captured["items"]} + assert ("no crash dialog", "at_end") in whens + assert ("alarm exists", "on_complete") in whens + + +def test_on_complete_without_history_evidence_never_passes(): + """The 'current state satisfied but historical evidence missing' case: no + substantive record -> unchecked; a final inconclusive verdict stays + inconclusive. Neither ever resolves to passed.""" + assert resolve_item_status("assert", []) == "unchecked" + inconclusive_only = [{"status": "inconclusive", "evidence": "evidence missing for past moment"}] + assert resolve_item_status("assert", inconclusive_only) == "inconclusive" + # First assert failure is permanent even if a later record passes + assert resolve_item_status("assert", [{"status": "failed"}, {"status": "passed"}]) == "failed" + # Verify: latest substantive verdict wins (repair-then-pass is passed) + assert resolve_item_status("verify", [{"status": "failed"}, {"status": "passed"}]) == "passed" + + +# --- §8.14: run outcome + SDK wrap-up ------------------------------------------------ + + +@pytest.mark.asyncio +async def test_assert_failure_yields_failed_tests_but_completed_status(tmp_path): + _write_plan(tmp_path, DONE_PLAN_WITH_CHECKS) + ctx = _make_ctx(tmp_path) + report = CheckReport( + verdicts=[ + CheckVerdict(item_text="alarm exists", kind="verify", status="passed", evidence="e"), + CheckVerdict(item_text="toast appeared", kind="assert", status="failed", evidence="e"), + CheckVerdict(item_text="no crash dialog", kind="assert", status="passed", evidence="e"), + ] + ) + + with patch("artemis.graph.graph.run_final_check", new=AsyncMock(return_value=report)): + update = await exit_settlement_node(_make_state(), ctx) + + # Assert failures never bounce the task back + assert update["exit_settlement_route"] == "end" + outcome = update["run_outcome"] + assert outcome["task_status"] == "completed" + assert outcome["tests"]["failed"] == 1 + assert outcome["tests"]["passed"] == 2 + assert outcome["tests"]["failed_items"][0]["item_text"] == "toast appeared" + + # SDK wrap-up: trace suffix distinguishes assertion failures from _PASS + assert resolve_trace_suffix("completed", outcome) == "_TESTFAIL" + assert resolve_trace_suffix("completed", {"tests": {"failed": 0}}) == "_PASS" + assert resolve_trace_suffix("failed", outcome) == "_FAIL" + + # And the return structure surfaces the machine-readable summary + wrapped = attach_test_summary("report text", outcome) + assert wrapped["result"] == "report text" + assert wrapped["test_summary"]["failed"] == 1 + as_dict = attach_test_summary({"answer": 42}, outcome) + assert as_dict["answer"] == 42 and as_dict["test_summary"]["failed"] == 1 + + +@pytest.mark.asyncio +async def test_verify_unmet_bounces_back_then_blocks_on_budget_exhaustion(tmp_path): + _write_plan(tmp_path, DONE_PLAN_WITH_CHECKS) + ctx = _make_ctx(tmp_path, final_check_max_attempts=2) + failing = CheckReport( + verdicts=[ + CheckVerdict( + item_text="alarm exists", + kind="verify", + status="failed", + evidence="e", + suggestion="create it", + ) + ], + unmet_subgoals=["Create alarm"], + ) + + with patch("artemis.graph.graph.run_final_check", new=AsyncMock(return_value=failing)): + first = await exit_settlement_node(_make_state(), ctx) + assert first["exit_settlement_route"] == "continue" + assert exit_settlement_gate(_make_state(exit_settlement_route="continue")) == "continue" + # The unmet subgoal was reverted forward to in-progress + plan = (tmp_path / "notes" / "task_plan.md").read_text(encoding="utf-8") + assert "- [/] Create alarm" in plan + assert any("verify failed" in f for f in first["operator_feedback"]) + + # Re-complete and exhaust the budget + _write_plan(tmp_path, DONE_PLAN_WITH_CHECKS) + second = await exit_settlement_node(_make_state(), ctx) + + assert ctx.final_check_attempts == 2 + assert second["exit_settlement_route"] == "end" + outcome = second["run_outcome"] + assert outcome["task_status"] == "blocked" + + +def test_no_run_outcome_without_check_material(tmp_path): + """§1.3: the test summary only exists when check items exist.""" + outcome = compute_run_outcome(parse_plan(DONE_PLAN_NO_CHECKS), [], verify_blocked=False) + assert outcome.tests.passed + outcome.tests.failed == 0 + assert attach_test_summary("plain", outcome.model_dump()) == "plain" + + +# --- §8.15: continuous + user_stop + FAIL never loops back --------------------------- + + +@pytest.mark.asyncio +async def test_user_stop_final_fail_does_not_reenter_loop(tmp_path): + plan = "- [x] [Loop:continuous] Monitor inbox\n - verify: monitoring log is complete\n" + _write_plan(tmp_path, plan) + ctx = _make_ctx(tmp_path) + failing = CheckReport( + verdicts=[ + CheckVerdict( + item_text="monitoring log is complete", + kind="verify", + status="failed", + evidence="gaps in the log", + ) + ] + ) + + with patch("artemis.graph.graph.run_final_check", new=AsyncMock(return_value=failing)): + update = await exit_settlement_node(_make_state(user_stop_requested=True), ctx) + + assert update["exit_settlement_route"] == "end" + assert update["run_outcome"]["task_status"] == "blocked" + + +# --- §8.16: four switch-combination matrix ------------------------------------------- + + +def _fresh_final_report(): + return CheckReport( + verdicts=[ + CheckVerdict(item_text="no crash dialog", kind="assert", status="passed", evidence="e") + ] + ) + + +@pytest.mark.asyncio +async def test_matrix_off_off_reports_all_unchecked(tmp_path): + _write_plan(tmp_path, DONE_PLAN_WITH_CHECKS) + ctx = _make_ctx(tmp_path, disable_midway_checks=True, disable_final_check=True) + with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as final_mock: + update = await exit_settlement_node(_make_state(), ctx) + final_mock.assert_not_awaited() + outcome = update["run_outcome"] + assert outcome["tests"]["unchecked"] == 3 + assert outcome["tests"]["passed"] == 0 + + +@pytest.mark.asyncio +async def test_matrix_off_on_final_judges_from_state_and_history(tmp_path): + _write_plan(tmp_path, DONE_PLAN_WITH_CHECKS) + ctx = _make_ctx(tmp_path, disable_midway_checks=True) + report = CheckReport( + verdicts=[ + CheckVerdict(item_text="no crash dialog", kind="assert", status="passed", evidence="e"), + CheckVerdict(item_text="alarm exists", kind="verify", status="passed", evidence="e"), + CheckVerdict( + item_text="toast appeared", + kind="assert", + status="inconclusive", + evidence="transient evidence was never captured", + ), + ] + ) + with patch("artemis.graph.graph.run_final_check", new=AsyncMock(return_value=report)): + update = await exit_settlement_node(_make_state(), ctx) + outcome = update["run_outcome"] + assert outcome["tests"]["passed"] == 2 + assert outcome["tests"]["inconclusive"] == 1 + assert outcome["tests"]["failed"] == 0 + + +@pytest.mark.asyncio +async def test_matrix_on_off_settles_without_final(tmp_path): + # covered structurally by test_settlement_runs_phase_one_only_when_final_disabled; + # here assert the summary is still produced with unchecked leftovers. + _write_plan(tmp_path, DONE_PLAN_WITH_CHECKS) + ctx = _make_ctx(tmp_path, disable_final_check=True) + with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as final_mock: + update = await exit_settlement_node(_make_state(), ctx) + final_mock.assert_not_awaited() + assert update["run_outcome"]["tests"]["unchecked"] == 3 + + +@pytest.mark.asyncio +async def test_matrix_on_on_full_pipeline(tmp_path): + _write_plan(tmp_path, DONE_PLAN_WITH_CHECKS) + ctx = _make_ctx(tmp_path) + # A checkpoint verdict already in the ledger... + append_ledger_record( + tmp_path, + { + "attempt_id": "c#1", + "checkpoint_id": "c", + "item_text": "alarm exists", + "kind": "verify", + "when": "on_complete", + "status": "passed", + "evidence": "probe output", + }, + ) + # ...and the final review supplements the rest. + report = CheckReport( + verdicts=[ + CheckVerdict(item_text="no crash dialog", kind="assert", status="passed", evidence="e"), + CheckVerdict(item_text="toast appeared", kind="assert", status="failed", evidence="e"), + ] + ) + with patch("artemis.graph.graph.run_final_check", new=AsyncMock(return_value=report)): + update = await exit_settlement_node(_make_state(), ctx) + outcome = update["run_outcome"] + assert outcome["tests"]["passed"] == 2 + assert outcome["tests"]["failed"] == 1 + assert outcome["tests"]["unchecked"] == 0 diff --git a/tests/unit/graph/test_graph.py b/tests/unit/graph/test_graph.py index 075bf91..58ae90b 100644 --- a/tests/unit/graph/test_graph.py +++ b/tests/unit/graph/test_graph.py @@ -14,218 +14,232 @@ from unittest.mock import AsyncMock, MagicMock, patch +from langchain_core.messages import ToolMessage + from artemis.context import ArtemisContext from artemis.graph.graph import wrap_note_tool, wrap_update_note_tool import pytest +ORIGINAL_PLAN = """# Test Plan +- [ ] Milestone 1 + - [ ] Substep 1.1 +- [ ] Milestone 2 +""" + + +def _make_ctx(base_dir, *, disable_checker=True, disable_planner_validation=False): + ctx = MagicMock(spec=ArtemisContext) + ctx.data_engine = MagicMock() + ctx.data_engine.base_dir = str(base_dir) + ctx.execution_setup = MagicMock() + ctx.execution_setup.disable_checker = disable_checker + ctx.execution_setup.disable_planner_validation = disable_planner_validation + ctx.planner_task = None + ctx.last_validated_plan = None + ctx.pending_checkpoints = [] + ctx.checkpoint_tasks = {} + ctx.pending_validated_plan = None + ctx.task_plan_content_before = None + return ctx + + +def _setup_plan(tmp_path, content=ORIGINAL_PLAN): + notes_dir = tmp_path / "notes" + notes_dir.mkdir(parents=True, exist_ok=True) + task_plan_path = notes_dir / "task_plan.md" + task_plan_path.write_text(content, encoding="utf-8") + return task_plan_path + + +def _mock_tool(name): + tool = MagicMock() + tool.name = name + tool.description = f"{name} tool" + tool.ainvoke = AsyncMock(return_value="Success") + return tool + + +def _fake_update_invoke(task_plan_path): + async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None): + content = task_plan_path.read_text(encoding="utf-8") + updated = content.replace(args["target"], args["replacement"]) + task_plan_path.write_text(updated, encoding="utf-8") + return "Success" + + return fake_invoke_tool + + +def _fake_save_invoke(task_plan_path): + async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None): + task_plan_path.write_text(args["content"], encoding="utf-8") + return "Success" + + return fake_invoke_tool + @pytest.mark.asyncio async def test_wrap_update_note_tool_valid_status_change(tmp_path): - base_dir = tmp_path - notes_dir = base_dir / "notes" - notes_dir.mkdir(parents=True, exist_ok=True) - task_plan_path = notes_dir / "task_plan.md" - - original_plan = """# Test Plan -- [ ] Milestone 1 - - [ ] Substep 1.1 -- [ ] Milestone 2 -""" - task_plan_path.write_text(original_plan, encoding="utf-8") - - mock_ctx = MagicMock(spec=ArtemisContext) - mock_ctx.data_engine = MagicMock() - mock_ctx.data_engine.base_dir = str(base_dir) - mock_ctx.execution_setup = None - - mock_original_tool = MagicMock() - mock_original_tool.name = "update_note" - mock_original_tool.description = "Update a note" - mock_original_tool.ainvoke = AsyncMock(return_value="Success") - - async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None): - content = task_plan_path.read_text(encoding="utf-8") - updated = content.replace(args["target"], args["replacement"]) - task_plan_path.write_text(updated, encoding="utf-8") - return "Success" - - target = "- [ ] Milestone 1" - replacement = "- [x] Milestone 1" + task_plan_path = _setup_plan(tmp_path) + ctx = _make_ctx(tmp_path) with patch( "artemis.graph.graph.invoke_tool_with_injection", - side_effect=fake_invoke_tool, + side_effect=_fake_update_invoke(task_plan_path), ): - # Instantiate inside patch context to capture the mocked import locally - wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool) + wrapped_tool = wrap_update_note_tool(ctx, _mock_tool("update_note")) result = await wrapped_tool.ainvoke( { "key": "task_plan", - "target": target, - "replacement": replacement, + "target": "- [ ] Milestone 1", + "replacement": "- [x] Milestone 1", "tool_call_id": "test_call_id", } ) - assert result == "Success" - updated_content = task_plan_path.read_text(encoding="utf-8") - assert "- [x] Milestone 1" in updated_content - assert "- [ ] Milestone 2" in updated_content + assert result == "Success" + updated_content = task_plan_path.read_text(encoding="utf-8") + assert "- [x] Milestone 1" in updated_content + assert "- [ ] Milestone 2" in updated_content + # Status flips never disturb the milestone texts: no validation spawned + assert ctx.planner_task is None @pytest.mark.asyncio -async def test_wrap_update_note_tool_invalid_milestone_text_change(tmp_path): - base_dir = tmp_path - notes_dir = base_dir / "notes" - notes_dir.mkdir(parents=True, exist_ok=True) - task_plan_path = notes_dir / "task_plan.md" +async def test_wrap_update_note_tool_intentional_rename_triggers_validation(tmp_path): + task_plan_path = _setup_plan(tmp_path) + ctx = _make_ctx(tmp_path) - original_plan = """# Test Plan -- [ ] Milestone 1 - - [ ] Substep 1.1 -- [ ] Milestone 2 -""" - task_plan_path.write_text(original_plan, encoding="utf-8") - - mock_ctx = MagicMock(spec=ArtemisContext) - mock_ctx.data_engine = MagicMock() - mock_ctx.data_engine.base_dir = str(base_dir) - mock_ctx.execution_setup = None - - mock_original_tool = MagicMock() - mock_original_tool.name = "update_note" - mock_original_tool.ainvoke = AsyncMock(return_value="Success") - - async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None): - content = task_plan_path.read_text(encoding="utf-8") - updated = content.replace(args["target"], args["replacement"]) - task_plan_path.write_text(updated, encoding="utf-8") - return "Success" - - target = "- [ ] Milestone 1" - replacement = "- [ ] Milestone Renamed" - - mock_validation = AsyncMock(return_value={"status": "failed", "feedback": "rejected"}) + mock_validation = AsyncMock(return_value={"status": "success", "feedback": ""}) with ( patch( "artemis.graph.graph.invoke_tool_with_injection", - side_effect=fake_invoke_tool, - ), - patch( - "artemis.agents.planner.planner.run_async_planner_validation", - mock_validation, + side_effect=_fake_update_invoke(task_plan_path), ), + patch("artemis.graph.graph.run_async_planner_validation", mock_validation), ): - wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool) + wrapped_tool = wrap_update_note_tool(ctx, _mock_tool("update_note")) result = await wrapped_tool.ainvoke( { "key": "task_plan", - "target": target, - "replacement": replacement, + "target": "- [ ] Milestone 1", + "replacement": "- [ ] Milestone Renamed", "tool_call_id": "test_call_id", } ) - assert result == "Success" - assert mock_ctx.planner_task is not None + # Declared update_note edit is applied, but ratchet validation fires + assert ctx.planner_task is not None + await ctx.planner_task + + assert "Milestone Renamed" in task_plan_path.read_text(encoding="utf-8") + mock_validation.assert_called_once() + # Validation compares against the ratchet baseline (the original plan) + assert mock_validation.call_args.args[2] == ORIGINAL_PLAN + + +@pytest.mark.asyncio +async def test_ratchet_baseline_survives_consecutive_small_edits(tmp_path): + """Salami-slicing: every small rename is judged against the ORIGINAL + validated baseline, not the immediately preceding write.""" + import asyncio + + task_plan_path = _setup_plan(tmp_path) + ctx = _make_ctx(tmp_path) + + gate = asyncio.Event() + + async def blocking_validation(*args, **kwargs): + await gate.wait() + return {"status": "success", "feedback": ""} + + mock_validation = AsyncMock(side_effect=blocking_validation) + with ( + patch( + "artemis.graph.graph.invoke_tool_with_injection", + side_effect=_fake_update_invoke(task_plan_path), + ), + patch("artemis.graph.graph.run_async_planner_validation", mock_validation), + ): + wrapped_tool = wrap_update_note_tool(ctx, _mock_tool("update_note")) + await wrapped_tool.ainvoke( + { + "key": "task_plan", + "target": "- [ ] Milestone 1", + "replacement": "- [ ] Milestone 1a", + "tool_call_id": "call_1", + } + ) + first_task = ctx.planner_task + assert first_task is not None + + await wrapped_tool.ainvoke( + { + "key": "task_plan", + "target": "- [ ] Milestone 1a", + "replacement": "- [ ] Milestone 1b", + "tool_call_id": "call_2", + } + ) + second_task = ctx.planner_task + assert second_task is not first_task + + # The superseded in-flight validation is cancelled + gate.set() + with pytest.raises(asyncio.CancelledError): + await first_task + await second_task + + # Both validation runs were anchored to the original baseline + for call in mock_validation.call_args_list: + assert call.args[2] == ORIGINAL_PLAN + # Rollback target preserves the last baseline-consistent content + assert ctx.task_plan_content_before == ORIGINAL_PLAN @pytest.mark.asyncio async def test_wrap_update_note_tool_allow_subgoal_changes(tmp_path): - base_dir = tmp_path - notes_dir = base_dir / "notes" - notes_dir.mkdir(parents=True, exist_ok=True) - task_plan_path = notes_dir / "task_plan.md" - - original_plan = """# Test Plan -- [ ] Milestone 1 - - [ ] Substep 1.1 -- [ ] Milestone 2 -""" - task_plan_path.write_text(original_plan, encoding="utf-8") - - mock_ctx = MagicMock(spec=ArtemisContext) - mock_ctx.data_engine = MagicMock() - mock_ctx.data_engine.base_dir = str(base_dir) - mock_ctx.execution_setup = None - - mock_original_tool = MagicMock() - mock_original_tool.name = "update_note" - mock_original_tool.ainvoke = AsyncMock(return_value="Success") - - async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None): - content = task_plan_path.read_text(encoding="utf-8") - updated = content.replace(args["target"], args["replacement"]) - task_plan_path.write_text(updated, encoding="utf-8") - return "Success" - - target = " - [ ] Substep 1.1" - replacement = " - [x] Substep 1.1 Completed" + task_plan_path = _setup_plan(tmp_path) + ctx = _make_ctx(tmp_path) with patch( "artemis.graph.graph.invoke_tool_with_injection", - side_effect=fake_invoke_tool, + side_effect=_fake_update_invoke(task_plan_path), ): - wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool) + wrapped_tool = wrap_update_note_tool(ctx, _mock_tool("update_note")) result = await wrapped_tool.ainvoke( { "key": "task_plan", - "target": target, - "replacement": replacement, + "target": " - [ ] Substep 1.1", + "replacement": " - [x] Substep 1.1 Completed", "tool_call_id": "test_call_id", } ) - assert result == "Success" - updated_content = task_plan_path.read_text(encoding="utf-8") - assert "Substep 1.1 Completed" in updated_content - assert "- [ ] Milestone 1" in updated_content + assert result == "Success" + updated_content = task_plan_path.read_text(encoding="utf-8") + assert "Substep 1.1 Completed" in updated_content + assert "- [ ] Milestone 1" in updated_content + # Sub-task edits never touch the top-level milestone texts + assert ctx.planner_task is None @pytest.mark.asyncio -async def test_wrap_note_tool_invalid_rewrite(tmp_path): - base_dir = tmp_path - notes_dir = base_dir / "notes" - notes_dir.mkdir(parents=True, exist_ok=True) - task_plan_path = notes_dir / "task_plan.md" - - original_plan = """# Test Plan -- [ ] Milestone 1 - - [ ] Substep 1.1 -- [ ] Milestone 2 -""" - task_plan_path.write_text(original_plan, encoding="utf-8") - - mock_ctx = MagicMock(spec=ArtemisContext) - mock_ctx.data_engine = MagicMock() - mock_ctx.data_engine.base_dir = str(base_dir) - mock_ctx.execution_setup = None - - mock_original_tool = MagicMock() - mock_original_tool.name = "save_note" - mock_original_tool.ainvoke = AsyncMock(return_value="Success") - - async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None): - task_plan_path.write_text(args["content"], encoding="utf-8") - return "Success" +async def test_wrap_note_tool_full_rewrite_hand_slip_rejected(tmp_path): + """A save_note full rewrite that rewords status-unchanged milestones is the + hand-slip signature: rejected and rolled back, no validation spawned.""" + task_plan_path = _setup_plan(tmp_path) + ctx = _make_ctx(tmp_path) corrupted_plan = """# Test Plan - [ ] Milestone 1 (Altered) - [ ] New Milestone Added """ - mock_validation = AsyncMock(return_value={"status": "failed", "feedback": "rejected"}) - with ( - patch( - "artemis.graph.graph.invoke_tool_with_injection", - side_effect=fake_invoke_tool, - ), - patch( - "artemis.agents.planner.planner.run_async_planner_validation", - mock_validation, - ), + with patch( + "artemis.graph.graph.invoke_tool_with_injection", + side_effect=_fake_save_invoke(task_plan_path), ): - wrapped_tool = wrap_note_tool(mock_ctx, mock_original_tool) + wrapped_tool = wrap_note_tool(ctx, _mock_tool("save_note")) result = await wrapped_tool.ainvoke( { "key": "task_plan", @@ -234,53 +248,30 @@ async def test_wrap_note_tool_invalid_rewrite(tmp_path): } ) - assert result == "Success" - assert mock_ctx.planner_task is not None + assert isinstance(result, ToolMessage) + assert task_plan_path.read_text(encoding="utf-8") == ORIGINAL_PLAN + assert ctx.planner_task is None @pytest.mark.asyncio -async def test_wrap_note_tool_valid_lexical_drift(tmp_path): - from langgraph.types import Command - from langchain_core.messages import ToolMessage - from artemis.constants import VALIDATOR_MESSAGES_KEY - - base_dir = tmp_path - notes_dir = base_dir / "notes" - notes_dir.mkdir(parents=True, exist_ok=True) - task_plan_path = notes_dir / "task_plan.md" - +async def test_wrap_note_tool_lexical_drift_rejected(tmp_path): + """Even a tiny same-status rewording in a full rewrite bounces back with + guidance — this is exactly the drift that similarity thresholds missed.""" original_plan = """# Test Plan - [ ] Open the Chrome browser and navigate to youtube.com """ - task_plan_path.write_text(original_plan, encoding="utf-8") + task_plan_path = _setup_plan(tmp_path, original_plan) + ctx = _make_ctx(tmp_path) - mock_ctx = MagicMock(spec=ArtemisContext) - mock_ctx.data_engine = MagicMock() - mock_ctx.data_engine.base_dir = str(base_dir) - mock_ctx.execution_setup = None - - # Create mock Command result - tool_msg = ToolMessage(tool_call_id="test_call_id", content="Success outcome", status="success") - mock_command = Command(update={VALIDATOR_MESSAGES_KEY: [tool_msg]}) - - mock_original_tool = MagicMock() - mock_original_tool.name = "save_note" - mock_original_tool.ainvoke = AsyncMock(return_value=mock_command) - - async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None): - task_plan_path.write_text(args["content"], encoding="utf-8") - return mock_command - - # Minor wording change that is semantically valid (similarity ~92%) minor_drift_plan = """# Test Plan - [ ] Open Chrome browser and navigate to youtube.com """ with patch( "artemis.graph.graph.invoke_tool_with_injection", - side_effect=fake_invoke_tool, + side_effect=_fake_save_invoke(task_plan_path), ): - wrapped_tool = wrap_note_tool(mock_ctx, mock_original_tool) + wrapped_tool = wrap_note_tool(ctx, _mock_tool("save_note")) result = await wrapped_tool.ainvoke( { "key": "task_plan", @@ -289,9 +280,45 @@ async def test_wrap_note_tool_valid_lexical_drift(tmp_path): } ) - assert isinstance(result, Command) - assert not hasattr(mock_ctx, "planner_task") or mock_ctx.planner_task is None + assert isinstance(result, ToolMessage) + assert "update_note" in str(result.content) + assert task_plan_path.read_text(encoding="utf-8") == original_plan + assert ctx.planner_task is None - # Verify that the drift plan is actually saved - saved_content = task_plan_path.read_text(encoding="utf-8") - assert "Open Chrome browser" in saved_content + +@pytest.mark.asyncio +async def test_wrap_note_tool_structural_replan_triggers_validation(tmp_path): + """Adding a milestone via full rewrite is a declared replan: applied, then + judged asynchronously by the planner against the ratchet baseline.""" + task_plan_path = _setup_plan(tmp_path) + ctx = _make_ctx(tmp_path) + + extended_plan = """# Test Plan +- [ ] Milestone 1 + - [ ] Substep 1.1 +- [ ] Milestone 2 +- [ ] Milestone 3 (new phase) +""" + + mock_validation = AsyncMock(return_value={"status": "success", "feedback": ""}) + with ( + patch( + "artemis.graph.graph.invoke_tool_with_injection", + side_effect=_fake_save_invoke(task_plan_path), + ), + patch("artemis.graph.graph.run_async_planner_validation", mock_validation), + ): + wrapped_tool = wrap_note_tool(ctx, _mock_tool("save_note")) + result = await wrapped_tool.ainvoke( + { + "key": "task_plan", + "content": extended_plan, + "tool_call_id": "test_call_id", + } + ) + assert result == "Success" + assert ctx.planner_task is not None + await ctx.planner_task + + assert "Milestone 3 (new phase)" in task_plan_path.read_text(encoding="utf-8") + mock_validation.assert_called_once() diff --git a/tests/unit/graph/test_state_contract.py b/tests/unit/graph/test_state_contract.py new file mode 100644 index 0000000..8dc8a58 --- /dev/null +++ b/tests/unit/graph/test_state_contract.py @@ -0,0 +1,110 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Phase 1 contract tests: explicit State + per-node visibility manifests.""" + +import pytest +from pydantic import ValidationError + +from artemis.graph.state import State +from artemis.graph.visibility import ( + NODE_VISIBILITY, + StateView, + VisibilityError, + check_update, + strict_state, + validate_manifest, +) + + +def test_initial_state_round_trips_through_channel_values(): + """sdk/agent.py rebuilds State(**values) from the astream values stream; + with extra="forbid" every dumped key must construct back cleanly.""" + state = State.initial("test goal") + rebuilt = State(**state.model_dump()) + assert rebuilt.initial_goal == "test goal" + assert rebuilt.subagent_calls == [] + + +def test_unknown_constructor_key_is_rejected(): + with pytest.raises(ValidationError): + State(initial_goal="g", ghost_key=1) + + +def test_unknown_attribute_write_is_rejected(): + state = State.initial("g") + with pytest.raises(ValueError): + state.ui_tree = [] # the historical ghost key must never come back + + +def test_deleted_dead_fields_are_gone(): + for dead in ( + "messages", + "validator_messages", + "remaining_steps", + "focused_app_info", + "device_date", + "subgoal_plan", + "operator_tactical_plan", + "complete_subgoals_by_ids", + "current_agent", + "operator_replan_reason", + "check_feedback", + ): + assert dead not in State.model_fields, dead + + +def test_manifest_only_references_declared_fields(): + assert validate_manifest() == [] + + +def test_state_view_allows_declared_and_blocks_undeclared_reads(): + state = State.initial("g") + view = StateView(state, "operator") + assert view.initial_goal == "g" # declared read + with pytest.raises(VisibilityError): + _ = view.exit_settlement_route # not in operator's manifest + + +def test_state_view_blocks_undeclared_mutation(): + state = State.initial("g") + view = StateView(state, "summarizer") # summarizer declares no writes + with pytest.raises(VisibilityError): + view.structured_decisions = "[]" + + +def test_state_view_passes_methods_through(): + view = StateView(State.initial("g"), "operator") + dumped = view.model_dump() + assert dumped["initial_goal"] == "g" + + +def test_check_update_flags_undeclared_write(): + with pytest.raises(VisibilityError): + check_update("summarizer", {"structured_decisions": "[]"}) + check_update("operator", {"structured_decisions": "[]"}) # declared: ok + check_update("operator", None) # empty update: ok + + +def test_strict_state_is_identity_when_disabled(monkeypatch): + monkeypatch.delenv("ARTEMIS_STRICT_STATE", raising=False) + state = State.initial("g") + assert strict_state(state, "operator") is state + + +def test_strict_state_wraps_when_enabled(monkeypatch): + monkeypatch.setenv("ARTEMIS_STRICT_STATE", "1") + state = State.initial("g") + view = strict_state(state, "operator") + assert isinstance(view, StateView) diff --git a/tests/unit/mcp/test_action_manifest.py b/tests/unit/mcp/test_action_manifest.py index 9ff6943..8e1f37f 100644 --- a/tests/unit/mcp/test_action_manifest.py +++ b/tests/unit/mcp/test_action_manifest.py @@ -54,9 +54,11 @@ SCOPED_PROMPT_FILES = [ "artemis/agents/validator/pixel_safety_net.md", ] -# Backtick-quoted identifiers in scoped prompts that are not tools: parameter names and -# note keys. Anything new landing here should be a conscious decision. -NON_TOOL_IDENTIFIERS = frozenset({"analysis", "task_plan"}) +# Backtick-quoted identifiers in scoped prompts that are not tools: parameter names, +# note keys, and the availability-slot template helper rendered by +# apply_operator_prompt_contract. Anything new landing here should be a conscious +# decision. +NON_TOOL_IDENTIFIERS = frozenset({"analysis", "task_plan", "tool_enum"}) _TOOL_REF = re.compile(r"(?:`([a-z][a-z0-9_]{2,})`|\b([a-z][a-z0-9_]{2,})\()") @@ -68,19 +70,12 @@ def _scoped_prompt_sources() -> dict[str, str]: path = REPO_ROOT / rel sources[path.name] = path.read_text(encoding="utf-8") + # The operator templates carry all operator prompt prose (teaching segments are + # gated in-template by availability slots; prompts.py holds no prompt text). operator_json = REPO_ROOT / "artemis/agents/operator/operator.json" for key, text in json.loads(operator_json.read_text(encoding="utf-8")).items(): sources[f"operator.json:{key}"] = text - # Only the prompt string constants from prompts.py -- scanning the module as text - # would pick up Python code (`len(`, `getattr(` ...). - from artemis.agents.operator import prompts as operator_prompts - - for name in dir(operator_prompts): - value = getattr(operator_prompts, name) - if name.startswith("_") and isinstance(value, str) and len(value) > 80: - sources[f"prompts.py:{name}"] = value - return sources @@ -102,8 +97,6 @@ def test_scan_finds_prompt_sources(): sources = _scoped_prompt_sources() assert "flash_runner.md" in sources assert "operator.json:main_template" in sources - assert "operator.json:troubleshooter_template" in sources - assert any(label.startswith("prompts.py:") for label in sources) def test_every_prompt_tool_reference_is_classified(): @@ -194,7 +187,7 @@ def test_minimal_actuator_passes_validation(): def test_missing_required_action_fails_fast(): - caps = (DEVICE_ACTIONS - {"click_sequence"}) + caps = DEVICE_ACTIONS - {"click_sequence"} with pytest.raises(ActuatorContractError, match="click_sequence"): validate_actuator(_StubActuator(caps=caps)) @@ -258,9 +251,9 @@ def test_filter_drops_unimplemented_optional_actions(): def test_filter_passes_through_unclassified_agent_tools(): """Agent-specific tools the manifest does not know must never be gated.""" actuator = _StubActuator(caps=REQUIRED_ACTIONS | INTERNAL_ACTIONS) - decls = _declarations("click_sequence", "reply_to_checker", "some_bespoke_tool") + decls = _declarations("click_sequence", "submit_findings", "some_bespoke_tool") kept = [d.name for d in filter_declarations(decls, actuator, agent="operator")] - assert kept == ["click_sequence", "reply_to_checker", "some_bespoke_tool"] + assert kept == ["click_sequence", "submit_findings", "some_bespoke_tool"] def test_filter_appends_extensions_for_targeted_agents_only(): diff --git a/tests/unit/mcp/test_action_specs.py b/tests/unit/mcp/test_action_specs.py new file mode 100644 index 0000000..9fead48 --- /dev/null +++ b/tests/unit/mcp/test_action_specs.py @@ -0,0 +1,229 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The canonical action manifest's conformance suite. + +Two kinds of protection: + +* **Fixture pins** freeze the exact schema each model surface receives (operator + shells, Validator/Flash declarations, the action server manifest). A schema change + must be a conscious act: regenerate the fixture in the same commit and say why. +* **Bridge derivation** re-derives every cross-dialect parameter difference from the + generated schemas and requires it to be declared in ``ActionSpec.param_bridge``. + Undeclared divergence between what one agent sees and what another sees -- the + historical definition of drift -- fails here instead of shipping. +""" + +import json +from pathlib import Path + +from langchain_core.utils.function_calling import convert_to_openai_tool +import pytest + +from artemis.agents.operator.prompts import ( + _PHYSICAL_ACTIONS_ORDER, + _TURN_ENDING_ORDER, + OPERATOR_PROMPT_TOOLSET, +) +from artemis.agents.validator.tool_declarations import VALIDATOR_TOOLS_DECLARATION +from artemis.mcp.action_manifest import ( + INTERNAL_ACTIONS, + OPTIONAL_ACTIONS, + REQUIRED_ACTIONS, +) +from artemis.mcp.action_names import OPERATOR_ACTION_TO_CANONICAL +from artemis.mcp.action_specs import ( + ACTION_SPECS, + OPERATOR_SHELL_ORDER, + operator_shell_tool, + tool_declaration, + wire_dialects, +) +from artemis.mcp.actuators.base import Actuator + +FIXTURES = Path(__file__).resolve().parents[2] / "fixtures" / "action_surfaces" + +FIXTURE_HINT = ( + "The generated schema differs from the pinned fixture. If this change is" + " intentional, regenerate the fixture in the same commit and explain the schema" + " change in the commit message; models see this schema directly." +) + + +def _fixture(name: str) -> dict: + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +# --- Fixture pins -------------------------------------------------------------------- + + +def test_operator_shells_match_fixture(): + expected = _fixture("operator_shells.json") + assert set(expected) == set(OPERATOR_SHELL_ORDER) + for name in OPERATOR_SHELL_ORDER: + generated = convert_to_openai_tool(operator_shell_tool(name)) + assert generated == expected[name], f"operator shell '{name}': {FIXTURE_HINT}" + + +def test_tool_declarations_match_fixture(): + expected = _fixture("tool_declarations.json") + by_constant = { + "CLICK_TOOL": "click", + "CLICK_SEQUENCE_TOOL": "click_sequence", + "LONG_PRESS_TOOL": "long_press", + "INPUT_TEXT_TOOL": "input_text", + "SWIPE_TOOL": "swipe", + "PRESS_KEY_TOOL": "press_key", + "MANAGE_APP_TOOL": "manage_app", + "WAIT_FOR_DELAY_TOOL": "wait_for_delay", + } + assert set(expected) == set(by_constant) + for constant, name in by_constant.items(): + assert dict(tool_declaration(name)) == expected[constant], ( + f"declaration '{name}': {FIXTURE_HINT}" + ) + + +@pytest.mark.asyncio +async def test_action_server_manifest_matches_fixture(): + from artemis.mcp.action_server import build_action_server + from artemis.mcp.actuators import MockActuator + + expected = _fixture("action_server_manifest.json") + server = build_action_server(MockActuator()) + tools = await server.list_tools() + generated = { + t.name: { + "description": t.description, + "inputSchema": t.inputSchema, + "outputSchema": t.outputSchema, + } + for t in tools + } + assert set(generated) == set(expected) + for name in expected: + assert generated[name] == expected[name], f"server tool '{name}': {FIXTURE_HINT}" + + +def test_validator_declaration_order_is_stable(): + assert [d.name for d in VALIDATOR_TOOLS_DECLARATION] == [ + "click", + "long_press", + "input_text", + "swipe", + "press_key", + "read_note", + "list_notes", + "manage_app", + "wait_for_delay", + "report_failure_analysis", + ] + + +# --- Bridge derivation: dialect differences must be declared ------------------------- + + +def _operator_param_names(name: str) -> list[str]: + schema = convert_to_openai_tool(operator_shell_tool(name)) + return list(schema["function"]["parameters"].get("properties", {})) + + +def _declaration_param_names(name: str) -> list[str]: + return list(tool_declaration(name).parameters.get("properties", {})) + + +def test_param_bridge_is_exact(): + """Every operator<->declaration parameter difference is declared, none invented.""" + for spec in ACTION_SPECS.values(): + if spec.operator is None or spec.declaration is None: + assert spec.param_bridge == {}, ( + f"'{spec.name}' declares a param_bridge without both dialects." + ) + continue + operator_names = _operator_param_names(spec.name) + declaration_names = _declaration_param_names(spec.name) + assert set(spec.param_bridge) == set(operator_names), ( + f"'{spec.name}': param_bridge keys must cover exactly the operator" + f" params. bridge={sorted(spec.param_bridge)} operator={operator_names}" + ) + assert set(spec.param_bridge.values()) == set(declaration_names), ( + f"'{spec.name}': param_bridge values must cover exactly the declaration" + f" params. bridge={sorted(spec.param_bridge.values())}" + f" declaration={declaration_names}" + ) + + +def test_renamed_params_are_documented(): + """A cross-dialect rename is a deliberate difference; it must carry prose.""" + for spec in ACTION_SPECS.values(): + renames = {k: v for k, v in spec.param_bridge.items() if k != v} + if renames: + assert spec.differences, ( + f"'{spec.name}' renames {renames} across dialects but documents no differences." + ) + + +# --- Manifest coverage --------------------------------------------------------------- + + +def test_specs_cover_exactly_the_llm_reachable_device_actions(): + assert set(ACTION_SPECS) == (REQUIRED_ACTIONS | OPTIONAL_ACTIONS) + assert not set(ACTION_SPECS) & INTERNAL_ACTIONS + + +def test_operator_shell_order_covers_operator_dialects(): + with_operator = {s.name for s in ACTION_SPECS.values() if s.operator is not None} + assert set(OPERATOR_SHELL_ORDER) == with_operator + assert len(OPERATOR_SHELL_ORDER) == len(set(OPERATOR_SHELL_ORDER)) + + +def test_every_wire_dialect_matches_the_actuator_protocol(): + for spec in wire_dialects(): + method = getattr(Actuator, spec.name, None) + assert callable(method), f"wire dialect '{spec.name}' has no corresponding Actuator method." + + +def test_prompt_enum_orders_come_from_the_manifest(): + assert _PHYSICAL_ACTIONS_ORDER == OPERATOR_SHELL_ORDER + assert set(_TURN_ENDING_ORDER) == set(OPERATOR_SHELL_ORDER) + assert set(OPERATOR_SHELL_ORDER) <= OPERATOR_PROMPT_TOOLSET + + +def test_operator_verbs_lower_onto_manifest_actions(): + """The Operator's internal decision verbs resolve to canonical manifest names.""" + unresolved = set(OPERATOR_ACTION_TO_CANONICAL.values()) - set(ACTION_SPECS) + assert not unresolved, ( + f"action_names maps operator verbs onto names the manifest does not know:" + f" {sorted(unresolved)}" + ) + + +def test_registry_wait_for_delay_seconds_drift_stays_dead(): + """Guards the removal of the duplicate device-action registration. + + ``artemis/tools/actions/device_actions.py`` used to shadow-register a second + ``wait_for_delay`` taking *seconds* while every prompt teaches milliseconds. The + registry (a direct-execution SDK surface, artemis/tools/mobile/exec_tools.py) + must never again carry a ``wait_for_delay`` whose unit disagrees with the + manifest's ``time_in_ms``. + """ + import artemis.tools.mobile.exec_tools # noqa: F401 - registration side effect + from artemis.tools.base import ToolRegistry + + tool = ToolRegistry.get("wait_for_delay") + if tool is None: + return # Not registered at all: nothing to drift. + fields = set(tool.args_schema.model_json_schema().get("properties", {})) + assert "seconds" not in fields + assert "time_in_ms" in fields diff --git a/tests/unit/test_config_package.py b/tests/unit/test_config_package.py index 08d5a9b..f1c4295 100644 --- a/tests/unit/test_config_package.py +++ b/tests/unit/test_config_package.py @@ -185,7 +185,10 @@ def test_agent_config_loading(): assert agent_cfg.committee.debate_rounds == 2 assert agent_cfg.checker.enabled is False assert agent_cfg.checker.max_iterations == 20 - assert agent_cfg.checker.max_chat_rounds == 4 + assert agent_cfg.checker.midway_checks is True + assert agent_cfg.checker.final_check is True + assert agent_cfg.checker.assert_failure_policy == "continue" + assert agent_cfg.checker.device_probes is True assert agent_cfg.outputter.enabled is True assert agent_cfg.outputter.force_synthesis is False assert agent_cfg.flash.max_turns == 30 @@ -219,6 +222,7 @@ def test_output_config_and_recording(): def test_runtime_state_and_ipc(tmp_path, monkeypatch): """Test IPC port and LS address state helpers.""" import urllib.request + monkeypatch.setattr(urllib.request, "urlopen", MagicMock(side_effect=Exception("offline"))) ipc_state_file = tmp_path / ".artemis_ipc_port" monkeypatch.setattr("artemis.config.runtime.get_ipc_port_file", lambda: ipc_state_file) @@ -247,9 +251,9 @@ def test_runtime_state_and_ipc(tmp_path, monkeypatch): def test_planner_validation_builder_and_milestones(): - """Test AgentConfigBuilder methods for planner validation and validate_milestones threshold.""" + """Test AgentConfigBuilder methods for planner validation and milestone drift detection.""" from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder - from artemis.graph.graph import validate_milestones + from artemis.utils.plan_grammar import milestones_changed, parse_plan # Default builder inherits from artemis.jsonc (enabled=False, similarity_threshold=0.85) builder = AgentConfigBuilder() @@ -270,12 +274,13 @@ def test_planner_validation_builder_and_milestones(): cfg_disabled = AgentConfigBuilder().with_disable_planner_validation(True).build() assert cfg_disabled.disable_planner_validation is True - # Test validate_milestones with custom thresholds - before = "- [ ] Tap Login button\n- [ ] Enter password" - after_minor = "- [ ] Tap the Login button\n- [ ] Enter password" - # Minor edit has ~0.94 similarity - assert validate_milestones(before, after_minor, similarity_threshold=0.85) is False - assert validate_milestones(before, after_minor, similarity_threshold=0.98) is True + # Milestone drift detection is threshold-free: any text change counts, + # status-only flips never do (see artemis.utils.plan_grammar) + before = parse_plan("- [ ] Tap Login button\n- [ ] Enter password") + after_minor = parse_plan("- [ ] Tap the Login button\n- [ ] Enter password") + after_status = parse_plan("- [x] Tap Login button\n- [/] Enter password") + assert milestones_changed(before, after_minor) is True + assert milestones_changed(before, after_status) is False @pytest.mark.asyncio @@ -333,22 +338,35 @@ def test_checker_builder_and_context_propagation(): from artemis.sdk.agent import Agent from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder - # Default builder inherits from artemis.jsonc (enabled=False, max_iterations=20, max_chat_rounds=4) + # Default builder inherits from artemis.jsonc (enabled=False, max_iterations=20) builder = AgentConfigBuilder() cfg = builder.build() assert cfg.disable_checker is True assert cfg.checker_max_iterations == 20 - assert cfg.checker_max_chat_rounds == 4 + assert cfg.disable_midway_checks is False + assert cfg.disable_final_check is False - # Fluent enabling + # Fluent enabling with the new two-gate knobs cfg_enabled = ( AgentConfigBuilder() - .with_checker(enabled=True, max_iterations=25, max_chat_rounds=5) + .with_checker( + enabled=True, + max_iterations=25, + midway_checks=True, + final_check=False, + checkpoint_max_repairs=1, + assert_failure_policy="halt", + device_probes=False, + ) .build() ) assert cfg_enabled.disable_checker is False assert cfg_enabled.checker_max_iterations == 25 - assert cfg_enabled.checker_max_chat_rounds == 5 + assert cfg_enabled.disable_midway_checks is False + assert cfg_enabled.disable_final_check is True + assert cfg_enabled.checkpoint_max_repairs == 1 + assert cfg_enabled.assert_failure_policy == "halt" + assert cfg_enabled.disable_device_probes is True # Fluent disabling cfg_disabled = AgentConfigBuilder().with_disable_checker(True).build() @@ -375,7 +393,13 @@ def test_checker_builder_and_context_propagation(): assert ctx.execution_setup is not None assert ctx.execution_setup.disable_checker is False assert ctx.execution_setup.checker_max_iterations == 25 - assert ctx.execution_setup.checker_max_chat_rounds == 5 + assert ctx.execution_setup.disable_final_check is True + assert ctx.execution_setup.checkpoint_max_repairs == 1 + assert ctx.execution_setup.assert_failure_policy == "halt" + assert ctx.execution_setup.disable_device_probes is True + # Effective gate semantics: master alias off + individual gates + assert ctx.execution_setup.midway_checks_enabled is True + assert ctx.execution_setup.final_check_enabled is False def test_explorer_builder_and_resolution(monkeypatch): @@ -644,4 +668,3 @@ def test_admin_console_config_facade_backward_compatibility(): assert DB_PATH == ac.DB_PATH assert TRACES_PATH == ac.TRACES_PATH assert callable(init_ls_address) - diff --git a/tests/unit/test_llm_gateway.py b/tests/unit/test_llm_gateway.py new file mode 100644 index 0000000..00bfc41 --- /dev/null +++ b/tests/unit/test_llm_gateway.py @@ -0,0 +1,206 @@ +"""Tests for the LLM gateway: complete(), classified recovery, and fallback.""" + +from types import SimpleNamespace + +from langchain_core.messages import AIMessage, AIMessageChunk +import pytest + +from artemis.llm.reliability import LLMExhaustedError, LLMPermanentError +from artemis.services import llm as llm_service +from artemis.services.llm import ( + RobustChatModelWrapper, + acomplete, + acomplete_structured, + with_fallback, +) + + +@pytest.fixture(autouse=True) +def _isolate_gateway_state(monkeypatch, tmp_path): + llm_service._ENDPOINT_BREAKER._states.clear() + llm_service._NON_STREAMING_ENDPOINTS.clear() + monkeypatch.setattr(llm_service, "PAUSE_FILE", tmp_path / ".artemis_paused") + monkeypatch.setattr( + llm_service, + "retry_policy_for", + lambda category: SimpleNamespace(max_attempts=2, delay_for=lambda attempt: 0.0), + ) + monkeypatch.setattr(llm_service.settings, "LLM_PAUSE_TIMEOUT_SECONDS", 1.0, raising=False) + yield + llm_service._ENDPOINT_BREAKER._states.clear() + llm_service._NON_STREAMING_ENDPOINTS.clear() + + +@pytest.mark.asyncio +async def test_mid_stream_failure_retries_without_duplicating_chunks(): + class BreaksMidStream: + def __init__(self): + self.astream_calls = 0 + + async def astream(self, *args, **kwargs): + self.astream_calls += 1 + if self.astream_calls == 1: + yield AIMessageChunk(content="Hello") + raise RuntimeError("503 service unavailable") + yield AIMessageChunk(content="Hello") + yield AIMessageChunk(content=" world") + + base = BreaksMidStream() + wrapper = RobustChatModelWrapper(base) + result = await wrapper.complete([]) + + assert base.astream_calls == 2 + # The partial first attempt was discarded entirely: no duplicated prefix. + assert result.content == "Hello world" + + +@pytest.mark.asyncio +async def test_auth_error_fails_fast_without_retry_or_pause(tmp_path): + class AuthFails: + def __init__(self): + self.astream_calls = 0 + self.ainvoke_calls = 0 + + async def astream(self, *args, **kwargs): + self.astream_calls += 1 + raise RuntimeError("401 unauthorized: invalid api key") + yield # pragma: no cover - makes this an async generator + + async def ainvoke(self, *args, **kwargs): + self.ainvoke_calls += 1 + return AIMessage(content="should never happen") + + base = AuthFails() + wrapper = RobustChatModelWrapper(base) + with pytest.raises(LLMPermanentError): + await wrapper.complete([]) + + assert base.astream_calls == 1 + assert base.ainvoke_calls == 0 + assert not llm_service.PAUSE_FILE.exists() + + +@pytest.mark.asyncio +async def test_stream_unsupported_downgrades_loudly_and_is_remembered(): + class NoStreaming: + def __init__(self): + self.astream_calls = 0 + self.ainvoke_calls = 0 + + async def astream(self, *args, **kwargs): + self.astream_calls += 1 + raise NotImplementedError("streaming is not supported") + yield # pragma: no cover + + async def ainvoke(self, *args, **kwargs): + self.ainvoke_calls += 1 + return AIMessage(content="ok") + + base = NoStreaming() + wrapper = RobustChatModelWrapper(base) + + first = await wrapper.complete([]) + assert first.content == "ok" + assert wrapper._endpoint_key() in llm_service._NON_STREAMING_ENDPOINTS + + second = await wrapper.complete([]) + assert second.content == "ok" + # The capability was cached: streaming is not probed again. + assert base.astream_calls == 1 + assert base.ainvoke_calls == 2 + + +@pytest.mark.asyncio +async def test_exhausted_retries_pause_then_raise_on_deadline(): + class AlwaysUnavailable: + def __init__(self): + self.calls = 0 + + async def astream(self, *args, **kwargs): + self.calls += 1 + raise RuntimeError("503 service unavailable") + yield # pragma: no cover + + wrapper = RobustChatModelWrapper(AlwaysUnavailable()) + with pytest.raises(LLMExhaustedError): + await wrapper.complete([]) + # The pause file was written for the pause window, then cleaned up. + assert not llm_service.PAUSE_FILE.exists() + + +@pytest.mark.asyncio +async def test_with_fallback_takes_over_without_pausing(): + class AlwaysUnavailable: + async def astream(self, *args, **kwargs): + raise RuntimeError("503 service unavailable") + yield # pragma: no cover + + wrapper = RobustChatModelWrapper(AlwaysUnavailable()) + + async def fallback_call(): + return AIMessage(content="fallback answer") + + result = await with_fallback( + main_call=lambda: wrapper.complete([]), + fallback_call=fallback_call, + ) + assert result.content == "fallback answer" + # Because a fallback existed, exhaustion handed over instead of pausing. + assert not llm_service.PAUSE_FILE.exists() + + +@pytest.mark.asyncio +async def test_with_fallback_refuses_to_hide_bad_requests(): + class CodedError(Exception): + code = 400 + + calls = {"fallback": 0} + + async def main_call(): + raise CodedError("malformed tool schema") + + async def fallback_call(): + calls["fallback"] += 1 + return "hidden bug" + + with pytest.raises(CodedError): + await with_fallback(main_call=main_call, fallback_call=fallback_call) + assert calls["fallback"] == 0 + + +@pytest.mark.asyncio +async def test_acomplete_structured_corrective_reask(): + class CorrectsItself: + def __init__(self): + self.calls = [] + + async def ainvoke(self, messages, **kwargs): + self.calls.append(messages) + if len(self.calls) == 1: + return AIMessage(content="oops, here is prose instead of JSON") + return AIMessage(content='{"a": 1}') + + base = CorrectsItself() + result = await acomplete_structured(base, []) + assert result == {"a": 1} + assert len(base.calls) == 2 + correction_text = str(base.calls[1][-1].content) + assert "could not be parsed" in correction_text + + +@pytest.mark.asyncio +async def test_acomplete_compat_path_for_plain_models(): + class PlainModel: + async def astream(self, *args, **kwargs): + yield AIMessageChunk(content="a") + yield AIMessageChunk(content="b") + + result = await acomplete(PlainModel(), []) + assert result.content == "ab" + + class InvokeOnly: + async def ainvoke(self, *args, **kwargs): + return AIMessage(content="direct") + + result = await acomplete(InvokeOnly(), []) + assert result.content == "direct" diff --git a/tests/unit/test_llm_reliability.py b/tests/unit/test_llm_reliability.py new file mode 100644 index 0000000..c57970b --- /dev/null +++ b/tests/unit/test_llm_reliability.py @@ -0,0 +1,68 @@ +"""Tests for the generic LLM failure taxonomy and circuit breaker.""" + +from artemis.llm.reliability import ( + CircuitBreaker, + FailureCategory, + classify_failure, + retry_policy_for, +) + + +def test_classifier_categories_and_recovery_decisions(): + rate = classify_failure(RuntimeError("429 rate limit exceeded")) + assert rate.category is FailureCategory.RATE_LIMIT + assert rate.retryable and rate.should_fallback + + outage = classify_failure(RuntimeError("503 service unavailable")) + assert outage.category is FailureCategory.PROVIDER_UNAVAILABLE + assert outage.retryable + + timeout = classify_failure(TimeoutError("request timed out")) + assert timeout.category is FailureCategory.TIMEOUT + assert timeout.retryable + + conn = classify_failure(ConnectionError("connection reset by peer")) + assert conn.category is FailureCategory.CONNECTION + assert conn.retryable + + auth = classify_failure(RuntimeError("401 unauthorized: invalid api key")) + assert auth.category is FailureCategory.AUTHENTICATION + assert not auth.retryable + assert auth.should_fallback # A fallback endpoint may hold a working key. + + class CodedError(Exception): + code = 400 + + bad = classify_failure(CodedError("invalid request payload")) + assert bad.category is FailureCategory.BAD_REQUEST + assert not bad.retryable + assert not bad.should_fallback # Falling back would only hide the bug. + + unknown = classify_failure(RuntimeError("something odd")) + assert unknown.category is FailureCategory.UNKNOWN + assert unknown.retryable + + +def test_non_retryable_categories_get_single_attempt(): + assert retry_policy_for(FailureCategory.AUTHENTICATION).max_attempts == 1 + assert retry_policy_for(FailureCategory.BAD_REQUEST).max_attempts == 1 + assert retry_policy_for(FailureCategory.RATE_LIMIT).max_attempts > 1 + + +def test_breaker_only_trips_on_transient_categories(): + breaker = CircuitBreaker(threshold=2, cooldown_seconds=10.0) + auth = classify_failure(RuntimeError("401 unauthorized")) + for _ in range(5): + breaker.record_failure("m", auth, now=1.0) + assert breaker.allow("m", now=1.5) + + outage = classify_failure(RuntimeError("503 unavailable")) + breaker.record_failure("m", outage, now=2.0) + breaker.record_failure("m", outage, now=3.0) + assert not breaker.allow("m", now=4.0) + assert breaker.open_remaining("m", now=4.0) > 0 + # Cooldown elapsed: half-open admits exactly one trial. + assert breaker.allow("m", now=13.5) + assert not breaker.allow("m", now=13.6) + breaker.record_success("m") + assert breaker.allow("m", now=13.7) diff --git a/tests/unit/test_llm_structured.py b/tests/unit/test_llm_structured.py new file mode 100644 index 0000000..54c85e7 --- /dev/null +++ b/tests/unit/test_llm_structured.py @@ -0,0 +1,73 @@ +"""Tests for the unified structured-output parsing pipeline.""" + +from pydantic import BaseModel + +from artemis.llm.structured import ( + ParseFailure, + content_to_text, + extract_json_candidate, + parse_structured, + repair_json, +) + + +def test_parses_fenced_json_block(): + text = 'Here is my plan.\n```json\n{"name": "click", "args": {"x": 1}}\n```\nDone.' + result = parse_structured(text) + assert result == {"name": "click", "args": {"x": 1}} + + +def test_parses_bare_json_embedded_in_prose(): + text = 'Sure! {"is_present": true, "confidence": 0.9} — hope that helps.' + result = parse_structured(text) + assert result == {"is_present": True, "confidence": 0.9} + + +def test_balanced_extraction_ignores_braces_inside_strings(): + text = '{"msg": "curly } inside", "n": [1, 2]}' + assert extract_json_candidate(text) == text + + +def test_repairs_trailing_commas_and_comments(): + noisy = '{\n // a comment\n "a": 1, /* block */\n "b": [1, 2,],\n}' + assert parse_structured(noisy) == {"a": 1, "b": [1, 2]} + # Repair must not touch string contents. + assert repair_json('{"s": "a, } // not a comment"}') == '{"s": "a, } // not a comment"}' + + +def test_strips_thinking_tags_before_extraction(): + text = "{not json}\n```json\n[1, 2, 3]\n```" + assert parse_structured(text) == [1, 2, 3] + + +def test_failure_is_typed_never_raw_text(): + result = parse_structured("no structured payload here at all") + assert isinstance(result, ParseFailure) + assert "no JSON" in result.error + + garbage = parse_structured("```json\n{definitely: not: json}\n```") + assert isinstance(garbage, ParseFailure) + + +def test_schema_validation(): + class Verdict(BaseModel): + is_present: bool + confidence: float + + ok = parse_structured('{"is_present": false, "confidence": 0.4}', schema=Verdict) + assert isinstance(ok, Verdict) and ok.confidence == 0.4 + + bad = parse_structured('{"is_present": "definitely"}', schema=Verdict) + assert isinstance(bad, ParseFailure) + assert "schema validation failed" in bad.error + + +def test_content_to_text_flattens_blocks_and_skips_thinking(): + blocks = [ + {"type": "thinking", "thinking": "hidden"}, + {"type": "text", "text": "hello "}, + "raw", + {"type": "text", "text": "world"}, + ] + assert content_to_text(blocks) == "hello rawworld" + assert content_to_text("plain") == "plain" diff --git a/tests/unit/test_tools.py b/tests/unit/test_tools.py index 010c5b5..d6c08b5 100644 --- a/tests/unit/test_tools.py +++ b/tests/unit/test_tools.py @@ -17,7 +17,7 @@ import pytest from artemis.drivers.mock.mock_driver import MockDeviceDriver from artemis.tools.base import ToolRegistry -import artemis.tools.actions.device_actions # noqa: F401 - Register action tools +import artemis.tools.mobile.exec_tools # noqa: F401 - Register action tools @pytest.mark.asyncio diff --git a/tests/unit/tools/test_append_note_tool.py b/tests/unit/tools/test_append_note_tool.py index 9bd1060..bf618c1 100644 --- a/tests/unit/tools/test_append_note_tool.py +++ b/tests/unit/tools/test_append_note_tool.py @@ -13,9 +13,8 @@ # limitations under the License. from pathlib import Path -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock, patch -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolRegistry @@ -28,7 +27,7 @@ from artemis.tools.scratchpad import ( append_note, append_note_wrapper, ) -from langgraph.types import Command +from langchain_core.messages import ToolMessage import pytest @@ -108,11 +107,9 @@ async def test_append_note_callable_execution(mock_ctx, tmp_path): @pytest.mark.asyncio -async def test_append_note_with_state_command(mock_ctx, tmp_path): - """Verify AppendNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY.""" +async def test_append_note_with_state_tool_message(mock_ctx, tmp_path): + """Verify AppendNoteTool.execute with state returns a ToolMessage directly.""" mock_state = MagicMock(spec=State) - mock_state.current_agent = "operator" - mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update) result = await append_note.execute( ctx=mock_ctx, @@ -122,15 +119,10 @@ async def test_append_note_with_state_command(mock_ctx, tmp_path): tool_call_id="call_append_999", ) - assert isinstance(result, Command) - assert VALIDATOR_MESSAGES_KEY in result.update - tool_messages = result.update[VALIDATOR_MESSAGES_KEY] - assert len(tool_messages) == 1 - assert tool_messages[0].tool_call_id == "call_append_999" - assert tool_messages[0].content == "Successfully appended note to state_append_note.md." - assert tool_messages[0].status == "success" - - mock_state.asanitize_update.assert_called_once() + assert isinstance(result, ToolMessage) + assert result.tool_call_id == "call_append_999" + assert result.content == "Successfully appended note to state_append_note.md." + assert result.status == "success" note_path = Path(tmp_path) / "notes" / "state_append_note.md" assert note_path.exists() diff --git a/tests/unit/tools/test_click_tool.py b/tests/unit/tools/test_click_tool.py index 262ed75..396fa69 100644 --- a/tests/unit/tools/test_click_tool.py +++ b/tests/unit/tools/test_click_tool.py @@ -14,7 +14,6 @@ from unittest.mock import AsyncMock, MagicMock, patch -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State @@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import ( click_wrapper, get_click_tool, ) -from langgraph.types import Command +from langchain_core.messages import ToolMessage import pytest @@ -50,13 +49,7 @@ def mock_driver(): @pytest.fixture def mock_state(): - state = MagicMock(spec=State) - - async def _mock_asanitize_update(ctx, update, agent): - return update - - state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update) - return state + return MagicMock(spec=State) def test_click_tool_subclass_and_registry(): @@ -108,7 +101,7 @@ async def test_click_direct_execution_with_ctx(mock_ctx): @pytest.mark.asyncio async def test_click_with_state_command(mock_ctx, mock_state): - """Verify ClickTool returns Command when state is provided.""" + """Verify ClickTool returns ToolMessage when state is provided.""" with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls: mock_controller_inst = MagicMock() mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None)) @@ -120,13 +113,10 @@ async def test_click_with_state_command(mock_ctx, mock_state): tool_call_id="call_click_1", state=mock_state, ) - assert isinstance(cmd, Command) - assert VALIDATOR_MESSAGES_KEY in cmd.update - messages = cmd.update[VALIDATOR_MESSAGES_KEY] - assert len(messages) == 1 - assert messages[0].tool_call_id == "call_click_1" - assert messages[0].status == "success" - assert "Clicked at [100, 200]" in messages[0].content + assert isinstance(cmd, ToolMessage) + assert cmd.tool_call_id == "call_click_1" + assert cmd.status == "success" + assert "Clicked at [100, 200]" in cmd.content @pytest.mark.asyncio diff --git a/tests/unit/tools/test_committee_tool.py b/tests/unit/tools/test_committee_tool.py index 6783097..7e6163e 100644 --- a/tests/unit/tools/test_committee_tool.py +++ b/tests/unit/tools/test_committee_tool.py @@ -62,8 +62,6 @@ class TestCommitteeTool(unittest.IsolatedAsyncioTestCase): mock_state.device_date = None mock_state.structured_decisions = None mock_state.complete_subgoals_by_ids = [] - mock_state.validator_messages = [] - mock_state.operator_replan_reason = "Mock Immediate Replan Reason" # Mock LLM responses for the members mock_llm_pl = MagicMock() @@ -165,9 +163,6 @@ class TestCommitteeTool(unittest.IsolatedAsyncioTestCase): "Timeout waiting for element", content ) # Check failed history is in blackboard self.assertIn("Test Goal", content) # Check initial goal is in blackboard - self.assertIn( - "Mock Immediate Replan Reason", content - ) # Check current replan reason is in blackboard finally: # Clean up temp directory diff --git a/tests/unit/tools/test_input_text_tool.py b/tests/unit/tools/test_input_text_tool.py index f7ff923..be7e648 100644 --- a/tests/unit/tools/test_input_text_tool.py +++ b/tests/unit/tools/test_input_text_tool.py @@ -14,7 +14,6 @@ from unittest.mock import AsyncMock, MagicMock, patch -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State @@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import ( input_text, input_text_wrapper, ) -from langgraph.types import Command +from langchain_core.messages import ToolMessage import pytest @@ -51,13 +50,7 @@ def mock_driver(): @pytest.fixture def mock_state(): - state = MagicMock(spec=State) - - async def _mock_asanitize_update(ctx, update, agent): - return update - - state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update) - return state + return MagicMock(spec=State) def test_input_text_tool_subclass_and_registry(): @@ -120,7 +113,7 @@ async def test_input_text_direct_execution_with_ctx(mock_ctx): @pytest.mark.asyncio async def test_input_text_with_state_command(mock_ctx, mock_state): - """Verify InputTextTool returns Command when state is provided.""" + """Verify InputTextTool returns ToolMessage when state is provided.""" with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls: mock_controller_inst = MagicMock() mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None)) @@ -135,13 +128,10 @@ async def test_input_text_with_state_command(mock_ctx, mock_state): tool_call_id="call_it_1", state=mock_state, ) - assert isinstance(cmd, Command) - assert VALIDATOR_MESSAGES_KEY in cmd.update - messages = cmd.update[VALIDATOR_MESSAGES_KEY] - assert len(messages) == 1 - assert messages[0].tool_call_id == "call_it_1" - assert messages[0].status == "success" - assert "Typed 'query'" in messages[0].content + assert isinstance(cmd, ToolMessage) + assert cmd.tool_call_id == "call_it_1" + assert cmd.status == "success" + assert "Typed 'query'" in cmd.content @pytest.mark.asyncio diff --git a/tests/unit/tools/test_launch_app_tool.py b/tests/unit/tools/test_launch_app_tool.py index 7f28282..4c21443 100644 --- a/tests/unit/tools/test_launch_app_tool.py +++ b/tests/unit/tools/test_launch_app_tool.py @@ -14,7 +14,6 @@ from unittest.mock import AsyncMock, MagicMock, patch -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State @@ -27,7 +26,7 @@ from artemis.tools.mobile.launch_app import ( launch_app, launch_app_wrapper, ) -from langgraph.types import Command +from langchain_core.messages import ToolMessage import pytest @@ -47,13 +46,7 @@ def mock_driver(): @pytest.fixture def mock_state(): - state = MagicMock(spec=State) - - async def _mock_asanitize_update(ctx, update, agent): - return update - - state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update) - return state + return MagicMock(spec=State) def test_launch_app_tool_subclass_and_registry(): @@ -144,7 +137,7 @@ async def test_launch_app_execution_failure(mock_ctx): @pytest.mark.asyncio async def test_launch_app_with_state_command(mock_ctx, mock_state): - """Verify LaunchAppTool returns Command when state is provided.""" + """Verify LaunchAppTool returns ToolMessage when state is provided.""" with ( patch( "artemis.tools.mobile.launch_app.find_package", @@ -163,13 +156,10 @@ async def test_launch_app_with_state_command(mock_ctx, mock_state): tool_call_id="call_la_1", state=mock_state, ) - assert isinstance(cmd, Command) - assert VALIDATOR_MESSAGES_KEY in cmd.update - messages = cmd.update[VALIDATOR_MESSAGES_KEY] - assert len(messages) == 1 - assert messages[0].tool_call_id == "call_la_1" - assert messages[0].status == "success" - assert "App 'State App' launched successfully." in messages[0].content + assert isinstance(cmd, ToolMessage) + assert cmd.tool_call_id == "call_la_1" + assert cmd.status == "success" + assert "App 'State App' launched successfully." in cmd.content @pytest.mark.asyncio diff --git a/tests/unit/tools/test_list_notes_tool.py b/tests/unit/tools/test_list_notes_tool.py index e0e25c7..224cf3d 100644 --- a/tests/unit/tools/test_list_notes_tool.py +++ b/tests/unit/tools/test_list_notes_tool.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock, patch -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolRegistry @@ -28,7 +27,7 @@ from artemis.tools.scratchpad import ( list_notes_wrapper, ) from artemis.utils.notes import save_note_content -from langgraph.types import Command +from langchain_core.messages import ToolMessage import pytest @@ -95,13 +94,11 @@ async def test_list_notes_callable_execution(mock_ctx, tmp_path): @pytest.mark.asyncio -async def test_list_notes_with_state_command(mock_ctx, tmp_path): - """Verify ListNotesTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY.""" +async def test_list_notes_with_state_tool_message(mock_ctx, tmp_path): + """Verify ListNotesTool.execute with state returns a ToolMessage directly.""" save_note_content(tmp_path, "state_note", "content") mock_state = MagicMock(spec=State) - mock_state.current_agent = "operator" - mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update) result = await list_notes.execute( ctx=mock_ctx, @@ -109,16 +106,11 @@ async def test_list_notes_with_state_command(mock_ctx, tmp_path): tool_call_id="call_list_999", ) - assert isinstance(result, Command) - assert VALIDATOR_MESSAGES_KEY in result.update - tool_messages = result.update[VALIDATOR_MESSAGES_KEY] - assert len(tool_messages) == 1 - assert tool_messages[0].tool_call_id == "call_list_999" - assert "Here are all the notes:" in tool_messages[0].content - assert "- state_note (1 lines)" in tool_messages[0].content - assert tool_messages[0].status == "success" - - mock_state.asanitize_update.assert_called_once() + assert isinstance(result, ToolMessage) + assert result.tool_call_id == "call_list_999" + assert "Here are all the notes:" in result.content + assert "- state_note (1 lines)" in result.content + assert result.status == "success" @pytest.mark.asyncio diff --git a/tests/unit/tools/test_long_press_tool.py b/tests/unit/tools/test_long_press_tool.py index 86925d3..fa0cde9 100644 --- a/tests/unit/tools/test_long_press_tool.py +++ b/tests/unit/tools/test_long_press_tool.py @@ -14,7 +14,6 @@ from unittest.mock import AsyncMock, MagicMock, patch -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State @@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import ( long_press, long_press_wrapper, ) -from langgraph.types import Command +from langchain_core.messages import ToolMessage import pytest @@ -50,13 +49,7 @@ def mock_driver(): @pytest.fixture def mock_state(): - state = MagicMock(spec=State) - - async def _mock_asanitize_update(ctx, update, agent): - return update - - state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update) - return state + return MagicMock(spec=State) def test_long_press_tool_subclass_and_registry(): @@ -111,7 +104,7 @@ async def test_long_press_direct_execution_with_ctx(mock_ctx): @pytest.mark.asyncio async def test_long_press_with_state_command(mock_ctx, mock_state): - """Verify LongPressTool returns Command when state is provided.""" + """Verify LongPressTool returns ToolMessage when state is provided.""" with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls: mock_controller_inst = MagicMock() mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None)) @@ -124,13 +117,10 @@ async def test_long_press_with_state_command(mock_ctx, mock_state): tool_call_id="call_lp_1", state=mock_state, ) - assert isinstance(cmd, Command) - assert VALIDATOR_MESSAGES_KEY in cmd.update - messages = cmd.update[VALIDATOR_MESSAGES_KEY] - assert len(messages) == 1 - assert messages[0].tool_call_id == "call_lp_1" - assert messages[0].status == "success" - assert "Long pressed at [100, 200]" in messages[0].content + assert isinstance(cmd, ToolMessage) + assert cmd.tool_call_id == "call_lp_1" + assert cmd.status == "success" + assert "Long pressed at [100, 200]" in cmd.content @pytest.mark.asyncio diff --git a/tests/unit/tools/test_object_detection_tool.py b/tests/unit/tools/test_object_detection_tool.py index acb2541..ffd0f35 100644 --- a/tests/unit/tools/test_object_detection_tool.py +++ b/tests/unit/tools/test_object_detection_tool.py @@ -17,6 +17,8 @@ import json from unittest.mock import AsyncMock, MagicMock, patch +from langchain_core.messages import ToolMessage + from artemis.context import ArtemisContext from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolRegistry @@ -110,13 +112,12 @@ async def test_object_detection_direct_execution_success(): @pytest.mark.asyncio async def test_operator_object_detection_direct_execution_with_state(): - """Verify operator_object_detection reads screenshot from state.""" + """Verify operator_object_detection reads screenshot from state and returns a ToolMessage.""" mock_ctx = MagicMock(spec=ArtemisContext) mock_ctx.data_engine = None mock_state = MagicMock(spec=State) mock_state.latest_screenshot = "/path/to/state_screen.jpg" - mock_state.asanitize_update = None # Non-langgraph direct call expected_result = { "detected": [{"label": "search bar", "point": [500, 100]}], @@ -137,7 +138,13 @@ async def test_operator_object_detection_direct_execution_with_state(): # Verify it used the latest_screenshot from state call_args = mock_run.call_args[0] assert call_args[1] == "/path/to/state_screen.jpg" - assert json.loads(result) == expected_result + + # With state, the tool returns a ToolMessage directly + assert isinstance(result, ToolMessage) + assert result.status == "success" + assert "Object Detection successful." in result.content + json_payload = result.content.split("Results:\n", 1)[1] + assert json.loads(json_payload) == expected_result @pytest.mark.asyncio diff --git a/tests/unit/tools/test_object_detection_trace.py b/tests/unit/tools/test_object_detection_trace.py index 8c73030..ec9ac06 100644 --- a/tests/unit/tools/test_object_detection_trace.py +++ b/tests/unit/tools/test_object_detection_trace.py @@ -27,7 +27,6 @@ async def test_object_detection_trace(): mock_state = MagicMock(spec=State) mock_state.latest_screenshot = "/path/to/screenshot.jpg" - mock_state.asanitize_update = AsyncMock(return_value={"status": "success"}) # We need to mock the decorator in object_detection_tool.py if we want to avoid it calling real get_llm # Or we can just mock get_llm. diff --git a/tests/unit/tools/test_ocr_tool.py b/tests/unit/tools/test_ocr_tool.py index 480dbe8..9eb67a6 100644 --- a/tests/unit/tools/test_ocr_tool.py +++ b/tests/unit/tools/test_ocr_tool.py @@ -15,7 +15,6 @@ import base64 from unittest.mock import AsyncMock, MagicMock, patch -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State @@ -30,7 +29,7 @@ from artemis.tools.mobile.ocr import ( ocr_recognition, ocr_recognition_wrapper, ) -from langgraph.types import Command +from langchain_core.messages import ToolMessage import pytest @@ -61,11 +60,6 @@ def mock_state(tmp_path): state = MagicMock(spec=State) state.latest_screenshot = str(screenshot_file) - - async def _mock_asanitize_update(ctx, update, agent): - return update - - state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update) return state @@ -125,8 +119,9 @@ async def test_ocr_direct_execution_with_state(mock_ctx, tmp_path): return_value=mock_results, ): result = await ocr_recognition.execute(ctx=mock_ctx, state=simple_state) - assert "OCR Recognition successful." in result - assert "Submit" in result + assert isinstance(result, ToolMessage) + assert "OCR Recognition successful." in result.content + assert "Submit" in result.content @pytest.mark.asyncio @@ -164,12 +159,13 @@ async def test_ocr_no_text_detected(mock_ctx, tmp_path): return_value=[], ): result = await ocr_recognition.execute(ctx=mock_ctx, state=simple_state) - assert "No text detected on the screen." in result + assert isinstance(result, ToolMessage) + assert "No text detected on the screen." in result.content @pytest.mark.asyncio async def test_ocr_with_state_command(mock_ctx, mock_state): - """Verify OcrRecognitionTool returns Command when state is provided.""" + """Verify OcrRecognitionTool returns ToolMessage when state is provided.""" mock_results = [{"text": "Hello", "position": [{"x": 100, "y": 100}]}] with patch( "artemis.tools.mobile.ocr.perform_ocr", @@ -181,13 +177,10 @@ async def test_ocr_with_state_command(mock_ctx, mock_state): tool_call_id="call_ocr_1", state=mock_state, ) - assert isinstance(cmd, Command) - assert VALIDATOR_MESSAGES_KEY in cmd.update - messages = cmd.update[VALIDATOR_MESSAGES_KEY] - assert len(messages) == 1 - assert messages[0].tool_call_id == "call_ocr_1" - assert messages[0].status == "success" - assert "Hello" in messages[0].content + assert isinstance(cmd, ToolMessage) + assert cmd.tool_call_id == "call_ocr_1" + assert cmd.status == "success" + assert "Hello" in cmd.content @pytest.mark.asyncio @@ -195,17 +188,15 @@ async def test_ocr_failure_missing_screenshot(mock_ctx): """Verify error handling when screenshot path is missing.""" empty_state = MagicMock(spec=State) empty_state.latest_screenshot = None - empty_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update) cmd = await ocr_recognition.execute( ctx=mock_ctx, tool_call_id="call_ocr_err", state=empty_state, ) - assert isinstance(cmd, Command) - messages = cmd.update[VALIDATOR_MESSAGES_KEY] - assert messages[0].status == "error" - assert "OCR Recognition failed" in messages[0].content + assert isinstance(cmd, ToolMessage) + assert cmd.status == "error" + assert "OCR Recognition failed" in cmd.content @pytest.mark.asyncio diff --git a/tests/unit/tools/test_press_key_tool.py b/tests/unit/tools/test_press_key_tool.py index 2eba60c..78b3fe0 100644 --- a/tests/unit/tools/test_press_key_tool.py +++ b/tests/unit/tools/test_press_key_tool.py @@ -14,7 +14,6 @@ from unittest.mock import AsyncMock, MagicMock, patch -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State @@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import ( press_key, press_key_wrapper, ) -from langgraph.types import Command +from langchain_core.messages import ToolMessage import pytest @@ -47,13 +46,7 @@ def mock_driver(): @pytest.fixture def mock_state(): - state = MagicMock(spec=State) - - async def _mock_asanitize_update(ctx, update, agent): - return update - - state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update) - return state + return MagicMock(spec=State) def test_press_key_tool_subclass_and_registry(): @@ -124,7 +117,7 @@ async def test_press_key_direct_execution_with_ctx(mock_ctx): @pytest.mark.asyncio async def test_press_key_with_state_command(mock_ctx, mock_state): - """Verify PressKeyTool returns Command when state is provided.""" + """Verify PressKeyTool returns ToolMessage when state is provided.""" with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls: mock_controller_inst = MagicMock() mock_controller_inst.go_home = AsyncMock(return_value=True) @@ -136,13 +129,10 @@ async def test_press_key_with_state_command(mock_ctx, mock_state): tool_call_id="call_pk_1", state=mock_state, ) - assert isinstance(cmd, Command) - assert VALIDATOR_MESSAGES_KEY in cmd.update - messages = cmd.update[VALIDATOR_MESSAGES_KEY] - assert len(messages) == 1 - assert messages[0].tool_call_id == "call_pk_1" - assert messages[0].status == "success" - assert "Pressed key 'HOME' successfully." in messages[0].content + assert isinstance(cmd, ToolMessage) + assert cmd.tool_call_id == "call_pk_1" + assert cmd.status == "success" + assert "Pressed key 'HOME' successfully." in cmd.content @pytest.mark.asyncio diff --git a/tests/unit/tools/test_read_note_tool.py b/tests/unit/tools/test_read_note_tool.py index 537aa27..f1e3d64 100644 --- a/tests/unit/tools/test_read_note_tool.py +++ b/tests/unit/tools/test_read_note_tool.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock, patch -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolRegistry @@ -28,7 +27,7 @@ from artemis.tools.scratchpad import ( read_note_wrapper, ) from artemis.utils.notes import save_note_content -from langgraph.types import Command +from langchain_core.messages import ToolMessage import pytest @@ -106,13 +105,11 @@ async def test_read_note_callable_execution(mock_ctx, tmp_path): @pytest.mark.asyncio -async def test_read_note_with_state_command(mock_ctx, tmp_path): - """Verify ReadNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY.""" +async def test_read_note_with_state_tool_message(mock_ctx, tmp_path): + """Verify ReadNoteTool.execute with state returns a ToolMessage directly.""" save_note_content(tmp_path, "state_sample", "state note content") mock_state = MagicMock(spec=State) - mock_state.current_agent = "operator" - mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update) result = await read_note.execute( ctx=mock_ctx, @@ -121,16 +118,11 @@ async def test_read_note_with_state_command(mock_ctx, tmp_path): tool_call_id="call_read_999", ) - assert isinstance(result, Command) - assert VALIDATOR_MESSAGES_KEY in result.update - tool_messages = result.update[VALIDATOR_MESSAGES_KEY] - assert len(tool_messages) == 1 - assert tool_messages[0].tool_call_id == "call_read_999" - assert "Successfully read note 'state_sample'." in tool_messages[0].content - assert "state note content" in tool_messages[0].content - assert tool_messages[0].status == "success" - - mock_state.asanitize_update.assert_called_once() + assert isinstance(result, ToolMessage) + assert result.tool_call_id == "call_read_999" + assert "Successfully read note 'state_sample'." in result.content + assert "state note content" in result.content + assert result.status == "success" @pytest.mark.asyncio diff --git a/tests/unit/tools/test_save_note_tool.py b/tests/unit/tools/test_save_note_tool.py index ed2a33e..94a31b2 100644 --- a/tests/unit/tools/test_save_note_tool.py +++ b/tests/unit/tools/test_save_note_tool.py @@ -13,9 +13,8 @@ # limitations under the License. from pathlib import Path -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock, patch -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolRegistry @@ -28,7 +27,7 @@ from artemis.tools.scratchpad import ( save_note, save_note_wrapper, ) -from langgraph.types import Command +from langchain_core.messages import ToolMessage import pytest @@ -98,11 +97,9 @@ async def test_save_note_callable_execution(mock_ctx, tmp_path): @pytest.mark.asyncio -async def test_save_note_with_state_command(mock_ctx, tmp_path): - """Verify SaveNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY.""" +async def test_save_note_with_state_tool_message(mock_ctx, tmp_path): + """Verify SaveNoteTool.execute with state returns a ToolMessage directly.""" mock_state = MagicMock(spec=State) - mock_state.current_agent = "operator" - mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update) result = await save_note.execute( ctx=mock_ctx, @@ -112,15 +109,10 @@ async def test_save_note_with_state_command(mock_ctx, tmp_path): tool_call_id="call_save_999", ) - assert isinstance(result, Command) - assert VALIDATOR_MESSAGES_KEY in result.update - tool_messages = result.update[VALIDATOR_MESSAGES_KEY] - assert len(tool_messages) == 1 - assert tool_messages[0].tool_call_id == "call_save_999" - assert tool_messages[0].content == "Successfully saved note to state_note.md." - assert tool_messages[0].status == "success" - - mock_state.asanitize_update.assert_called_once() + assert isinstance(result, ToolMessage) + assert result.tool_call_id == "call_save_999" + assert result.content == "Successfully saved note to state_note.md." + assert result.status == "success" note_path = Path(tmp_path) / "notes" / "state_note.md" assert note_path.exists() diff --git a/tests/unit/tools/test_swipe_tool.py b/tests/unit/tools/test_swipe_tool.py index d5a01b8..d1208f5 100644 --- a/tests/unit/tools/test_swipe_tool.py +++ b/tests/unit/tools/test_swipe_tool.py @@ -14,7 +14,6 @@ from unittest.mock import AsyncMock, MagicMock, patch -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State @@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import ( swipe, swipe_wrapper, ) -from langgraph.types import Command +from langchain_core.messages import ToolMessage import pytest @@ -51,13 +50,7 @@ def mock_driver(): @pytest.fixture def mock_state(): - state = MagicMock(spec=State) - - async def _mock_asanitize_update(ctx, update, agent): - return update - - state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update) - return state + return MagicMock(spec=State) def test_swipe_tool_subclass_and_registry(): @@ -131,7 +124,7 @@ async def test_swipe_direct_execution_with_ctx_coords(mock_ctx): @pytest.mark.asyncio async def test_swipe_with_state_command(mock_ctx, mock_state): - """Verify SwipeTool returns Command when state is provided.""" + """Verify SwipeTool returns ToolMessage when state is provided.""" with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls: mock_controller_inst = MagicMock() mock_controller_inst.swipe_coords = AsyncMock(return_value=None) @@ -143,13 +136,10 @@ async def test_swipe_with_state_command(mock_ctx, mock_state): tool_call_id="call_sw_1", state=mock_state, ) - assert isinstance(cmd, Command) - assert VALIDATOR_MESSAGES_KEY in cmd.update - messages = cmd.update[VALIDATOR_MESSAGES_KEY] - assert len(messages) == 1 - assert messages[0].tool_call_id == "call_sw_1" - assert messages[0].status == "success" - assert "Swipe completed successfully." in messages[0].content + assert isinstance(cmd, ToolMessage) + assert cmd.tool_call_id == "call_sw_1" + assert cmd.status == "success" + assert "Swipe completed successfully." in cmd.content @pytest.mark.asyncio diff --git a/tests/unit/tools/test_update_note_tool.py b/tests/unit/tools/test_update_note_tool.py index b16dcbe..3acad96 100644 --- a/tests/unit/tools/test_update_note_tool.py +++ b/tests/unit/tools/test_update_note_tool.py @@ -13,9 +13,8 @@ # limitations under the License. from pathlib import Path -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock, patch -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolRegistry @@ -29,7 +28,7 @@ from artemis.tools.scratchpad import ( update_note_wrapper, ) from artemis.utils.notes import save_note_content -from langgraph.types import Command +from langchain_core.messages import ToolMessage import pytest @@ -123,13 +122,11 @@ async def test_update_note_callable_execution(mock_ctx, tmp_path): @pytest.mark.asyncio -async def test_update_note_with_state_command(mock_ctx, tmp_path): - """Verify UpdateNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY.""" +async def test_update_note_with_state_tool_message(mock_ctx, tmp_path): + """Verify UpdateNoteTool.execute with state returns a ToolMessage directly.""" save_note_content(tmp_path, "state_plan", "- [ ] Step 1") mock_state = MagicMock(spec=State) - mock_state.current_agent = "operator" - mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update) result = await update_note.execute( ctx=mock_ctx, @@ -140,15 +137,10 @@ async def test_update_note_with_state_command(mock_ctx, tmp_path): tool_call_id="call_update_999", ) - assert isinstance(result, Command) - assert VALIDATOR_MESSAGES_KEY in result.update - tool_messages = result.update[VALIDATOR_MESSAGES_KEY] - assert len(tool_messages) == 1 - assert tool_messages[0].tool_call_id == "call_update_999" - assert tool_messages[0].content == "Successfully updated note 'state_plan'." - assert tool_messages[0].status == "success" - - mock_state.asanitize_update.assert_called_once() + assert isinstance(result, ToolMessage) + assert result.tool_call_id == "call_update_999" + assert result.content == "Successfully updated note 'state_plan'." + assert result.status == "success" note_path = Path(tmp_path) / "notes" / "state_plan.md" assert note_path.read_text(encoding="utf-8") == "- [x] Step 1" diff --git a/tests/unit/tools/test_video_analyzer_tool.py b/tests/unit/tools/test_video_analyzer_tool.py index df280d6..ea891b4 100644 --- a/tests/unit/tools/test_video_analyzer_tool.py +++ b/tests/unit/tools/test_video_analyzer_tool.py @@ -14,7 +14,6 @@ from unittest.mock import AsyncMock, MagicMock, patch -from artemis.constants import VALIDATOR_MESSAGES_KEY from artemis.context import ArtemisContext from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolRegistry @@ -27,7 +26,7 @@ from artemis.tools.video_tool import ( get_video_analyzer_tool_pure, video_analyzer, ) -from langgraph.types import Command +from langchain_core.messages import ToolMessage import pytest @@ -38,14 +37,8 @@ def mock_ctx(): @pytest.fixture -def mock_state(mock_ctx): - state = MagicMock(spec=State) - - async def _mock_asanitize_update(ctx, update, agent): - return update - - state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update) - return state +def mock_state(): + return MagicMock(spec=State) def test_video_analyzer_tool_subclass_and_registry(): @@ -104,7 +97,7 @@ async def test_video_analyzer_direct_execution_failed(mock_ctx): @pytest.mark.asyncio async def test_video_analyzer_with_state_command(mock_ctx, mock_state): - """Verify execute returns a Command with ToolMessage when state is provided.""" + """Verify execute returns a ToolMessage when state is provided.""" with patch( "artemis.tools.video_tool.VideoAnalyzer.run", new_callable=AsyncMock, @@ -117,13 +110,10 @@ async def test_video_analyzer_with_state_command(mock_ctx, mock_state): tool_call_id="call_video_123", state=mock_state, ) - assert isinstance(cmd, Command) - assert VALIDATOR_MESSAGES_KEY in cmd.update - messages = cmd.update[VALIDATOR_MESSAGES_KEY] - assert len(messages) == 1 - assert messages[0].tool_call_id == "call_video_123" - assert messages[0].content == "Action verified in video" - assert messages[0].status == "success" + assert isinstance(cmd, ToolMessage) + assert cmd.tool_call_id == "call_video_123" + assert cmd.content == "Action verified in video" + assert cmd.status == "success" @pytest.mark.asyncio @@ -150,11 +140,9 @@ async def test_video_analyzer_exception_handling(mock_ctx, mock_state): tool_call_id="call_err", state=mock_state, ) - assert isinstance(cmd, Command) - assert VALIDATOR_MESSAGES_KEY in cmd.update - messages = cmd.update[VALIDATOR_MESSAGES_KEY] - assert messages[0].status == "error" - assert "Error running video analyzer" in messages[0].content + assert isinstance(cmd, ToolMessage) + assert cmd.status == "error" + assert "Error running video analyzer" in cmd.content @pytest.mark.asyncio diff --git a/tests/unit/utils/test_plan_grammar.py b/tests/unit/utils/test_plan_grammar.py new file mode 100644 index 0000000..a6b0a43 --- /dev/null +++ b/tests/unit/utils/test_plan_grammar.py @@ -0,0 +1,241 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from pathlib import Path + +from artemis.utils.plan_grammar import ( + CONTINUOUS_LOOP_TAG, + PLAN_GRAMMAR_SPEC, + check_items_changed, + milestones_changed, + missing_check_items, + new_top_level_completions, + parse_plan, + render_plan_grammar_spec, + restore_missing_check_items, + subgoal_hash, + unintended_milestone_edits, +) + +PLAN = """# Task Plan +Some free-form context the parser must ignore. +- [x] Open the email app +- [/] [Loop] Inspect candidates (Exit: all inspected; Interval: n/a) + - [x] Candidate #1 + - [/] Candidate #2 +- [ ] [Loop:continuous] Monitor inbox (Interval: every 5 minutes) +- [!] Blocked milestone +""" + + +def test_parse_plan_machine_channel_only(): + snap = parse_plan(PLAN) + assert len(snap.items) == 6 + assert len(snap.top_level) == 4 + assert snap.milestone_texts[0] == "Open the email app" + + top = snap.top_level + assert top[0].is_done + assert top[1].is_active and top[1].is_loop and not top[1].is_continuous + assert top[2].is_pending and top[2].is_loop and top[2].is_continuous + assert top[3].status == "!" + + sub = [i for i in snap.items if not i.is_top_level] + assert all(i.indent == 2 for i in sub) + + +def test_parse_plan_empty_and_none(): + assert parse_plan(None).items == () + assert parse_plan("").items == () + assert not parse_plan("just prose, no checkboxes").has_top_level + + +def test_snapshot_termination_properties(): + assert not parse_plan(PLAN).all_top_level_done + done = "- [x] A\n- [x] B" + assert parse_plan(done).all_top_level_done + assert parse_plan("- [x] A\n- [x] [Loop:continuous] M").continuous_top_level + + +def test_active_item_navigation(): + snap = parse_plan(PLAN) + assert snap.first_active().text.startswith("[Loop] Inspect") + assert snap.last_active().text == "Candidate #2" + parent = snap.parent_of(snap.last_active()) + assert parent is not None and parent.text.startswith("[Loop] Inspect") + + +def test_milestones_changed_only_on_text(): + before = parse_plan("- [ ] A\n- [ ] B") + status_flip = parse_plan("- [x] A\n- [/] B") + reworded = parse_plan("- [ ] A\n- [ ] B improved") + added = parse_plan("- [ ] A\n- [ ] B\n- [ ] C") + assert not milestones_changed(before, status_flip) + assert milestones_changed(before, reworded) + assert milestones_changed(before, added) + + +def test_new_top_level_completions_ordered(): + before = parse_plan("- [x] A\n- [/] B\n- [ ] C") + after = parse_plan("- [x] A\n- [x] B\n- [x] C") + assert new_top_level_completions(before, after) == ["B", "C"] + # Sub-task completions never trigger + sub_after = parse_plan("- [x] A\n- [/] B\n - [x] B1\n- [ ] C") + assert new_top_level_completions(before, sub_after) == [] + + +def test_unintended_milestone_edits_pairing(): + before = parse_plan("- [x] A\n- [/] B\n- [ ] C") + reword_untouched = parse_plan("- [x] A modified\n- [/] B\n- [ ] C") + assert unintended_milestone_edits(before, reword_untouched) == [("A", "A modified")] + + # Status changed on the same line: counts as an intentional touch + complete_and_annotate = parse_plan("- [x] A\n- [x] B (verified)\n- [ ] C") + assert unintended_milestone_edits(before, complete_and_annotate) == [] + + # Structural change: not this check's business + removed = parse_plan("- [x] A\n- [/] B") + assert unintended_milestone_edits(before, removed) == [] + + +def test_subgoal_hash_stability(): + assert subgoal_hash(" A ") == subgoal_hash("A") + item = parse_plan("- [/] Goal text").items[0] + assert item.key == subgoal_hash("Goal text") + + +CHECK_PLAN = """# Task Plan +- [x] Create the 7:30 AM alarm in the Clock app + - [ ] open clock app + - verify: the alarm list shows 7:30 AM with its toggle ON + - assert: an "Alarm set" toast should appear after creation +- [ ] Send the report email + - verify: the email appears in the Sent folder +- assert@end: no application crash dialog should ever appear +- verify@end: the device is back on the home screen +""" + + +def test_check_lines_parse_with_anchor_and_timing(): + snap = parse_plan(CHECK_PLAN) + checks = snap.all_check_items + assert len(checks) == 5 + + alarm = next(i for i in snap.top_level if i.text.startswith("Create the")) + alarm_checks = snap.check_items_of(alarm) + assert [c.kind for c in alarm_checks] == ["verify", "assert"] + assert all(c.when == "on_complete" for c in alarm_checks) + assert alarm_checks[0].text == "the alarm list shows 7:30 AM with its toggle ON" + assert all(c.parent_key == alarm.key for c in alarm_checks) + + task_level = snap.task_level_check_items + assert len(task_level) == 2 + assert all(c.when == "at_end" for c in task_level) + assert all(c.parent_key is None for c in task_level) + assert task_level[0].kind == "assert" + assert task_level[1].kind == "verify" + + +def test_check_lines_do_not_disturb_machine_channel(): + """Check lines must be invisible to milestone hashing, drift detection, + and completion triggers.""" + without = CHECK_PLAN + for line in list(without.splitlines()): + if "verify" in line or ("assert" in line and "- [" not in line): + without = without.replace(line + "\n", "") + with_snap = parse_plan(CHECK_PLAN) + without_snap = parse_plan(without) + + assert with_snap.milestone_texts == without_snap.milestone_texts + assert [i.key for i in with_snap.items] == [i.key for i in without_snap.items] + assert not milestones_changed(without_snap, with_snap) + assert new_top_level_completions(without_snap, with_snap) == [] + assert unintended_milestone_edits(without_snap, with_snap) == [] + + +def test_check_items_changed_multiset(): + a = parse_plan("- [ ] G\n - verify: X\n - verify: X") + b = parse_plan("- [ ] G\n - verify: X") + c = parse_plan("- [ ] G\n - verify: X\n - verify: X") + assert check_items_changed(a, b) + assert not check_items_changed(a, c) + # Status flips never register as check changes + d = parse_plan("- [x] G\n - verify: X\n - verify: X") + assert not check_items_changed(a, d) + + +def test_restore_missing_check_items_reanchors_under_parent(): + before = "- [ ] G1\n - verify: V1\n- [ ] G2\n" + after = "- [x] G1\n- [ ] G2\n" + merged = restore_missing_check_items(before, after) + assert merged is not None + lines = merged.splitlines() + assert lines[0] == "- [x] G1" + assert lines[1] == " - verify: V1" + assert lines[2] == "- [ ] G2" + # Idempotent: nothing missing after the merge + assert restore_missing_check_items(before, merged) is None + + +def test_restore_missing_check_items_orphan_becomes_task_level_at_end(): + before = "- [ ] G1\n - assert: A1\n- [ ] G2\n" + after = "- [ ] G2\n" # parent subgoal removed entirely + merged = restore_missing_check_items(before, after) + assert merged is not None + assert merged.splitlines()[-1] == "- assert@end: A1" + + +def test_restore_missing_check_items_keeps_additions(): + before = "- [ ] G1\n - verify: V1\n" + after = "- [ ] G1\n - verify: V1\n - verify: V2 (new)\n" + assert restore_missing_check_items(before, after) is None + assert not missing_check_items(parse_plan(before), parse_plan(after)) + + +def test_render_plan_grammar_spec_conditional(): + base = render_plan_grammar_spec(include_checks=False) + extended = render_plan_grammar_spec(include_checks=True) + # Both-gates-off output carries zero trace of the checking feature + assert base == PLAN_GRAMMAR_SPEC + assert "verify" not in base and "assert" not in base + assert extended.startswith(base) + assert "- verify:" in extended and "- assert:" in extended and "@end" in extended + # The capability boundary is declared honestly + assert "POST-HOC" in extended.upper() or "post-hoc" in extended + + +def test_grammar_spec_renders_constants(): + assert CONTINUOUS_LOOP_TAG in PLAN_GRAMMAR_SPEC + assert "[Loop]" in PLAN_GRAMMAR_SPEC + assert "[/]" in PLAN_GRAMMAR_SPEC and "[x]" in PLAN_GRAMMAR_SPEC + + +def test_prompts_carry_the_single_sourced_grammar(): + """Drift guard: the prompts that teach the plan format must splice in the + machine-enforced grammar spec and reference the formal continuous tag, + and must no longer key loop protection on Exit wording.""" + agents_dir = Path(__file__).resolve().parents[3] / "artemis" / "agents" + + planner_raw = (agents_dir / "planner" / "planner.json").read_text(encoding="utf-8") + operator_raw = (agents_dir / "operator" / "operator.json").read_text(encoding="utf-8") + + for raw in (planner_raw, operator_raw): + assert "{{ plan_grammar }}" in raw + assert "[Loop:continuous]" in raw + + # The old wording-keyed protection contract must not resurface + planner = json.loads(planner_raw) + guidelines = planner["blocks"]["validator_guidelines"] + assert "until manually stopped" not in guidelines diff --git a/tests/unit/utils/test_verification.py b/tests/unit/utils/test_verification.py deleted file mode 100644 index 7a5b1cf..0000000 --- a/tests/unit/utils/test_verification.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -from artemis.utils.verification import ( - append_verification_chat, - get_verification_chat_path, - get_verification_chat_rounds, - read_verification_chat, -) - - -def test_get_verification_chat_path(tmp_path): - subgoal_hash = "test_hash" - expected_path = tmp_path / "notes" / f"verification_chat_{subgoal_hash}.json" - path = get_verification_chat_path(tmp_path, subgoal_hash) - assert path == expected_path - - -def test_read_verification_chat_not_exists(tmp_path): - path = tmp_path / "non_existent.json" - turns = read_verification_chat(path) - assert turns == [] - - -def test_read_verification_chat_exists(tmp_path): - path = tmp_path / "chat.json" - expected_turns = [{"role": "operator", "round": 1, "content": "hello"}] - path.write_text(json.dumps(expected_turns), encoding="utf-8") - turns = read_verification_chat(path) - assert turns == expected_turns - - -def test_get_verification_chat_rounds(): - turns = [ - {"role": "operator", "round": 1, "content": "op1"}, - {"role": "checker", "round": 1, "content": "chk1"}, - {"role": "operator", "round": 2, "content": "op2"}, - ] - max_op, max_chk = get_verification_chat_rounds(turns) - assert max_op == 2 - assert max_chk == 1 - - -def test_get_verification_chat_rounds_empty(): - max_op, max_chk = get_verification_chat_rounds([]) - assert max_op == 0 - assert max_chk == 0 - - -def test_append_verification_chat(tmp_path): - path = tmp_path / "chat.json" - - # Append to empty/non-existent - success = append_verification_chat(path, "operator", "hello", 1) - assert success - assert path.exists() - turns = json.loads(path.read_text(encoding="utf-8")) - assert turns == [{"role": "operator", "round": 1, "content": "hello"}] - - # Append again - success = append_verification_chat(path, "checker", "hi", 2) - assert success - turns = json.loads(path.read_text(encoding="utf-8")) - assert len(turns) == 2 - assert turns[1] == {"role": "checker", "round": 2, "content": "hi"}