diff --git a/artemis/agents/diagnoser/diagnoser.py b/artemis/agents/diagnoser/diagnoser.py index 4b5df7c..dc8a71c 100644 --- a/artemis/agents/diagnoser/diagnoser.py +++ b/artemis/agents/diagnoser/diagnoser.py @@ -44,8 +44,8 @@ from artemis.tools.tool_wrapper import ( from artemis.tools.video_tool import get_video_analyzer_tool from artemis.tools.wait_tool import get_wait_tool from artemis.utils.logger import get_logger +from artemis.memory.context_policy import build_history_for from artemis.utils.task_tree import ( - build_plan_and_history, get_active_subgoal_hashes, get_recent_subgoal_hashes, ) @@ -160,14 +160,16 @@ class Diagnoser: keep_hashes = get_recent_subgoal_hashes( steps, subgoal_hash, - window_steps=self.ctx.agent_config.history_window_steps, + self.ctx.data_engine.base_dir, ) - plan_and_history = build_plan_and_history( + plan_and_history = build_history_for( + "diagnoser", task_plan, steps, subgoal_hash, keep_subgoal_hashes=keep_hashes, + engine=self.ctx.data_engine, ) except Exception as e: logger.error(f"Failed to build plan and history in Diagnoser: {e}") diff --git a/artemis/agents/flash/context_compressor.py b/artemis/agents/flash/context_compressor.py index 3238519..bb293d4 100644 --- a/artemis/agents/flash/context_compressor.py +++ b/artemis/agents/flash/context_compressor.py @@ -14,16 +14,24 @@ """Universal Context Compressor for Artemis Flash profile. -Replaces historical screenshots with high-density visual step summaries only -after they are ready. Screenshots remain in context while summarization or -retry is still in flight. +:class:`ScrubEdgeCompressor` implements the scrub-edge (擦洗沿) discipline +from the history-module redesign §3.2: each message is mutated a bounded +number of times near the tail (XML stripped at depth 1, screenshot resolved +at depth K), then frozen forever. A late summary never backfills a frozen +message. FlashRunner uses this implementation; the Pro transcript ledger +reuses it over its active region (see the class docstring). + +The legacy per-turn full-rescan ``compress_flash_messages`` was removed in M5 +(2026-09-01) after serving as the M1/M2 semantic reference — its surviving +semantics are asserted directly against ``ScrubEdgeCompressor`` in +``tests/unit/agents/test_flash_scrub_edge.py``. """ -from typing import Any +from typing import Any, Callable from langchain_core.messages import BaseMessage -from artemis.agents.flash.summarizer import VisualStepSummarizer +from artemis.memory.step_memory import StepMemoryService from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -32,134 +40,298 @@ _HISTORY_SUMMARY_PREFIX = "--- Historical Visual Transition ---\n" _UI_LIST_MARKER = "--- UI Element List ---" -def _without_historical_ui_list(text: str) -> str: - """Remove a historical UI list without discarding text before its marker.""" - marker_index = text.find(_UI_LIST_MARKER) +def _without_marked_suffix(text: str, markers: tuple[str, ...] = (_UI_LIST_MARKER,)) -> str: + """Remove a marked heavy block without discarding text before its marker.""" + marker_index = -1 + for marker in markers: + idx = text.find(marker) + if idx >= 0 and (marker_index < 0 or idx < marker_index): + marker_index = idx if marker_index < 0: return text return text[:marker_index].rstrip() -def compress_flash_messages( - messages: list[BaseMessage], - summarizer: VisualStepSummarizer | None = None, - prune_history_xml: bool = True, -) -> None: - """Compresses multi-turn conversation messages in-place. +class ScrubEdgeCompressor: + """Scrub-edge context compressor (history-module redesign §3.2, M1). - Rules: - 1. Keeps the latest screenshot and live UI hierarchy completely intact. - 2. For earlier steps, if a VisualStepSummarizer has completed the step's summary, - replaces the intermediate image block with the objective summary text. - 3. If the background summary is not ready yet, retains the original image until - a successful summary can replace it. - 4. If prune_history_xml is True, removes heavy outdated UI Element lists from past turns. + Mutation discipline — every message is touched at most twice, always near + the tail, and is then frozen forever: + + 1. Depth 1 (XML): once a newer observation exists, historical UI Element + lists are stripped (same semantics as the legacy ``prune_history_xml``). + 2. Depth K (image, ``image_scrub_depth``): the K-th most recent + image-bearing message has its screenshot resolved — replaced by the + ready visual summary; if the summary is still pending the image is + retained for up to ``pending_grace_steps`` further image-depths, after + which it is replaced by a pending placeholder. A failed summary is + replaced by an unavailable placeholder immediately. Both placeholders + carry the DataEngine step reference. + 3. Frozen: after resolution the message is never read or written again — + a late summary never backfills a frozen message. + + Contract: the message list is append-only (FlashRunner never removes or + reorders entries), so bookkeeping is index-based and each pass only + touches the few messages near the scrub edge instead of rescanning the + whole history. + + Profile generalization (M2): the Pro transcript ledger reuses this exact + discipline over its active region. Its observation messages carry no + ``tool_call_id``, so ``summary_key_getter`` supplies the summary-job key + (the DataEngine step id) per message; ``strip_markers`` extends the + depth-1 strip to the Pro tail blocks (UI element list + plan recitation); + and ``tail_offset=1`` accounts for the live observation living outside + the tracked list (the ledger renders it as a separate tail), so depth + arithmetic and the keep-window stay aligned with the Flash semantics + where the live observation is *inside* the list. Defaults leave the + Flash behavior byte-identical. """ - if not messages: - return - # 1. Identify the last message containing an active screenshot - last_img_msg_idx = -1 - for idx in range(len(messages) - 1, -1, -1): - msg = messages[idx] - msg_content = getattr(msg, "content", None) - if isinstance(msg_content, list): - has_image = any( - isinstance(block, dict) and block.get("type") in ("image_url", "image") - for block in msg_content - ) + def __init__( + self, + summarizer: StepMemoryService | None = None, + *, + prune_history_xml: bool = True, + image_scrub_depth: int = 3, + pending_grace_steps: int = 3, + xml_scrub_depth: int = 1, + summary_key_getter: Callable[[BaseMessage], str | None] | None = None, + strip_markers: tuple[str, ...] = (_UI_LIST_MARKER,), + tail_offset: int = 0, + ): + self._summarizer = summarizer + self._prune_history_xml = prune_history_xml + self._image_scrub_depth = max(1, image_scrub_depth) + self._pending_grace_steps = max(0, pending_grace_steps) + self._xml_scrub_depth = max(1, xml_scrub_depth) + self._summary_key_getter = summary_key_getter + self._strip_markers = tuple(strip_markers) or (_UI_LIST_MARKER,) + self._tail_offset = max(0, tail_offset) + + self._scanned_until = 0 + self._tool_msg_count = 0 + # Append-only records of image-bearing messages, in message order: + # {"idx", "key" (str tool_call_id | None), "legacy" (ordinal | None), "is_tool"} + self._tracked: list[dict[str, Any]] = [] + self._frozen: set[int] = set() + self._xml_candidates: list[int] = [] + + def compress(self, messages: list[BaseMessage]) -> None: + """Advance the scrub edge over newly appended messages in-place.""" + self._discover(messages) + if not self._tracked: + return + self._scrub_xml(messages) + self._scrub_images(messages) + + # ------------------------------------------------------------------ + # Discovery (incremental; frozen prefix is never rescanned) + # ------------------------------------------------------------------ + + def _discover(self, messages: list[BaseMessage]) -> None: + for idx in range(self._scanned_until, len(messages)): + msg = messages[idx] + tool_call_id = getattr(msg, "tool_call_id", None) + if tool_call_id is not None: + self._tool_msg_count += 1 + content = getattr(msg, "content", None) + if not isinstance(content, list): + continue + + has_image = False + has_xml = False + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") in ("image_url", "image"): + has_image = True + elif block.get("type") == "text" and self._has_strip_marker( + block.get("text", "") + ): + has_xml = True + + if has_xml: + self._xml_candidates.append(idx) if has_image: - last_img_msg_idx = idx - break + key = str(tool_call_id) if tool_call_id is not None else None + if key is None and self._summary_key_getter is not None: + try: + getter_key = self._summary_key_getter(msg) + except Exception: + getter_key = None + if getter_key is not None: + key = str(getter_key) + self._tracked.append( + { + "idx": idx, + "key": key, + "legacy": self._tool_msg_count if tool_call_id is not None else None, + "is_tool": tool_call_id is not None, + } + ) + self._scanned_until = len(messages) - # 2. Iterate through historical messages and apply compression/fallback. - # ``tool_call_id`` is the stable identity of an action result. The legacy - # ordinal is retained only for summaries produced by older callers/tests. - tool_step_counter = 0 - for idx in range(len(messages)): - msg = messages[idx] - msg_content = getattr(msg, "content", None) - if not isinstance(msg_content, list): - continue + def _has_strip_marker(self, text: str) -> bool: + return any(marker in text for marker in self._strip_markers) - # Keep the latest live screen intact - if idx == last_img_msg_idx: - continue + # ------------------------------------------------------------------ + # Depth-1 XML scrub + # ------------------------------------------------------------------ - # Count tool steps strictly corresponding to FlashRunner action turns - tool_call_id = getattr(msg, "tool_call_id", None) - is_tool_msg = tool_call_id is not None - if is_tool_msg: - tool_step_counter += 1 - legacy_step_num = tool_step_counter - else: - legacy_step_num = None - - summary_text = None - summary_pending = False - if summarizer and is_tool_msg: - # New FlashRunner instances key summaries by tool_call_id, so generic - # tools, no-tool turns, and parallel tool calls cannot shift memories. - action_key = str(tool_call_id) - summary_text = summarizer.get_summary(action_key) - summary_pending = summarizer.is_pending(action_key) - if ( - summary_text is None - and not summarizer.has_job(action_key) - and legacy_step_num is not None - ): - summary_text = summarizer.get_summary(legacy_step_num) - summary_pending = summarizer.is_pending(legacy_step_num) + def _scrub_xml(self, messages: list[BaseMessage]) -> None: + if not self._prune_history_xml: + return + # Keep the UI list only in the newest xml_scrub_depth observation + # messages; everything older (or non-image XML carriers) is stripped. + # With a live tail outside the tracked list (tail_offset > 0) the tail + # itself occupies the newest slots of the keep window. + keep_count = max(0, self._xml_scrub_depth - self._tail_offset) + keep_indices = ( + {rec["idx"] for rec in self._tracked[-keep_count:]} if keep_count else set() + ) + still_pending: list[int] = [] + for idx in self._xml_candidates: + if idx in keep_indices: + still_pending.append(idx) + continue + if idx in self._frozen: + continue + self._strip_xml_blocks(messages[idx]) + self._xml_candidates = still_pending + def _strip_xml_blocks(self, msg: BaseMessage) -> None: + content = getattr(msg, "content", None) + if not isinstance(content, list): + return new_blocks: list[Any] = [] - - for b in msg_content: - if not isinstance(b, dict): - # LangChain permits string parts in multimodal content lists. - # They are lightweight historical evidence and must not vanish. - new_blocks.append(b) - continue - b_type = b.get("type", "") - b_text = b.get("text", "") - - if b_type == "text" and b_text.startswith(_HISTORY_SUMMARY_PREFIX): - # Rebuild this generated block on every pass. This makes the - # compressor idempotent and lets a late summary replace itself. - continue - - # Filter heavy outdated XML hierarchy from past steps to save tokens. - # Some adapters combine action text and the hierarchy in one block; - # retain the useful prefix instead of dropping the entire block. - if prune_history_xml and b_type == "text" and _UI_LIST_MARKER in b_text: - retained_text = _without_historical_ui_list(b_text) + for block in content: + if ( + isinstance(block, dict) + and block.get("type") == "text" + and self._has_strip_marker(block.get("text", "")) + ): + retained_text = _without_marked_suffix( + block.get("text", ""), self._strip_markers + ) if retained_text: - retained_block = dict(b) + retained_block = dict(block) retained_block["text"] = retained_text new_blocks.append(retained_block) continue + new_blocks.append(block) + msg.content = new_blocks - if b_type in ("image_url", "image"): - # A submitted image remains visible until its summary is ready. - # This temporarily favors context size over losing visual evidence. - if summary_pending: - new_blocks.append(b) + # ------------------------------------------------------------------ + # Depth-K image scrub + # ------------------------------------------------------------------ + + def _scrub_images(self, messages: list[BaseMessage]) -> None: + total = len(self._tracked) + scrub_depth = self._image_scrub_depth + grace_limit = scrub_depth + self._pending_grace_steps + + for pos, rec in enumerate(self._tracked): + if rec["idx"] in self._frozen: continue + # 1 == most recent image-bearing message; a live tail outside the + # tracked list (tail_offset > 0) counts as the newest depths. + depth = total - pos + self._tail_offset + if depth < scrub_depth: + continue # active window + if self._tail_offset == 0 and pos == total - 1: + continue # the live observation is never scrubbed - new_blocks.append(b) - - # Do not tie injection to the presence of an image: the image is normally - # pruned before the background summary finishes. A stable tool-call key lets - # a later compression pass backfill the summary into the correct result. - if summary_text: - new_blocks.append( - { + summary, pending, failed, step_no = self._lookup(rec) + if summary is not None: + replacement = { "type": "text", - "text": f"{_HISTORY_SUMMARY_PREFIX}{summary_text}", + "text": f"{_HISTORY_SUMMARY_PREFIX}{summary}", } - ) + elif failed: + replacement = { + "type": "text", + "text": f"[visual summary unavailable; evidence at DataEngine step {step_no}]", + } + elif pending and depth <= grace_limit: + continue # grace period: retain the image, revisit next pass + elif pending: + replacement = { + "type": "text", + "text": f"[visual summary pending; evidence at DataEngine step {step_no}]", + } + else: + replacement = None # no summary job exists; drop the image silently + + self._resolve(messages, rec, replacement) + + def _lookup(self, rec: dict[str, Any]) -> tuple[str | None, bool, bool, int | None]: + """Mirror the legacy keying: tool_call_id first, ordinal fallback.""" + summarizer = self._summarizer + if summarizer is None: + return None, False, False, None + + effective_key: Any = rec["key"] + if effective_key is None: + return None, False, False, None + + summary = summarizer.get_summary(effective_key) + if ( + summary is None + and not summarizer.has_job(effective_key) + and rec["legacy"] is not None + ): + effective_key = rec["legacy"] + summary = summarizer.get_summary(effective_key) + + failed = summarizer.has_failed(effective_key) + pending = summarizer.is_pending(effective_key) and not failed + step_no = summarizer.get_step_number(effective_key) + if step_no is None: + step_no = rec["legacy"] + return summary, pending, failed, step_no + + def _resolve( + self, + messages: list[BaseMessage], + rec: dict[str, Any], + replacement: dict[str, Any] | None, + ) -> None: + """Replace the message's image blocks and freeze it permanently.""" + msg = messages[rec["idx"]] + content = getattr(msg, "content", None) + new_blocks: list[Any] = [] + if isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") in ("image_url", "image"): + continue + if ( + self._prune_history_xml + and isinstance(block, dict) + and block.get("type") == "text" + and self._has_strip_marker(block.get("text", "")) + ): + # Freezing implies fully scrubbed: no stale UI list survives + # even when xml_scrub_depth exceeds the image depth. + retained_text = _without_marked_suffix( + block.get("text", ""), self._strip_markers + ) + if retained_text: + retained_block = dict(block) + retained_block["text"] = retained_text + new_blocks.append(retained_block) + continue + new_blocks.append(block) + + if replacement is not None: + new_blocks.append(replacement) # Defensive fallback: never leave ToolMessage or HumanMessage content empty if not new_blocks: - new_blocks = [{"type": "text", "text": "Action completed." if is_tool_msg else ""}] + new_blocks = [ + {"type": "text", "text": "Action completed." if rec["is_tool"] else ""} + ] msg.content = new_blocks + self._frozen.add(rec["idx"]) + if rec["idx"] in self._xml_candidates: + self._xml_candidates.remove(rec["idx"]) diff --git a/artemis/agents/flash/flash_summarizer.md b/artemis/agents/flash/flash_summarizer.md index f00587d..281f596 100644 --- a/artemis/agents/flash/flash_summarizer.md +++ b/artemis/agents/flash/flash_summarizer.md @@ -6,7 +6,7 @@ You are the Step Summarizer for an Android UI automation agent. Your task is to # PERSPECTIVE & FORMAT CONSTRAINTS 1. **First-Person Perspective**: Write strictly from the agent's first-person perspective using **"I"** (e.g., "In Step {{ step_number }}, I tapped...", "I swiped up on... and observed..."). NEVER use third-person terms like "The agent", "The operator", or "The system". 2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–3 sentences, 35–65 words). -3. **No Lists or Formatting**: Do NOT use bullet points, numbered lists, markdown headers, bold labels, or line breaks in your output. +3. **No Lists or Formatting**: Do NOT use bullet points, numbered lists, markdown headers, bold labels, or line breaks in your output. Your output must NEVER contain `---` separators or section-marker lines — those belong to the input, not the summary. --- diff --git a/artemis/agents/flash/flash_summarizer_single.md b/artemis/agents/flash/flash_summarizer_single.md new file mode 100644 index 0000000..8f1515e --- /dev/null +++ b/artemis/agents/flash/flash_summarizer_single.md @@ -0,0 +1,42 @@ +# ROLE & OBJECTIVE +You are the Step Summarizer for an Android UI automation agent. Exactly ONE screenshot is available for this step — the decision frame captured when the action was issued. There is NO after-action screenshot. Your task is to describe, strictly objectively and in first person, what THIS screen showed and where the physical action landed. + +--- + +# SINGLE-FRAME EVIDENCE RULES +1. **Missing after-frame ≠ unchanged screen**: The absence of an after-action screenshot ONLY means there is no independent post-action evidence for this step. It does NOT mean the screen stayed the same, and it does NOT license any guess about what happened next. +2. **Describe only this frame**: Report the screen/page identity, the key visible content (titles, list entries, values, toggle states, prices, codes, error banners), and the control targeted by the action — referencing the red visual marker when one is present. +3. **Never synthesize a transition**: Do NOT describe, predict, or imply the post-action screen state. The step-to-step story is reconstructed from neighboring steps' summaries — not by you. + +--- + +# PERSPECTIVE & FORMAT CONSTRAINTS +1. **First-Person Perspective**: Write strictly from the agent's first-person perspective using **"I"** (e.g., "In Step {{ step_number }}, I tapped... on a screen showing..."). NEVER use third-person terms like "The agent", "The operator", or "The system". +2. **Single Continuous Paragraph**: Your output MUST be exactly **one compact, continuous paragraph** (1–3 sentences, 35–65 words). +3. **No Lists or Formatting**: Do NOT use bullet points, numbered lists, markdown headers, bold labels, or line breaks. Your output must NEVER contain `---` separators or section-marker lines — those belong to the input, not the summary. + +--- + +# CORE OBSERVATION RULES +1. **Zero Subjective Validation (STRICT)**: + - Absolutely avoid declaring semantic goal achievement, subjective success, or final failure. In historical compression, subjective assumptions cause future turns to falsely believe a subgoal was definitively accomplished or permanently blocked. + - **STRICTLY BANNED WORDS**: `successfully`, `completed`, `achieved`, `entered`, `navigated to`, `arrived at`, `failed`, `unsuccessful`, `could not`, `impossible`. +2. **Preserve Critical Data**: Faithfully transcribe visible text, toast alerts, error banners, verification codes, prices, account names, or tracking numbers that appear on this screen. + +--- + +# EXAMPLES OF HIGH-QUALITY SINGLE-FRAME SUMMARIES + +- **Click on a list row**: + "In Step 5, I tapped the 'Battery' row marked in red in the Settings list, which showed sections for 'Network & internet', 'Display', and 'Battery' with the battery level reading '82%'." + +- **Text input**: + "In Step 8, I typed 'Tokyo Hotel' into the search field at the top of the Maps screen; before my input the field read 'Search here' and the keyboard occupied the lower half of the screen." + +- **Swipe / Scroll**: + "In Step 3, I swiped up starting from the red-marked point on the flight results list, which at that moment displayed 4 morning flights priced from '$310' under the header 'Best departing flights'." + +--- + +# TASK +Generate the single-paragraph first-person summary for Step {{ step_number }} following all constraints above: diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index 892d5f3..f1f59ee 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -33,7 +33,7 @@ from langchain_core.messages import ( ToolMessage, ) -from artemis.agents.flash.context_compressor import compress_flash_messages +from artemis.agents.flash.context_compressor import ScrubEdgeCompressor from artemis.agents.flash.summarizer import VisualStepSummarizer from artemis.agents.validator.tool_declarations import ( ASK_EXPLORER_TOOL, @@ -43,7 +43,12 @@ from artemis.agents.validator.tool_declarations import ( capture_screenshot_and_parse_ui, prune_intermediate_screenshots, ) -from artemis.config import StepSummarizerConfig, load_agent_config +from artemis.config import ( + MemoryRuntimeConfig, + MemoryTranscriptConfig, + StepSummarizerConfig, + load_agent_config, +) from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController from artemis.data_engine.trace import trace @@ -74,17 +79,34 @@ class FlashRunner: cfg = load_agent_config() self.max_turns = max_turns if max_turns is not None else cfg.flash.max_turns self.step_summarizer_cfg = cfg.flash.step_summarizer + self.memory_runtime_cfg = cfg.memory.runtime + self.transcript_cfg = cfg.memory.transcript except Exception: self.max_turns = max_turns if max_turns is not None else 30 self.step_summarizer_cfg = StepSummarizerConfig() + self.memory_runtime_cfg = MemoryRuntimeConfig() + self.transcript_cfg = MemoryTranscriptConfig() self.controller = UnifiedMobileController(ctx) self.executor = McpActionExecutor(ctx, self.controller) self.summarizer = ( - VisualStepSummarizer(ctx, model_name=self.step_summarizer_cfg.model) + VisualStepSummarizer( + ctx, + model_name=self.step_summarizer_cfg.model, + retry_limit=self.memory_runtime_cfg.retry_limit, + max_concurrency=self.memory_runtime_cfg.max_concurrency, + flush_timeout_s=self.memory_runtime_cfg.flush_timeout_s, + ) if self.step_summarizer_cfg.enabled else None ) + # Publish the service on the composition root (ctx.step_memory, M2) so + # any co-resident consumer shares this run's summary runtime. + if self.summarizer is not None and getattr(ctx, "step_memory", None) is None: + try: + ctx.step_memory = self.summarizer + except Exception: + pass def _get_tools(self) -> list: tools = [t for t in VALIDATOR_TOOLS_DECLARATION if t.name != "report_failure_analysis"] @@ -109,6 +131,16 @@ class FlashRunner: async def run(self, state: State) -> dict: logger.info(f"Starting Artemis Flash reactive loop for goal: {self.goal}") + # Fresh scrub-edge ledger per run: the frozen/watermark bookkeeping is + # execution state bound to this run's append-only message list. + compressor = ScrubEdgeCompressor( + summarizer=self.summarizer, + prune_history_xml=self.step_summarizer_cfg.prune_history_xml, + image_scrub_depth=self.transcript_cfg.image_scrub_depth, + pending_grace_steps=self.transcript_cfg.pending_grace_steps, + xml_scrub_depth=self.transcript_cfg.xml_scrub_depth, + ) + # 1. Initialize Universal LLM via Service Layer try: llm = get_llm(self.ctx, name="operator") @@ -204,13 +236,10 @@ class FlashRunner: except Exception as e: logger.warning(f"Failed to check injected instruction in FlashRunner: {e}") - # Compress history even when visual summarization is disabled, so - # screenshot and historical XML pruning have identical semantics. - compress_flash_messages( - messages, - summarizer=self.summarizer, - prune_history_xml=self.step_summarizer_cfg.prune_history_xml, - ) + # Advance the scrub edge even when visual summarization is + # disabled, so screenshot and historical XML pruning keep one + # code path. Messages behind the edge are frozen permanently. + compressor.compress(messages) # Tool restriction on the final turn if turns == self.max_turns: @@ -477,6 +506,15 @@ class FlashRunner: action_dict["normalized_start_coordinates"] = norm_start action_dict["normalized_end_coordinates"] = norm_end + # Record-time enrichment computed by the executor from + # the pre-action frame (target_text / target_class / + # target_resource_id / target_label_source). + target_semantics = (exec_result.metadata or {}).get( + "target_semantics" + ) + if isinstance(target_semantics, dict): + action_dict.update(target_semantics) + recorded_step_id = self.ctx.data_engine.record_step( pre_screenshot_bytes=current_pre_screenshot_bytes, post_screenshot_bytes=post_img_bytes, diff --git a/artemis/agents/flash/summarizer.py b/artemis/agents/flash/summarizer.py index 77e815e..b6820f6 100644 --- a/artemis/agents/flash/summarizer.py +++ b/artemis/agents/flash/summarizer.py @@ -16,7 +16,13 @@ Processes Before/After screenshot pairs and action metadata in the background to generate high-density, strictly objective visual state transition summaries. -Ensures zero blocking latency for the main runner and independently retries failed jobs. + +Scheduling (zero-blocking dispatch, bounded retry, bounded flush, step_id +keying with tool_call_id aliases) lives in the shared +:class:`artemis.memory.step_memory.StepMemoryService`; this class contributes +only the visual-transition lens: red action-marker overlay on the BEFORE +frame, the ``flash_summarizer.md`` neutral-wording contract, and versioned +``summary_status`` writes to the DataEngine. """ import asyncio @@ -29,32 +35,61 @@ from jinja2 import Template from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage from artemis.context import ArtemisContext -from artemis.services.llm import get_google_llm, get_llm +from artemis.memory.step_memory import JobKey, StepMemoryService +from artemis.services.llm import RobustChatModelWrapper, get_google_llm, get_llm +from artemis.services.token_meter import record_llm_usage from artemis.utils.logger import get_logger from artemis.utils.visualization import draw_action_overlay_on_image logger = get_logger(__name__) +# Degenerate-output guard (§5 echo validation): a summary that echoes the +# input's section markers, or falls outside sane length bounds, fails the +# attempt so the bounded service retry regenerates it. +SUMMARY_ECHO_MARKER = "--- [" +SUMMARY_MIN_CHARS = 15 +SUMMARY_MAX_CHARS = 1500 -class VisualStepSummarizer: - """Non-blocking background worker generating objective visual step summaries. - Key design properties: +def degenerate_summary_reason(text: str) -> str | None: + """Why a lens output is unusable (echo/length), or None when it is fine.""" + if SUMMARY_ECHO_MARKER in text or text.startswith("---"): + return "echoes an input section marker" + if len(text) < SUMMARY_MIN_CHARS: + return f"too short ({len(text)} chars < {SUMMARY_MIN_CHARS})" + if len(text) > SUMMARY_MAX_CHARS: + return f"too long ({len(text)} chars > {SUMMARY_MAX_CHARS})" + return None + + +class VisualStepSummarizer(StepMemoryService): + """Visual-transition lens on top of the shared step-memory runtime. + + Key design properties (inherited from StepMemoryService): 1. Zero-blocking dispatch: The main Flash runner dispatches and proceeds immediately. 2. Lossless pending state: The compressor retains the source image until its summary is ready. - 3. Independent retry: Every action owns a retry loop and does not depend on later actions arriving. + 3. Independent bounded retry: Every action owns a retry loop capped at 1 + retry_limit attempts. """ - def __init__(self, ctx: ArtemisContext, model_name: str | None = None): - self.ctx = ctx - self._summaries: dict[int | str, str] = {} - self._pending_tasks: dict[int | str, asyncio.Task] = {} - self._step_inputs: dict[int | str, dict[str, Any]] = {} - self._retry_counts: dict[int | str, int] = {} - self._retry_delays = (0.0, 0.5, 1.0, 2.0, 3.0) + def __init__( + self, + ctx: ArtemisContext, + model_name: str | None = None, + retry_limit: int = 3, + *, + max_concurrency: int = 1, + flush_timeout_s: float = 30.0, + ): + super().__init__( + ctx, + max_concurrency=max_concurrency, + retry_limit=retry_limit, + flush_timeout_s=flush_timeout_s, + ) # Initialize lightweight VLM: prioritize explicit model_name target_model = model_name or "gemini-2.5-flash-lite" + self._model_name = target_model try: if model_name: self._llm = get_google_llm(model_name=target_model, temperature=0.0) @@ -62,8 +97,19 @@ class VisualStepSummarizer: self._llm = get_llm(ctx, name="summarizer", is_utils=True) except Exception: self._llm = get_google_llm(model_name=target_model, temperature=0.0) + try: + configured = getattr(self._llm, "model", None) or getattr( + self._llm, "model_name", None + ) + if isinstance(configured, str) and configured: + self._model_name = configured + except Exception: + pass - # Load system prompt template + # Load system prompt templates: the dual-frame transition prompt and + # the single-frame variant (§5 revision: describe whichever frames + # exist — a Pro step record never carries an independent after-frame, + # by design, and must not be prompted as if one were expected). prompt_path = Path(__file__).parent / "flash_summarizer.md" if prompt_path.exists(): self._prompt_template = prompt_path.read_text(encoding="utf-8") @@ -74,6 +120,19 @@ class VisualStepSummarizer: "continuous first-person paragraph using 'I' (e.g., 'In Step {{ step_number }}, I tapped... and observed...').\n" "Strictly avoid subjective validation words: successfully, completed, failed, achieved, navigated to." ) + single_path = Path(__file__).parent / "flash_summarizer_single.md" + if single_path.exists(): + self._single_prompt_template = single_path.read_text(encoding="utf-8") + else: + self._single_prompt_template = ( + "You are the Step Summarizer for an Android UI automation agent.\n" + "Exactly ONE screenshot (the decision frame) is available; there is NO after-action screenshot —" + " that only means no independent post-action evidence exists.\n" + "Describe strictly what THIS screen shows and where the action landed (red marker), in exactly ONE" + " continuous first-person paragraph using 'I' for Step {{ step_number }}. Never describe or guess" + " the post-action state.\n" + "Strictly avoid subjective validation words: successfully, completed, failed, achieved, navigated to." + ) def dispatch( self, @@ -87,9 +146,23 @@ class VisualStepSummarizer: action_key: str | None = None, data_engine_step_id: UUID | str | None = None, ) -> None: - """Dispatches an asynchronous summarization task without blocking the caller.""" - key: int | str = action_key or step_number - self._step_inputs[key] = { + """Dispatches an asynchronous summarization task without blocking the caller. + + Jobs are keyed by the DataEngine step id when one is available; the + tool_call_id (``action_key``) is retained as an alias so the message + compressor can keep querying by it. Callers with neither provide the + step ordinal, matching the legacy keying. + """ + key: JobKey + aliases: tuple[JobKey, ...] = () + if data_engine_step_id is not None: + key = str(data_engine_step_id) + if action_key is not None: + aliases = (action_key,) + else: + key = action_key if action_key is not None else step_number + + payload = { "step_number": step_number, "action_name": action_name, "action_args": action_args, @@ -98,31 +171,49 @@ class VisualStepSummarizer: "exec_outcome": exec_outcome, "data_engine_step_id": data_engine_step_id, } + self.submit(key, payload, aliases=aliases) - task = asyncio.create_task(self._run_summary_until_ready(key)) - self._pending_tasks[key] = task + def _meter_lens_call(self, response: Any) -> None: + """Meter one raw-model lens call as an ``llm_usage`` trace, best-effort. - async def _run_summary_until_ready(self, action_key: int | str) -> None: - """Retry one action independently until a non-empty summary is committed.""" - while action_key not in self._summaries: - if await self._run_summary_once(action_key): - return + Gateway-wrapped models already meter at the wrapper exit; only the raw + ``get_google_llm`` bypass needs explicit metering here. Lens prompts + are tiny and must not overwrite the session's ``last_prompt_tokens`` + (the compaction thresholds' live context base), hence + ``update_last_prompt=False``. + """ + if isinstance(self._llm, RobustChatModelWrapper): + return + engine = getattr(self.ctx, "data_engine", None) if self.ctx else None + record_llm_usage( + engine, + response, + source=f"lens:visual_transition:{self._model_name}", + update_last_prompt=False, + ) - retry_count = self._retry_counts.get(action_key, 0) + 1 - self._retry_counts[action_key] = retry_count - delay = self._retry_delays[min(retry_count - 1, len(self._retry_delays) - 1)] - input_data = self._step_inputs.get(action_key) - display_step = input_data.get("step_number") if input_data else action_key - logger.info( - f"VisualStepSummarizer: Retrying Step {display_step} " - f"(attempt {retry_count + 1}) after {delay:.1f}s" + def _on_status(self, key: JobKey, status: str) -> None: + """Best-effort DataEngine summary-status write (pending/failed).""" + if not self.ctx.data_engine: + return + payload = self._step_inputs.get(key) or {} + target_step = payload.get("data_engine_step_id") + if not target_step: + return + try: + self.ctx.data_engine.update_step_summary( + target_step, + None, + status=status, + source="visual_transition", + model=self._model_name, ) - if delay: - await asyncio.sleep(delay) + except Exception as de_err: + logger.debug(f"DataEngine summary status update skipped: {de_err}") - async def _run_summary_once(self, action_key: int | str) -> bool: + async def _attempt(self, key: JobKey) -> bool: """Execute one lightweight VLM attempt for a step transition.""" - input_data = self._step_inputs.get(action_key) + input_data = self._step_inputs.get(key) if not input_data: return False @@ -134,7 +225,14 @@ class VisualStepSummarizer: exec_outcome = input_data["exec_outcome"] try: - rendered_prompt = Template(self._prompt_template).render(step_number=step_number) + # §5 revision: the lens input adapts to whichever frames exist. + # Both frames -> the classic BEFORE/AFTER transition prompt; a + # single frame (Pro steps never carry an independent after-frame, + # by design) -> the single-frame variant, which has no AFTER + # ACTION section and forbids synthesizing a transition. + dual = bool(pre_bytes) and bool(post_bytes) + template = self._prompt_template if dual else self._single_prompt_template + rendered_prompt = Template(template).render(step_number=step_number) content_blocks: list[dict[str, Any]] = [ { "type": "text", @@ -156,7 +254,12 @@ class VisualStepSummarizer: content_blocks.append( { "type": "text", - "text": "--- [1] BEFORE ACTION SCREEN (Action Marked Visually in Red) ---", + "text": ( + "--- [1] BEFORE ACTION SCREEN (Action Marked Visually in Red) ---" + if dual + else "--- [1] DECISION FRAME: SCREEN AT ACTION TIME" + " (Action Marked Visually in Red) ---" + ), } ) content_blocks.append( @@ -165,7 +268,17 @@ class VisualStepSummarizer: if post_bytes: b64_post = base64.b64encode(post_bytes).decode("utf-8") - content_blocks.append({"type": "text", "text": "--- [2] AFTER ACTION SCREEN ---"}) + content_blocks.append( + { + "type": "text", + "text": ( + "--- [2] AFTER ACTION SCREEN ---" + if dual + else "--- [1] SCREEN OBSERVED AFTER THE ACTION" + " (no decision frame available) ---" + ), + } + ) content_blocks.append( { "type": "image_url", @@ -179,6 +292,7 @@ class VisualStepSummarizer: ] response = await asyncio.wait_for(self._llm.ainvoke(messages), timeout=25.0) + self._meter_lens_call(response) summary_raw = response.content if isinstance(response.content, str) else "" if isinstance(response.content, list): summary_raw = "".join( @@ -188,22 +302,35 @@ class VisualStepSummarizer: ) summary_text = summary_raw.strip() + degenerate = degenerate_summary_reason(summary_text) if summary_text else None + if degenerate: + logger.warning( + f"VisualStepSummarizer: Discarding degenerate summary for" + f" Step {step_number} ({degenerate}): {summary_text[:80]!r}" + ) + return False if summary_text: - self._summaries[action_key] = summary_text + self._summaries[key] = summary_text logger.info( f"VisualStepSummarizer: Generated summary for Step {step_number}: {summary_text[:80]}..." ) # Free binary image buffers from memory once summary is secured - if action_key in self._step_inputs: - self._step_inputs[action_key]["pre_img_bytes"] = None - self._step_inputs[action_key]["post_img_bytes"] = None + if key in self._step_inputs: + self._step_inputs[key]["pre_img_bytes"] = None + self._step_inputs[key]["post_img_bytes"] = None # Update DataEngine telemetry if active if self.ctx.data_engine: try: target_step = input_data.get("data_engine_step_id") or step_number - self.ctx.data_engine.update_step_summary(target_step, summary_text) + self.ctx.data_engine.update_step_summary( + target_step, + summary_text, + status="ready", + source="visual_transition", + model=self._model_name, + ) except Exception as de_err: logger.debug(f"DataEngine step summary update skipped: {de_err}") return True @@ -213,36 +340,3 @@ class VisualStepSummarizer: f"VisualStepSummarizer: Error generating summary for step {step_number}: {e}" ) return False - - def get_summary( - self, action_key: int | str, fallback_text: str | None = None - ) -> str | None: - """Retrieve the summary for a stable action key, or fallback text.""" - return self._summaries.get(action_key, fallback_text) - - def has_summary(self, action_key: int | str) -> bool: - """Check whether an action summary has completed.""" - return action_key in self._summaries - - def has_job(self, action_key: int | str) -> bool: - """Check whether an action has been submitted for visual summarization.""" - return action_key in self._step_inputs - - def is_pending(self, action_key: int | str) -> bool: - """Check whether an action still needs a summary.""" - return self.has_job(action_key) and not self.has_summary(action_key) - - async def flush(self, timeout_seconds: float = 30.0) -> None: - """Wait for active jobs up to a bound, then cancel remaining retry loops.""" - tasks = [t for t in self._pending_tasks.values() if not t.done()] - if tasks: - logger.info(f"VisualStepSummarizer: Flushing {len(tasks)} pending summary tasks...") - _, pending = await asyncio.wait(tasks, timeout=timeout_seconds) - if pending: - logger.warning( - f"VisualStepSummarizer: Cancelling {len(pending)} unfinished summary tasks " - f"after {timeout_seconds:.1f}s flush timeout." - ) - for task in pending: - task.cancel() - await asyncio.gather(*pending, return_exceptions=True) diff --git a/artemis/agents/history_analyzer/history_analyzer.py b/artemis/agents/history_analyzer/history_analyzer.py index 656229c..142d9f1 100644 --- a/artemis/agents/history_analyzer/history_analyzer.py +++ b/artemis/agents/history_analyzer/history_analyzer.py @@ -35,7 +35,8 @@ from artemis.utils.notes import ( get_note_file_path, get_notes_dir, ) -from artemis.utils.task_tree import build_plan_and_history, get_active_subgoal_hashes +from artemis.memory.context_policy import build_history_for +from artemis.utils.task_tree import get_active_subgoal_hashes logger = get_logger(__name__) @@ -110,12 +111,12 @@ class HistoryAnalyzer: if current_plan: active_subgoal_hash, _ = get_active_subgoal_hashes(current_plan) - plan_and_history = build_plan_and_history( + plan_and_history = build_history_for( + "history_analyzer", current_plan, self.history_steps, active_subgoal_hash, - last_n_detailed=1, - min_summaries=len(self.history_steps), + engine=self.ctx.data_engine, ) except Exception as e: logger.error(f"Failed to build plan and history in HistoryAnalyzer: {e}") diff --git a/artemis/agents/operator/operator.json b/artemis/agents/operator/operator.json index 0c3b004..907828c 100644 --- a/artemis/agents/operator/operator.json +++ b/artemis/agents/operator/operator.json @@ -1,3 +1,3 @@ { - "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.[% if \"swipe\" in available_tools %] For any task requiring continuous range interactions, sliding, or boundary selection, prefer using targeted swipe/drag gestures spanning from the element's start bounds [xmin, ymin] to end bounds [xmax, ymax] to ensure the action is strictly confined to the target element. For drag-and-drop, list reordering, or sliding sliders/SeekBars, set duration >= 1000 (e.g. 1500) and always drag slightly PAST the target position to overcome touch slop and reliably trigger the drop/snap/value update. When setting a slider to Maximum (100%) or Minimum (0%), always swipe fully to the extreme boundary to guarantee reaching the absolute limits.[% endif %]\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n- **Tool Calling Contract**: Tools that return information needed for the decision ([[ tool_enum(pre_decision_tools, \"and\") ]]) are result-dependent Pre-Decision Tools. Do not submit them in the same tool-call list as a Turn-Ending Action; inspect their result first. The write-through tools `update_note` and `append_note` may accompany at most one Turn-Ending Action when their content is based entirely on evidence already available before that action. Never write an action's expected outcome as if it had already been observed.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Memory Preservation & Paginated Exploration**: Using note tools is entirely zero-cost. Actively use this tool to maintain goals and logs. The note tool is the only reliable way to persist memory. Therefore, when you find information on the screen that might be relevant to the task, even if you are not certain, you should first save the information currently visible on the screen before exploring further. If you have already recorded some information, you can use `append_note` to append the new information to the existing content. The note tool can also help you organize lists or information across multiple screens; you can consolidate all information into notes for unified processing.\n- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] into note (Exit: ; Interval: )` or `[Loop:continuous] into note (Interval: )`, adhere strictly to this grammar contract and 5-step cyclic state machine:\n 1. **Grammar & Invariant Contract**:\n - `[Loop]` / `[Loop:continuous]`: Represents a multi-turn lifecycle container. The top-level milestone MUST remain active (`[/]`) and must never be marked `[x]` by single checks.\n - ``: The persistent baseline and diffing ledger. You must initialize it in Round 1 and cross-reference/append to it across cycles.\n - `[Loop:continuous]`: Strict unbounded mode. Never self-terminate or mark the top-level milestone as `[x]`; the system mechanically rejects premature completion. Cycle continuously until the user injects an explicit stop signal.\n - `Exit: `: Bounded mode. Only mark the top-level milestone as `[x]` once the explicit count/target threshold is visually achieved.\n - `Interval: `: Declared polling cadence. You MUST strictly adhere to the exact duration declared in ``[% if \"wait_for_delay\" in available_tools %] (convert minutes to milliseconds accurately for `wait_for_delay`)[% endif %].\n 2. **The 5-Step Cyclic State Machine (SOP)**:\n - **Step 1 (Baseline Sync)**: On initial cycle, observe all visible entries/metrics in the primary monitoring view and save full snapshot into ``.\n - **Step 2 (Pacing & Refresh)**: Respect `` and execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n- **Short-Term Memory**: short_term_memory is a supplement to the task plan. If needed, use the tag in your raw thinking text to record it. Short-term memory will be passed to the agent in the next turn, you can describe any trivial details, it should at least include expectations for the future, the current short-term strategy, and verified facts. Example:\n\nExpectation: Duplicate targets were not found in e-h, now start looking for items starting with i-l. (Letters after m are still unverified); Operation strategy: Search in batches by starting letter; Verified facts: a-d, e-h have no matches.\n\n\n# SYSTEM WORKFLOW: SAFETY NET & AUTO-RECOVERY\nYour decisions are processed through downstream pipeline guardians to guarantee accuracy:\n- **Pre-Execution Safety Net**: Automatically checks that your target is visible and safe to interact with prior to action execution.\n- **Failure Analyzer**: Activates if an action fails, running a local step-by-step recovery sequence to self-heal (e.g., closing keyboards or handling wait states).\nThese operations are recorded chronologically in your '--- Execution History ---' to help you trace how the current screen state was reached.\n\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Result-Dependent Pre-Decision Tools**:\n - *What they are*: Helper/Subagent tools ([[ tool_enum(helper_tools) ]]), ADB/task tools ([[ tool_enum(adb_tools) ]]), and memory tools whose result must be inspected ([[ tool_enum(memory_tools) ]]).\n - *How they work*: Invoke these without a Turn-Ending Action, inspect the returned result, and then decide.\n2. **Write-Through Memory Tools**:\n - `update_note` and `append_note` may run alongside at most one Turn-Ending Action only when recording facts already observed in the current context. If the note content depends on the action's result, wait for the next observation before writing it.\n3. **Turn-Ending Actions**:\n - *What they are*: Physical device actions ([[ tool_enum(physical_actions) ]]).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, proceed through these three phases in sequence.\n\n## Phase 1: Observation & Diagnosis\n1. **Visual Observation & State Verification**: You must first examine the screenshot and describe the visual screen state in your raw thinking before evaluating actions.\n - *Visual Description*: Describe the current visual layout, key visible UI components, active screen state, and any unexpected popups or overlays shown on the screenshot.\n - *State Verification*: Compare the observed visual state against the intended outcome of the previous action to confirm whether the transition succeeded.\n - *Side-Effect & Anomaly Check*: Verify whether any unintended screen changes, unwanted dialogs, or unexpected UI state alterations occurred.\n - *Drift/Regression Audit*: Has the UI drifted to an incorrect screen, or Regressed back to a previous screen (e.g. from an accidental browser back-navigation)?\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen screen (static stuck)?\n[% if \"video_analyzer\" in available_tools %] - *Missed Transient States*: If the current state jumped ahead and you suspect you missed an important initial or intermediate screen (e.g., a number or flash message that was overwritten by your last action), consider using the `video_analyzer` tool to look back at the recent history.\n[% endif %]2. **Path Decision (Recover vs. Adapt)**: Encourage flexible problem-solving. First, evaluate if the current path can still be recovered or advanced (e.g., dismissing a blocker, retrying, or restoring the view). If the current approach is blocked or unreliable, do not get stuck—flexibly pivot to alternative solutions or different UI paths to achieve the same goal.\n3. **Verify & Diagnose (Mandatory Diagnosis Triggers & Loop Exemptions)**: You **MUST** use the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage (before calling any Turn-Ending Actions) if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and are EXEMPT from triggering mandatory diagnosis, provided you are making progress or adhering to the planned loop pacing.\n{% if verification_active %} - **Ambiguous Validation Rejection or Verification Finding**: Use `ask_diagnoser` only when the rejection or finding's cause is unclear, the evidence conflicts, or no safe local correction is evident. A rejection or verification finding alone is not a mandatory diagnosis trigger.\n{% endif %} - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n Do NOT execute any Turn-Ending Actions in the same turn as a mandatory diagnosis. Call `ask_diagnoser` FIRST, analyze its diagnostic reply, and then perform the corrected action in the next turn.\n\n## Phase 2: Planning & Memory Preservation\n1. **Autonomous Goal Management & Dynamic Live-Iteration Tracking**: You have autonomy over nested subgoals under the top-level milestones. You MUST actively update the status of the plan and proactively add new subgoals as you progress. Use `[/]` to denote the currently active milestone and subgoal. Only mark a step as completed (`[x]`) AFTER visual confirmation. Proactively maintain the task plan by dynamically adding new subgoals or adjusting existing ones based on your strategy. Keep the plan as a pure Markdown checklist in `task_plan`. Its grammar is machine-enforced:\n{{ plan_grammar }}\n - *Milestone Structure Preservation*: You are strictly forbidden from deleting, omitting, or modifying top-level `[Loop]`/`[Loop:continuous]` milestones or their declared exit conditions.\n - *Surgical Plan Edits*: Modify `task_plan` with targeted `update_note` (target/replacement) edits. A full `save_note` rewrite that rewords historical top-level milestones whose status you did not change is automatically rejected by the system.\n - *Dynamic Live-Iteration Ledger in Loops*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), dynamically log each concrete iteration item in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle (e.g., ` - [x] Polling Check #1 (10:00): Refreshed, no new items, waiting 5m`). This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a loop, evaluate whether the milestone's Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit). For `[Loop:continuous]` milestones, the milestone MUST remain `[/]` and CANNOT be marked `[x]` by you until the user injects an explicit external stop signal. Once a bounded exit condition is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**\n2. **Preserve Memory & Pre-Action Snapshotting**: You MUST perform the following risk assessment before any UI interaction: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain if you will need these details, you should immediately save the extracted critical values to the note tools to ensure safe downstream access. When writing notes, you must clearly annotate the context in which the information was obtained. Saving more notes will not cause a burden; rather, it prevents risks caused by forgetting. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated. Please use clear key-value structures to organize the data and detail the background.[% if \"video_analyzer\" in available_tools %] If you have already missed certain critical information and cannot re-view it by going back or other operations, you can use the `video_analyzer` to look back at historical information.[% endif %]\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data.\n2. **Action Selection & Mandatory Cognitive Protocol**: Before calling any Turn-Ending Action ([[ tool_enum(turn_ending_actions, \"or\") ]]), you MUST write a single, fluid narrative paragraph of text as your raw thinking. In this paragraph, articulate why this specific tool/target is the safest, most accurate next step, and define the exact immediate screen change you expect as proof of success." + "main_template": "# ROLE\nYou are the Android Device Operator. Your priority is execution accuracy. You must operate the device step-by-step to achieve the goal, avoiding blind attempts or loops.\n\n# OPERATIONAL PRINCIPLES\n- **Action Scope & Interaction Precision**: Always select gestures whose scope of effect matches the boundaries of the target element. Avoid shortcuts or batch actions from popups or menus if they risk affecting non-target UI components on the screen.[% if \"swipe\" in available_tools %] For any task requiring continuous range interactions, sliding, or boundary selection, prefer using targeted swipe/drag gestures spanning from the element's start bounds [xmin, ymin] to end bounds [xmax, ymax] to ensure the action is strictly confined to the target element. For drag-and-drop, list reordering, or sliding sliders/SeekBars, set duration >= 1000 (e.g. 1500) and always drag slightly PAST the target position to overcome touch slop and reliably trigger the drop/snap/value update. When setting a slider to Maximum (100%) or Minimum (0%), always swipe fully to the extreme boundary to guarantee reaching the absolute limits.[% endif %]\n- **Vision-First Priority**: The screenshot is the primary source of truth for the real device state. Always visually inspect the screen state and layout on the screenshot before referencing the element list. Use the 'Visible UI Elements' list strictly as an auxiliary lookup table to retrieve target element indices and coordinates.\n- **Tool Calling Contract**: Tools that return information needed for the decision ([[ tool_enum(pre_decision_tools, \"and\") ]]) are result-dependent Pre-Decision Tools. Do not submit them in the same tool-call list as a Turn-Ending Action; inspect their result first. The write-through tools `update_note` and `append_note` may accompany at most one Turn-Ending Action when their content is based entirely on evidence already available before that action. Never write an action's expected outcome as if it had already been observed.\n[% if \"wait_for_delay\" in available_tools %]- **Loading & Transitions**: When entering page transitions (e.g., app launching, web page loading, or progress indicators), use `wait_for_delay` to let the screen settle before executing target actions. Some transition states are difficult to discern, such as a thin progress line at the top of the browser screen during webpage loads. Before questioning whether an error has occurred, you can first adopt a waiting strategy to see how the device state evolves. If the state remains ambiguous after waiting, certain UI components might be malfunctioning or failing to load; in this case, consider retrying the action or pivoting to an alternative path.\n[% endif %]- **Memory Preservation & Paginated Exploration**: Using note tools is entirely zero-cost. Actively use this tool to maintain goals and logs. The note tool is the only reliable way to persist memory. Therefore, when you find information on the screen that might be relevant to the task, even if you are not certain, you should first save the information currently visible on the screen before exploring further. If you have already recorded some information, you can use `append_note` to append the new information to the existing content. The note tool can also help you organize lists or information across multiple screens; you can consolidate all information into notes for unified processing.\n- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:\n 1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.\n 2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.\n 3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.\n 4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for \"double-checking\", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content.\n- **Dynamic Monitoring & Reactive Loop Protocol (Monitor-Trigger-Action)**: When executing an active milestone matching `[Loop] into note (Exit: ; Interval: )` or `[Loop:continuous] into note (Interval: )`, adhere strictly to this grammar contract and 5-step cyclic state machine:\n 1. **Grammar & Invariant Contract**:\n - `[Loop]` / `[Loop:continuous]`: Represents a multi-turn lifecycle container. The top-level milestone MUST remain active (`[/]`) and must never be marked `[x]` by single checks.\n - ``: The persistent baseline and diffing ledger. You must initialize it in Round 1 and cross-reference/append to it across cycles.\n - `[Loop:continuous]`: Strict unbounded mode. Never self-terminate or mark the top-level milestone as `[x]`; the system mechanically rejects premature completion. Cycle continuously until the user injects an explicit stop signal.\n - `Exit: `: Bounded mode. Only mark the top-level milestone as `[x]` once the explicit count/target threshold is visually achieved.\n - `Interval: `: Declared polling cadence. You MUST strictly adhere to the exact duration declared in ``[% if \"wait_for_delay\" in available_tools %] (convert minutes to milliseconds accurately for `wait_for_delay`)[% endif %].\n 2. **The 5-Step Cyclic State Machine (SOP)**:\n - **Step 1 (Baseline Sync)**: On initial cycle, observe all visible entries/metrics in the primary monitoring view and save full snapshot into ``.\n - **Step 2 (Pacing & Refresh)**: Respect `` and execute a refresh action (e.g., pull-to-refresh, sync, or reload) on the primary monitoring view.\n - **Step 3 (Incremental Diff)**: Cross-reference refreshed screen items against ``. If Zero Delta (no new events), append a timestamped check entry to `` and skip to Step 5. If Positive Delta (new events found), proceed to Step 4.\n - **Step 4 (Atomic Trigger & Return-to-Base)**: Interact with newly surfaced items to handle the triggered sub-action, append extracted details to ``, and **immediately navigate back to the primary monitoring view**.\n - **Step 5 (Ledger Advance)**: In `task_plan`, mark current check subtask as `[x]` (e.g., ` - [x] Polling Check #N: ...`), initialize next check as `[/]` (e.g., ` - [/] Polling Check #(N+1): ...`), and preserve parent `[Loop]` milestone as `[/]`.\n\n# SYSTEM WORKFLOW: SAFETY NET & AUTO-RECOVERY\nYour decisions are processed through downstream pipeline guardians to guarantee accuracy:\n- **Pre-Execution Safety Net**: Automatically checks that your target is visible and safe to interact with prior to action execution.\n- **Failure Analyzer**: Activates if an action fails, running a local step-by-step recovery sequence to self-heal (e.g., closing keyboards or handling wait states).\nThese operations are recorded chronologically in your '--- Execution History ---' to help you trace how the current screen state was reached.\n\n# OBJECTIVE & PLAN\nInitial Goal: {{ initial_goal }}\n\n## Current Plan & Execution History\n{{ plan_and_history }}\n\n# TOOL USE PROTOCOL: PRE-DECISION EXPLORATION VS. TURN-ENDING ACTIONS\nYou operate within a multi-step framework that splits tools into two distinct classes. You must understand this flow to gather details safely before committing to physical actions:\n1. **Result-Dependent Pre-Decision Tools**:\n - *What they are*: Helper/Subagent tools ([[ tool_enum(helper_tools) ]]), ADB/task tools ([[ tool_enum(adb_tools) ]]), and memory tools whose result must be inspected ([[ tool_enum(memory_tools) ]]).\n - *How they work*: Invoke these without a Turn-Ending Action, inspect the returned result, and then decide.\n[% if \"recall_history\" in available_tools %] - *History recall (`recall_history`)*: Deterministic lookup into older execution history that has been compressed out of your visible context (step summaries, exact actions and validator results, reasoning, notes, per-step action ledgers). Call it when: the current screen closely resembles a much older state; you need an exact old value (text, id, amount, path); two consecutive steps made no progress; you are about to attempt the same action a third time; a history summary conflicts with what you observe; or the user asks to return to an earlier state. A `(Step-level ledger via recall_history for steps a–b)` guidance line under an `[Era ...]` period paragraph in your history is an explicit entry point — query that step range to retrieve the full per-step action ledger. Do NOT call it speculatively on ordinary steps: the visible history is normally sufficient.\n[% endif %]2. **Write-Through Memory Tools**:\n - `update_note` and `append_note` may run alongside at most one Turn-Ending Action only when recording facts already observed in the current context. If the note content depends on the action's result, wait for the next observation before writing it.\n3. **Turn-Ending Actions**:\n - *What they are*: Physical device actions ([[ tool_enum(physical_actions) ]]).\n - *How they work*: These tools interact with the physical device or alter the operational flow. Executing any of these tools terminates your turn.\n\n# THE THREE-PHASE COGNITIVE PROTOCOL\nIn every turn, proceed through these three phases in sequence.\n\n## Phase 1: Observation & Diagnosis\n1. **Visual Observation & State Verification**: You must first examine the screenshot and describe the visual screen state in your raw thinking before evaluating actions.\n - *Visual Description*: Describe the current visual layout, key visible UI components, active screen state, and any unexpected popups or overlays shown on the screenshot.\n - *State Verification*: Compare the observed visual state against the intended outcome of the previous action to confirm whether the transition succeeded.\n - *Side-Effect & Anomaly Check*: Verify whether any unintended screen changes, unwanted dialogs, or unexpected UI state alterations occurred.\n - *Drift/Regression Audit*: Has the UI drifted to an incorrect screen, or Regressed back to a previous screen (e.g. from an accidental browser back-navigation)?\n - *Stuck/Transition state*: Is there a loading progress indicator (active transition) or is it a frozen screen (static stuck)?\n[% if \"video_analyzer\" in available_tools %] - *Missed Transient States*: If the current state jumped ahead and you suspect you missed an important initial or intermediate screen (e.g., a number or flash message that was overwritten by your last action), consider using the `video_analyzer` tool to look back at the recent history.\n[% endif %]2. **Path Decision (Recover vs. Adapt)**: Encourage flexible problem-solving. First, evaluate if the current path can still be recovered or advanced (e.g., dismissing a blocker, retrying, or restoring the view). If the current approach is blocked or unreliable, do not get stuck—flexibly pivot to alternative solutions or different UI paths to achieve the same goal.\n3. **Verify & Diagnose (Mandatory Diagnosis Triggers & Loop Exemptions)**: You **MUST** use the diagnostic tool (`ask_diagnoser`) during the Pre-Decision Exploration stage (before calling any Turn-Ending Actions) if you detect any of the following symptoms:\n - **Action Loop or State Stagnation**: You are about to execute the exact same physical action or click the same target for a 3rd consecutive time, or you find yourself alternating between the same actions (e.g., swiping up then down repeatedly) in a cyclical loop. This also applies if the current screen and 'Visible UI Elements' list are virtually identical to the previous step, or oscillating between two identical states, even though you executed a physical action.\n *EXEMPTION FOR PLANNED LOOPS*: Periodic pull-to-refresh actions (with planned wait intervals), standard \"inspect candidate -> return to list\" navigations across distinct items, and purposeful continuous swipes (e.g., scrolling to drain battery or traverse long content) are valid planned operations and are EXEMPT from triggering mandatory diagnosis, provided you are making progress or adhering to the planned loop pacing.\n{% if verification_active %} - **Ambiguous Validation Rejection or Verification Finding**: Use `ask_diagnoser` only when the rejection or finding's cause is unclear, the evidence conflicts, or no safe local correction is evident. A rejection or verification finding alone is not a mandatory diagnosis trigger.\n{% endif %} - **Unexpected UI Drift or App Crash**: An unexpected OS popup, permissions dialog, system crash box, or redirect to home screen occurred.\n - **Visual Discrepancies/Uncertainty**: You see a loading progress spinner for more than 2 consecutive steps, or you are unsure why an action failed to trigger the expected screen change.\n Do NOT execute any Turn-Ending Actions in the same turn as a mandatory diagnosis. Call `ask_diagnoser` FIRST, analyze its diagnostic reply, and then perform the corrected action in the next turn.\n\n## Phase 2: Planning & Memory Preservation\n1. **Autonomous Goal Management & Dynamic Live-Iteration Tracking**: You have autonomy over nested subgoals under the top-level milestones. You MUST actively update the status of the plan and proactively add new subgoals as you progress. Use `[/]` to denote the currently active milestone and subgoal. Only mark a step as completed (`[x]`) AFTER visual confirmation. Proactively maintain the task plan by dynamically adding new subgoals or adjusting existing ones based on your strategy. Keep the plan as a pure Markdown checklist in `task_plan`. Its grammar is machine-enforced:\n{{ plan_grammar }}\n - *Milestone Structure Preservation*: You are strictly forbidden from deleting, omitting, or modifying top-level `[Loop]`/`[Loop:continuous]` milestones or their declared exit conditions.\n - *Surgical Plan Edits*: Modify `task_plan` with targeted `update_note` (target/replacement) edits. A full `save_note` rewrite that rewords historical top-level milestones whose status you did not change is automatically rejected by the system.\n - *Dynamic Live-Iteration Ledger in Loops*: When executing a `[Loop]` milestone (e.g., candidate iteration or periodic monitoring), dynamically log each concrete iteration item in `task_plan` as you proceed (e.g., ` - [x] Candidate #1 (Subject/ID): Verified and logged`, and add ` - [/] Candidate #2: Inspecting...`). In monitoring tasks, log each cycle (e.g., ` - [x] Polling Check #1 (10:00): Refreshed, no new items, waiting 5m`). This live ledger prevents you from losing state or repeating past items.\n - *Loop Exit Evaluation*: In every turn of a loop, evaluate whether the milestone's Exit Condition has been satisfied (e.g., required data completely found, candidate inspection cap reached, battery threshold hit). For `[Loop:continuous]` milestones, the milestone MUST remain `[/]` and CANNOT be marked `[x]` by you until the user injects an explicit external stop signal. Once a bounded exit condition is legitimately met, mark the top-level `[Loop]` milestone as `[x]` to seamlessly transition to the next phase.\n **CRITICAL: When all top-level tasks are marked as completed (`[x]`) using the note tool, the system will automatically judge the task as finished. At this point, you do not need to output a Turn-Ending Action.**\n2. **Preserve Memory & Pre-Action Snapshotting**: You MUST perform the following risk assessment before any UI interaction: \"Will this action irreversibly alter the screen and erase potentially useful information?\" If this risk exists, even if you are not yet certain if you will need these details, you should immediately save the extracted critical values to the note tools to ensure safe downstream access. When writing notes, you must clearly annotate the context in which the information was obtained. Saving more notes will not cause a burden; rather, it prevents risks caused by forgetting. Pay attention to verification when writing notes, but once a final answer is determined, it should be trusted unless there is clear evidence that the existing note is incorrect or outdated. Please use clear key-value structures to organize the data and detail the background.[% if \"video_analyzer\" in available_tools %] If you have already missed certain critical information and cannot re-view it by going back or other operations, you can use the `video_analyzer` to look back at historical information.[% endif %]\n3. **Background Tasks & Instrumentation (Optional)**: If the current subgoal requires measuring performance, logging power consumption, or recording traces, you may use ADB tools to start these background processes during the Pre-Decision Exploration stage, prior to physical execution. You have full control over all ADB tasks. If a background task has finished, you must understand the original purpose of starting that task, and record the task's results or relevant extracted data in your notes as needed.\n\n## Phase 3: Target Selection & Physical Execution\n1. **Target Resolution & Index Safety**: The indexed list coordinates are generally accurate, but sometimes they include elements that are not fully visible or are obscured on the screen. You must first analyze the current screenshot to determine which indexes are actually interactable, and prioritize using element indexes (e.g., `target=3`) over raw coordinates or cached state whenever possible. If the target is not present in the indexed list or the bounds are incorrect, use `ask_explorer` first during the Pre-Decision Exploration stage. Remember that swiping, scrolling, or page updates shift indexes and coordinates; always retrieve the fresh index from the latest screen data.\n2. **Action Selection & Mandatory Cognitive Protocol**: Before calling any Turn-Ending Action ([[ tool_enum(turn_ending_actions, \"or\") ]]), you MUST write a single, fluid narrative paragraph of text as your raw thinking. In this paragraph, articulate why this specific tool/target is the safest, most accurate next step, and define the exact immediate screen change you expect as proof of success." } diff --git a/artemis/agents/operator/operator.py b/artemis/agents/operator/operator.py index 09c9e00..65ed44e 100644 --- a/artemis/agents/operator/operator.py +++ b/artemis/agents/operator/operator.py @@ -13,7 +13,6 @@ # limitations under the License. import json -import re from typing import Any from uuid import uuid4 @@ -46,12 +45,11 @@ from artemis.utils.coordinates import ( parse_swipe_parameters, ) from artemis.utils.decorators import wrap_with_callbacks +from artemis.utils.element_hit_test import find_element_at_point from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path -from artemis.utils.task_tree import ( - build_plan_and_history, - get_active_subgoal_hashes, -) +from artemis.memory.context_policy import build_history_for +from artemis.utils.task_tree import get_active_subgoal_hashes from artemis.utils.visualization import format_minimal_list_with_elements logger = get_logger(__name__) @@ -62,6 +60,7 @@ DEFERRING_TOOLS = { "read_note", "list_notes", "save_note", + "recall_history", "run_adb_command", "manage_task", "analyze_task_output", @@ -73,14 +72,16 @@ from artemis.agents.operator.prompts import ( PromptComponent, TemplatePromptComponent, ObservationPromptComponent, + PlanRecitationPromptComponent, ScreenshotSimilarityPromptComponent, + HistoricalStateHintPromptComponent, FeedbackPromptComponent, CheckItemsExplainerPromptComponent, BackgroundTasksPromptComponent, - ShortTermMemoryPromptComponent, TaskPlanWarningPromptComponent, ToolLimitWarningPromptComponent, InjectedInstructionPromptComponent, + render_transcript_static_system, ) from artemis.agents.operator.prompt_builder import load_operator_prompts @@ -92,6 +93,7 @@ class OperatorNode: tools: list[BaseTool] | None = None, prompt_components: list[PromptComponent] | None = None, last_n_detailed: int = 1, + transcript_config=None, ): self.ctx = ctx self.tools = tools or [] @@ -103,6 +105,24 @@ class OperatorNode: logger.error(f"Failed to load operator prompts: {e}") self.prompts = {} + # M2 transcript flag (agent.memory.transcript.enabled). Off keeps the + # legacy 2-message prompt path byte-for-byte; an explicit + # ``transcript_config`` overrides the loaded configuration (tests). + if transcript_config is not None: + self._transcript_cfg = transcript_config + else: + try: + from artemis.config import load_agent_config + + self._transcript_cfg = load_agent_config().memory.transcript + except Exception: + from artemis.config.agent import MemoryTranscriptConfig + + self._transcript_cfg = MemoryTranscriptConfig() + # Index into the rendered message list where this turn's tail begins; + # set by the transcript build and consumed after the tool loop. + self._transcript_turn_base: int | None = None + def _available_device_actions(self) -> frozenset[str]: """Device actions the installed actuator backend provides. @@ -170,6 +190,26 @@ class OperatorNode: active_background_tasks: list[dict] = None, newly_finished_tasks: list[dict] = None, ) -> list: + self._transcript_turn_base = None + if getattr(self._transcript_cfg, "enabled", False): + try: + return await self._build_prompt_transcript( + state, + latest_screenshot_b64, + minimal_list, + current_step_num, + steps, + task_plan, + active_background_tasks=active_background_tasks, + newly_finished_tasks=newly_finished_tasks, + ) + except Exception as e: + logger.error( + f"Transcript prompt path failed; falling back to the legacy" + f" 2-message build for this turn: {e}" + ) + self._transcript_turn_base = None + plan_and_history = self._build_plan_and_history(steps, task_plan) builder = PromptBuilder() @@ -187,8 +227,8 @@ class OperatorNode: (CheckItemsExplainerPromptComponent(), {}), (ObservationPromptComponent(), {}), (ScreenshotSimilarityPromptComponent(), {}), + (HistoricalStateHintPromptComponent(), {}), (InjectedInstructionPromptComponent(), {}), - (ShortTermMemoryPromptComponent(), {}), (BackgroundTasksPromptComponent(), {}), (FeedbackPromptComponent(), {}), (TaskPlanWarningPromptComponent(), {}), @@ -217,6 +257,174 @@ class OperatorNode: return builder.build() + # ------------------------------------------------------------------ + # Transcript prompt path (M2, flag agent.memory.transcript.enabled) + # ------------------------------------------------------------------ + + def _ensure_transcript_ledger(self, state=None): + """Session ledger on the composition root, created on first use.""" + from artemis.memory import TranscriptLedger, ensure_step_memory + + ledger = getattr(self.ctx, "transcript_ledger", None) + if isinstance(ledger, TranscriptLedger): + return ledger + cfg = self._transcript_cfg + ledger = TranscriptLedger( + step_memory=ensure_step_memory(self.ctx), + image_scrub_depth=getattr(cfg, "image_scrub_depth", 3), + pending_grace_steps=getattr(cfg, "pending_grace_steps", 3), + xml_scrub_depth=getattr(cfg, "xml_scrub_depth", 1), + ) + # L2/L3 chunk compression (M3) rides the same flag; without a + # DataEngine there are no step records to chunk, so the ledger simply + # runs scrub-edge-only. + if self.ctx.data_engine is not None: + try: + from artemis.memory import HistoryChunkManager + + chunking_cfg = None + try: + from artemis.config import load_agent_config + + chunking_cfg = load_agent_config().memory.chunking + except Exception: + pass + ledger.attach_chunker( + HistoryChunkManager( + engine=self.ctx.data_engine, + ctx=self.ctx, + chunking_config=chunking_cfg, + transcript_config=cfg, + goal=getattr(state, "initial_goal", None) if state else None, + ) + ) + except Exception as e: + logger.error(f"History chunk manager unavailable: {e}") + try: + self.ctx.transcript_ledger = ledger + except Exception: + pass + return ledger + + async def _build_prompt_transcript( + self, + state: State, + latest_screenshot_b64: str, + minimal_list: str, + current_step_num: int, + steps: list[dict], + task_plan: str, + active_background_tasks: list[dict] = None, + newly_finished_tasks: list[dict] = None, + ) -> list: + """Build ``S + F + A + tail`` from the session transcript ledger. + + The static system message renders once per session (byte-stable S + region); the previous turn is committed into the append-only active + region together with its step key and validator result; the fresh tail + carries the current observation plus the task-plan recitation. + """ + from langchain_core.messages import HumanMessage + + ledger = self._ensure_transcript_ledger(state) + + # 1. S region: rendered exactly once per session. + if not ledger.has_static_prefix: + static_text = render_transcript_static_system(self.prompts, self.ctx, state) + ledger.set_static_prefix([SystemMessage(content=static_text)]) + + # 2. F region cold start: an empty ledger over an existing step record + # trail means the process restarted — freeze the compiled history once. + # A staged (not yet committed) turn means this is turn 2+ of a live + # session, not a cold start — step records already exist then, and + # seeding would be rejected by the empty-ledger invariant. + if ( + ledger.turn_count == 0 + and not ledger.has_staged_turn + and not ledger.has_restored_history + and steps + ): + restored = build_history_for( + "operator_cold_start", + task_plan, + steps, + self._get_active_subgoal_hash(), + ) + ledger.set_restored_history( + "[Restored history] Rebuilt from the step records of this task" + " after a process restart; step times below are relative to the" + " original session start.\n" + f"{restored}" + ) + + # 3. Commit the previous turn (its step id and validator result only + # exist now). A turn without a terminal action never ran the validator, + # and ``structured_decisions`` is cleared on planner rejection — the + # gate keeps stale reports out of the ledger. + validator_result = ( + getattr(state, "last_execution_result", None) + if getattr(state, "structured_decisions", None) + else None + ) + ledger.commit_staged( + step_key=getattr(state, "current_step_id", None), + validator_result=validator_result, + ) + + # 4. Current tail: observation + plan recitation + injected components. + builder = PromptBuilder() + builder.add_human_content(f"# CURRENT OBSERVATION [{ledger.elapsed_label()}]") + components = [ + (PlanRecitationPromptComponent(), {}), + (CheckItemsExplainerPromptComponent(), {}), + (ObservationPromptComponent(), {}), + (ScreenshotSimilarityPromptComponent(), {}), + (HistoricalStateHintPromptComponent(), {}), + (InjectedInstructionPromptComponent(), {}), + (BackgroundTasksPromptComponent(), {}), + (FeedbackPromptComponent(), {}), + (TaskPlanWarningPromptComponent(), {}), + (ToolLimitWarningPromptComponent(), {}), + ] + for component, extra_kwargs in components: + kwargs_to_pass = { + "prompts": self.prompts, + "task_plan": task_plan, + "latest_screenshot_b64": latest_screenshot_b64, + "minimal_list": minimal_list, + "current_step_num": current_step_num, + "active_background_tasks": active_background_tasks or [], + "newly_finished_tasks": newly_finished_tasks or [], + "steps": steps, + } + kwargs_to_pass.update(extra_kwargs) + await component(builder, state, self.ctx, **kwargs_to_pass) + + tail_content = [] + for part in builder.human_parts: + if isinstance(part, str): + tail_content.append({"type": "text", "text": part}) + else: + tail_content.append(part) + tail = HumanMessage(content=tail_content) + + messages = ledger.render([tail]) + self._transcript_turn_base = len(messages) - 1 + return messages + + def _stage_transcript_turn(self, messages: list) -> None: + """Hand this turn's tail + tool-loop products to the ledger.""" + base = self._transcript_turn_base + self._transcript_turn_base = None + if base is None: + return + try: + ledger = getattr(self.ctx, "transcript_ledger", None) + if ledger is not None: + ledger.stage_turn(messages[base:]) + except Exception as e: + logger.error(f"Failed to stage transcript turn: {e}") + async def _invoke_llm_loop( self, base_llm, @@ -729,30 +937,18 @@ class OperatorNode: state=state, ) + # 8b. Transcript path: hold this turn's messages (observation tail + + # tool-loop products) for commit at the next build, when the turn's + # step id and validator result exist. + self._stage_transcript_turn(messages) + # 9. Update State structured_decisions = json.dumps(action_result) if action_result else None - short_term_memory = None - combined_thinking = "" - if raw_thinking: - combined_thinking += raw_thinking + "\n" - if native_thinking: - combined_thinking += native_thinking - - if combined_thinking: - stm_match = re.search( - r"(.*?)", - combined_thinking, - re.DOTALL, - ) - if stm_match: - short_term_memory = stm_match.group(1).strip() - return { "structured_decisions": structured_decisions, "operator_raw_thinking": raw_thinking, "operator_native_thinking": native_thinking, - "short_term_memory": short_term_memory, "indexed_points": state.indexed_points, "indexed_elements": state.indexed_elements, "current_step_id": state.current_step_id, @@ -785,14 +981,12 @@ class OperatorNode: """Builds a plan checklist and separate execution history.""" active_subgoal_hash = self._get_active_subgoal_hash() - return build_plan_and_history( + return build_history_for( + "operator", task_plan, steps, active_subgoal_hash, last_n_detailed=self.last_n_detailed, - strict_milestone_pruning=True, - recent_window_size=3, - chronological_last_step=True, ) def _translate_and_validate_tool(self, tc: dict, state: State) -> tuple[list[dict], str | None]: @@ -838,7 +1032,7 @@ class OperatorNode: return None, f"Error: Invalid target index {target}." indexed_elements = getattr(state, "indexed_elements", None) or [] if 1 <= target_int <= len(indexed_elements): - return indexed_elements[target_int - 1], None + return {**indexed_elements[target_int - 1], "label_source": "index"}, None return ( None, ( @@ -851,13 +1045,21 @@ class OperatorNode: nx, ny = map(float, target) x = int(max(0, min(width - 1, nx * width / 1000))) y = int(max(0, min(height - 1, ny * height / 1000))) + # Record-time enrichment: a bare-coordinate target carries no + # element semantics, so hit test the pre-action frame's indexed + # elements to best-effort recover what sits under the point. + hit_el, hit_source = find_element_at_point( + getattr(state, "indexed_elements", None), x, y + ) + hit_el = hit_el or {} return { "center": [x, y], - "text": None, - "bounds": None, - "class": None, - "resource_id": None, - "is_ocr": False, + "text": hit_el.get("text"), + "bounds": hit_el.get("bounds"), + "class": hit_el.get("class"), + "resource_id": hit_el.get("resource_id"), + "is_ocr": bool(hit_el.get("is_ocr")), + "label_source": hit_source, }, None except (ValueError, TypeError): return ( @@ -894,6 +1096,7 @@ class OperatorNode: "target_bounds": el.get("bounds"), "target_resource_id": el.get("resource_id"), "target_class": el.get("class"), + "target_label_source": el.get("label_source", "none"), } ], None @@ -917,6 +1120,7 @@ class OperatorNode: "target_bounds": el.get("bounds"), "target_resource_id": el.get("resource_id"), "target_class": el.get("class"), + "target_label_source": el.get("label_source", "none"), } ], None @@ -953,6 +1157,7 @@ class OperatorNode: "target_bounds": el.get("bounds"), "target_resource_id": el.get("resource_id"), "target_class": el.get("class"), + "target_label_source": el.get("label_source", "none"), } ) return actions, None diff --git a/artemis/agents/operator/prompts.py b/artemis/agents/operator/prompts.py index a32d11f..d016e79 100644 --- a/artemis/agents/operator/prompts.py +++ b/artemis/agents/operator/prompts.py @@ -52,7 +52,7 @@ from artemis.mcp.action_specs import OPERATOR_SHELL_ORDER _PRE_DECISION_HELPER_TOOLS = ("ask_explorer", "ask_diagnoser", "video_analyzer") _PRE_DECISION_ADB_TOOLS = ("run_adb_command", "manage_task", "analyze_task_output") -_PRE_DECISION_MEMORY_TOOLS = ("read_note", "list_notes", "save_note") +_PRE_DECISION_MEMORY_TOOLS = ("read_note", "list_notes", "save_note", "recall_history") _PRE_DECISION_ALL_TOOLS = ( _PRE_DECISION_HELPER_TOOLS + _PRE_DECISION_ADB_TOOLS + _PRE_DECISION_MEMORY_TOOLS ) @@ -159,6 +159,119 @@ class PromptComponent: raise NotImplementedError +def resolve_operator_prompt_tools(ctx: ArtemisContext) -> frozenset[str]: + """The tool set the operator prompt may advertise for this run. + + Extracted verbatim from the template component so the transcript static + prefix (M2) and the legacy per-turn render assemble against the identical + availability set. + """ + # video_analyzer is bound conditionally (graph.py gates it on + # video_recording_tools_enabled); the prompt must not advertise it when the + # tool is not actually available this run. + available = set(OPERATOR_PROMPT_TOOLSET) + setup = getattr(ctx, "execution_setup", None) + if not (setup and getattr(setup, "video_recording_tools_enabled", False)): + available.discard("video_analyzer") + + # recall_history (M4) needs a DataEngine to search and is config-gated; + # the prompt must not advertise it when the tool is not bound this run. + if getattr(ctx, "data_engine", None) is None or not _recall_enabled(): + available.discard("recall_history") + + # Device actions the installed actuator backend does not implement disappear + # from the prompt in lockstep with their tool declarations. + actuator = getattr(ctx, "actuator", None) + if actuator is not None and callable(getattr(actuator, "capabilities", None)): + try: + from artemis.mcp.action_manifest import ( + DEVICE_ACTIONS, + available_device_actions, + ) + + available -= DEVICE_ACTIONS - available_device_actions(actuator) + except Exception as e: + logger.warning(f"Failed to assemble prompt against actuator: {e}") + + return frozenset(available) + + +def _operator_grammar_flags(ctx: ArtemisContext) -> tuple[bool, bool]: + """(include_checks, verification_active) for the template render.""" + setup = getattr(ctx, "execution_setup", None) + # Grammar spec assembly is a function of configuration: with both check + # gates disabled, the check-line grammar never enters any prompt. + include_checks = bool(setup and getattr(setup, "checks_enabled", False)) + # The rejection/finding diagnosis trigger only exists while a mechanism + # that can produce rejections or findings is active. + verification_active = include_checks or bool( + setup and not getattr(setup, "disable_planner_validation", True) + ) + return include_checks, verification_active + + +# --- M2 template split ----------------------------------------------------------- +# The only volatile span of operator.json's main_template is the plan+history +# section below. The transcript path replaces it with a static pointer so the +# whole system message becomes a byte-stable S region; the legacy path renders +# the untouched template and stays byte-identical. + +#: The volatile section of ``main_template`` (must occur exactly once). +PLAN_HISTORY_TEMPLATE_SECTION = "## Current Plan & Execution History\n{{ plan_and_history }}" + +#: Static replacement used by the transcript S region. +PLAN_HISTORY_STATIC_POINTER = ( + "## Current Plan & Execution History\n" + "Provided in the conversation that follows: earlier turns carry the raw" + " step-by-step execution history (a restored-history block precedes them" + " after a process restart), and each turn's observation message recites" + " the current task plan." +) + + +def render_transcript_static_system( + prompts: dict, + ctx: ArtemisContext, + state: State, + template_name: str = "main_template", +) -> str: + """Render the transcript path's byte-stable static system prompt (S region). + + Same template, same availability assembly, and the same render inputs as + the legacy path — except the volatile plan+history section is swapped for + a static pointer, so the output never changes across the session. + """ + prompt_template = prompts.get(template_name) + if not prompt_template: + raise KeyError( + "Failed to format prompt, template not found in operator prompts config." + ) + if prompt_template.count(PLAN_HISTORY_TEMPLATE_SECTION) != 1: + # Hard dependency of the M2 split (redesign §9): without a clean + # section boundary the S region cannot be byte-stable. + raise ValueError( + "operator main_template no longer contains exactly one plan+history" + " section; the transcript static split cannot be applied." + ) + static_template = prompt_template.replace( + PLAN_HISTORY_TEMPLATE_SECTION, PLAN_HISTORY_STATIC_POINTER + ) + + available = resolve_operator_prompt_tools(ctx) + static_template = apply_operator_prompt_contract( + static_template, available_tools=available + ) + include_checks, verification_active = _operator_grammar_flags(ctx) + return Template(static_template).render( + initial_goal=state.initial_goal, + subgoals_status="", + plan_and_history="", + unified_history="", + plan_grammar=render_plan_grammar_spec(include_checks), + verification_active=verification_active, + ) + + class TemplatePromptComponent(PromptComponent): async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisContext, **kwargs): prompts = kwargs.get("prompts", {}) @@ -169,42 +282,15 @@ class TemplatePromptComponent(PromptComponent): "Failed to format prompt, template not found in operator prompts config." ) - # video_analyzer is bound conditionally (graph.py gates it on - # video_recording_tools_enabled); the prompt must not advertise it when the - # tool is not actually available this run. - available = set(OPERATOR_PROMPT_TOOLSET) - setup = getattr(ctx, "execution_setup", None) - if not (setup and getattr(setup, "video_recording_tools_enabled", False)): - available.discard("video_analyzer") - - # Device actions the installed actuator backend does not implement disappear - # from the prompt in lockstep with their tool declarations. - actuator = getattr(ctx, "actuator", None) - if actuator is not None and callable(getattr(actuator, "capabilities", None)): - try: - from artemis.mcp.action_manifest import ( - DEVICE_ACTIONS, - available_device_actions, - ) - - available -= DEVICE_ACTIONS - available_device_actions(actuator) - except Exception as e: - logger.warning(f"Failed to assemble prompt against actuator: {e}") + available = resolve_operator_prompt_tools(ctx) prompt_template = apply_operator_prompt_contract( - prompt_template, available_tools=frozenset(available) + prompt_template, available_tools=available ) plan_and_history = kwargs.get("plan_and_history", "No plan or history yet.") - # Grammar spec assembly is a function of configuration: with both check - # gates disabled, the check-line grammar never enters any prompt. - include_checks = bool(setup and getattr(setup, "checks_enabled", False)) - # The rejection/finding diagnosis trigger only exists while a mechanism - # that can produce rejections or findings is active. - verification_active = include_checks or bool( - setup and not getattr(setup, "disable_planner_validation", True) - ) + include_checks, verification_active = _operator_grammar_flags(ctx) full_prompt = Template(prompt_template).render( initial_goal=state.initial_goal, @@ -237,6 +323,21 @@ class ObservationPromptComponent(PromptComponent): builder.add_human_content(f"--- Visible UI Elements ---\n{minimal_list}") +class PlanRecitationPromptComponent(PromptComponent): + """Per-turn task-plan recitation for the transcript tail (M2). + + With the plan+history section moved out of the (now static) system + message, every observation recites the live task plan; the scrub edge + strips the copies from older turns at depth 1. + """ + + async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisContext, **kwargs): + from artemis.memory.transcript import PLAN_RECITATION_MARKER + + task_plan = kwargs.get("task_plan") or "No task plan yet." + builder.add_human_content(f"{PLAN_RECITATION_MARKER}\n{task_plan}") + + class FeedbackPromptComponent(PromptComponent): """Append-only injection of source-tagged findings. @@ -336,14 +437,6 @@ class BackgroundTasksPromptComponent(PromptComponent): builder.add_human_content("\n".join(lines) + "\n") -class ShortTermMemoryPromptComponent(PromptComponent): - async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisContext, **kwargs): - if state.short_term_memory: - builder.add_human_content( - f"--- Short-Term Memory (Scratchpad) ---\n{state.short_term_memory}\n" - ) - - class TaskPlanWarningPromptComponent(PromptComponent): async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisContext, **kwargs): steps = kwargs.get("steps", []) @@ -506,3 +599,95 @@ class ScreenshotSimilarityPromptComponent(PromptComponent): if diff_count > max_allowed: break return diff_count + + +def _recall_enabled() -> bool: + """Whether the recall_history tool is enabled by configuration (M4).""" + try: + from artemis.config import load_agent_config + + return bool(load_agent_config().memory.recall.enabled) + except Exception: + return True + + +class HistoricalStateHintPromptComponent(PromptComponent): + """Local historical-state hint from stored perceptual hashes (M4). + + Compares the current screenshot's dHash against the post-action hashes + stamped on every recorded step (``extra_metadata["post_image_dhash"]``, + including steps already chunk-compressed out of the visible transcript) — + an O(n) integer scan, no model call, no historical image bytes. On a + close match to a step *older* than the recent window it injects a one-line + hint pointing at ``recall_history``. + + Division of labor with :class:`ScreenshotSimilarityPromptComponent`: that + component's pixel-exact 3-step look-back covers "stuck on the same + screen"; this one covers "returned to a much earlier state". When any of + the last :attr:`RECENT_SILENT_STEPS` steps also matches, this hint stays + silent so the two never fire on the same regime. + """ + + #: Matches within this many most-recent steps stay silent (the pixel + #: same-screen note owns that window). + RECENT_SILENT_STEPS: int = 3 + #: Upper bound on how many historical steps are scanned (most recent first). + SCAN_CAP: int = 500 + + async def __call__(self, builder: PromptBuilder, state: State, ctx: ArtemisContext, **kwargs): + latest_screenshot_b64 = kwargs.get("latest_screenshot_b64") + steps = kwargs.get("steps") or [] + if not latest_screenshot_b64 or len(steps) <= self.RECENT_SILENT_STEPS: + return + + max_distance = 8 + try: + from artemis.config import load_agent_config + + transcript_cfg = load_agent_config().memory.transcript + if not getattr(transcript_cfg, "similarity_hint", True): + return + max_distance = int(getattr(transcript_cfg, "similarity_max_distance", 8)) + except Exception: + pass + + try: + from artemis.utils.image_hash import dhash_hex, hamming_distance_hex + + if "," in latest_screenshot_b64: + latest_screenshot_b64 = latest_screenshot_b64.split(",", 1)[1] + current_hash = dhash_hex(base64.b64decode(latest_screenshot_b64)) + except Exception: + return + if not current_hash: + return + + def _step_hash(step: dict) -> str | None: + meta = step.get("extra_metadata") or {} + return meta.get("post_image_dhash") or meta.get("pre_image_dhash") + + # Silence rule first: a close match inside the recent window belongs + # to the pixel-level same-screen note, not this hint. + for step in steps[-self.RECENT_SILENT_STEPS :]: + distance = hamming_distance_hex(current_hash, _step_hash(step)) + if distance is not None and distance <= max_distance: + return + + older_steps = steps[: -self.RECENT_SILENT_STEPS][-self.SCAN_CAP :] + best_step_number = None + best_distance = None + for step in older_steps: + distance = hamming_distance_hex(current_hash, _step_hash(step)) + if distance is None or distance > max_distance: + continue + if best_distance is None or distance <= best_distance: + # <= keeps the most recent step on ties. + best_distance = distance + best_step_number = step.get("step_number") + + if best_step_number is not None: + builder.add_human_content( + f"Historical state hint: current screen closely resembles the" + f" post-action screen from Step {best_step_number}. Use" + " recall_history only if its details are needed." + ) diff --git a/artemis/agents/outputter/outputter.py b/artemis/agents/outputter/outputter.py index 554eb64..0ac75f5 100644 --- a/artemis/agents/outputter/outputter.py +++ b/artemis/agents/outputter/outputter.py @@ -38,7 +38,8 @@ from artemis.tools.scratchpad import ( from artemis.tools.video_tool import get_video_analyzer_tool_pure from artemis.tools.tool_wrapper import invoke_tool_with_injection from artemis.utils.logger import get_logger -from artemis.utils.task_tree import build_plan_and_history, get_active_subgoal_hashes +from artemis.memory.context_policy import build_history_for +from artemis.utils.task_tree import get_active_subgoal_hashes from pydantic import BaseModel logger = get_logger(__name__) @@ -78,13 +79,12 @@ async def outputter( except Exception as e: logger.error(f"Failed to parse active subgoal in outputter: {e}") - plan_and_history = build_plan_and_history( + plan_and_history = build_history_for( + "outputter", current_plan, history, active_subgoal_hash, - last_n_detailed=0, - min_summaries=len(history), - strict_milestone_pruning=False, + engine=ctx.data_engine, ) except Exception as e: logger.error(f"Failed to resolve plan and history in outputter: {e}") diff --git a/artemis/agents/planner/planner.py b/artemis/agents/planner/planner.py index 984d6e1..f1013c1 100644 --- a/artemis/agents/planner/planner.py +++ b/artemis/agents/planner/planner.py @@ -47,7 +47,7 @@ from artemis.utils.decorators import wrap_with_callbacks from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path from artemis.utils.plan_grammar import render_plan_grammar_spec -from artemis.utils.task_tree import build_plan_and_history +from artemis.memory.context_policy import build_history_for logger = get_logger(__name__) @@ -130,14 +130,11 @@ async def run_async_planner_validation( if ctx.data_engine: try: steps = ctx.data_engine.get_agent_friendly_steps() - history_str = build_plan_and_history( + history_str = build_history_for( + "planner", "", # Pass empty string to avoid rendering the task plan in the history steps, "default", - last_n_detailed=2, - strict_milestone_pruning=True, - recent_window_size=5, - chronological_last_step=True, ) except Exception as e: diff --git a/artemis/agents/summarizer/summarizer.py b/artemis/agents/summarizer/summarizer.py index 5b0c313..2807e3f 100644 --- a/artemis/agents/summarizer/summarizer.py +++ b/artemis/agents/summarizer/summarizer.py @@ -12,27 +12,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio -import json -from pathlib import Path -from uuid import UUID +"""Pro-graph step summarizer node (M2: visual-transition lens dispatch). -from langchain_core.messages import ( - HumanMessage, - SystemMessage, -) +Per the history-module redesign §5 (single step-level lens, confirmed +2026-08-31) this node no longer generates a per-step text capsule with its own +LLM call. It dispatches the step to the shared +:class:`~artemis.memory.step_memory.StepMemoryService` visual-transition lens +(``ctx.step_memory``): pre/post screenshots are pulled from the DataEngine by +step id, the red action-overlay and the ``flash_summarizer.md`` neutral-wording +contract apply unchanged, and the result lands as a versioned +``source="visual_transition"`` summary write (M0 semantics). + +The auxiliary agents' compiled view (``build_plan_and_history``) is untouched: +a step whose summary is not ready yet falls back to detailed rendering +(``task_tree.py`` semantics), so the transition is safe. The intent/strategy +capsule moves up to the chunk-level ``StepCapsuleLens`` in M3. +""" + +import json +from typing import Any from artemis.context import ArtemisContext -from artemis.data_engine.trace import CURRENT_TRACE_ID, trace +from artemis.data_engine.trace import trace from artemis.graph.state import State from artemis.graph.visibility import strict_state -from artemis.services.llm import acomplete, get_llm -from artemis.utils.coordinates import normalize_any_structure +from artemis.memory import ensure_step_memory from artemis.utils.logger import get_logger -from artemis.utils.task_tree import ( - build_plan_and_history, - get_active_subgoal_hashes, -) +from artemis.utils.task_tree import format_result_clean logger = get_logger(__name__) @@ -44,180 +50,90 @@ class SummarizerNode: @trace(type="agent", name="summarizer") async def __call__(self, state: State): state = strict_state(state, "summarizer") - # 1. Trigger background task for summary and data engine update - if self.ctx.data_engine and state.current_step_id: - step_id = UUID(state.current_step_id) - decisions = state.structured_decisions - operator_raw_thinking = getattr(state, "operator_raw_thinking", None) - operator_native_thinking = getattr(state, "operator_native_thinking", None) - last_execution_result = getattr(state, "last_execution_result", None) - - # Spawn background task - trace_id = CURRENT_TRACE_ID.get() - task = asyncio.create_task( - self._generate_summary_and_update_engine( - step_id, - decisions, - operator_raw_thinking, - operator_native_thinking, - last_execution_result, - trace_id, - ), - name=f"summarizer_step_{step_id}", - ) - if hasattr(self.ctx, "background_tasks") and isinstance( - self.ctx.background_tasks, list - ): - self.ctx.background_tasks.append(task) - + if not (self.ctx.data_engine and state.current_step_id): + return {} + try: + self._dispatch_visual_summary(state) + except Exception as e: + logger.error(f"Failed to dispatch visual step summary: {e}") return {} - async def _generate_summary_and_update_engine( - self, - step_id, - decisions, - operator_raw_thinking, - operator_native_thinking, - last_execution_result, - trace_id, - ): - """Background task to generate summary and update Data Engine with retries.""" + def _dispatch_visual_summary(self, state: State) -> None: + engine = self.ctx.data_engine + step_id = state.current_step_id - max_retries = 3 - retry_delay = 2 + step_number = None + try: + step_number = engine.get_step_number(step_id) + except Exception as e: + logger.warning(f"Could not resolve step number for {step_id}: {e}") - for attempt in range(max_retries): + record = None + if isinstance(step_number, int): try: - logger.info(f"Generating summary for step {step_id} (attempt {attempt + 1})") - - # Prepare prompt for summary - llm = get_llm(ctx=self.ctx, name="summarizer") - - prompt_path = Path(__file__).parent.joinpath("summarizer.md") - if not prompt_path.exists(): - prompt = ( - "You are a summarizer. Summarize the latest action and" - " its result in 1-2 concise sentences. Focus on: Action" - " taken -> Result -> Observation." - ) - else: - prompt = prompt_path.read_text(encoding="utf-8") - - # Normalize coordinates before presenting to the summarizer LLM - width = ( - getattr(self.ctx.device, "device_width", 1080) - if self.ctx and self.ctx.device - else 1080 - ) - height = ( - getattr(self.ctx.device, "device_height", 2400) - if self.ctx and self.ctx.device - else 2400 - ) - - content = [] - - # Consistently use optimal build_plan_and_history to construct accurate context - plan_and_history_str = "" - if self.ctx.data_engine: - try: - friendly_steps = self.ctx.data_engine.get_agent_friendly_steps() - if isinstance(friendly_steps, list): - # 1. Trim history to only include steps up to the current step being summarized (inclusive) - current_steps = [] - for s in friendly_steps: - current_steps.append(s) - if str(s.get("step_id")) == str(step_id): - break - - # 2. Read task plan to provide goal orientation to the summarizer - task_plan = "" - base_dir = getattr(self.ctx.data_engine, "base_dir", None) - if base_dir and isinstance(base_dir, (str, Path)): - notes_dir = Path(base_dir) / "notes" - task_plan_path = notes_dir / "task_plan.md" - if task_plan_path.exists(): - task_plan = task_plan_path.read_text(encoding="utf-8") - - # 3. Unified rendering: concise for first N steps (with self-healing facts), detailed for current step (without redundant summary header) - - subgoal_hash = "default" - if task_plan: - subgoal_hash, _ = get_active_subgoal_hashes(task_plan) - - plan_and_history_str = build_plan_and_history( - task_plan=task_plan, - steps=current_steps, - current_subgoal_hash=subgoal_hash, - last_n_detailed=1, - min_summaries=10, - strict_milestone_pruning=False, - ) - except Exception as e: - logger.error(f"Failed to build plan and history for Summarizer: {e}") - - if plan_and_history_str: - content.append(plan_and_history_str) - else: - # Fallback to standard raw fields if chronological trace couldn't be generated - if decisions: - try: - dec_obj = json.loads(decisions) - dec_obj = normalize_any_structure(dec_obj, width, height) - content.append( - f"Decisions made: {json.dumps(dec_obj, ensure_ascii=False)}" - ) - except Exception: - content.append(f"Decisions made: {decisions}") - - if operator_raw_thinking: - content.append(f"Operator explicit thoughts:\n{operator_raw_thinking}") - - if operator_native_thinking: - content.append(f"Operator native thoughts:\n{operator_native_thinking}") - - messages = [ - SystemMessage(content=prompt), - HumanMessage(content="\n\n".join(content)), - ] - - aggregated_chunk = await acomplete(llm, messages) - - generated_text = "" - content = getattr(aggregated_chunk, "content", "") if aggregated_chunk else "" - if isinstance(content, str): - generated_text = content - elif isinstance(content, list): - for item in content: - if isinstance(item, dict) and item.get("type") == "text": - generated_text += item.get("text", "") - - summary = generated_text.strip() - logger.info(f"Summary generated: {summary}") - - if aggregated_chunk and getattr(aggregated_chunk, "usage_metadata", None): - usage = aggregated_chunk.usage_metadata - cached_tokens = usage.get("input_token_details", {}).get("cache_read", 0) - logger.info( - "Summarizer LLM usage:" - f" prompt_tokens={usage.get('input_tokens')}" - f" (cached={cached_tokens})," - f" completion_tokens={usage.get('output_tokens')}" - ) - - # Update Data Engine - self.ctx.data_engine.update_step_summary(step_id, summary) - logger.info(f"Data Engine updated with summary for step {step_id}") - return # Success - + record = engine.get_step_record(step_number) except Exception as e: - logger.error( - f"Failed to generate summary or update engine (attempt {attempt + 1}): {e}" - ) - if attempt < max_retries - 1: - await asyncio.sleep(retry_delay) - retry_delay *= 2 # Exponential backoff - else: - logger.error( - f"Max retries reached for step {step_id}. Summary generation failed." - ) + logger.warning(f"Could not load step record {step_number}: {e}") + + # Graceful degradation: a missing pre/post image simply dispatches with + # None bytes — the lens renders a text-only transition account. + pre_bytes = self._read_step_image(step_number, "pre") + post_bytes = self._read_step_image(step_number, "post") + + action_name, action_args = self._extract_action(record, state) + result = getattr(state, "last_execution_result", None) + # format_result_clean reports only errors/repairs; a clean run keeps + # the controller's bare status word. + exec_outcome = format_result_clean(result) + if not exec_outcome: + if isinstance(result, dict) and result.get("status"): + exec_outcome = str(result["status"]) + else: + exec_outcome = "unknown" + + service = ensure_step_memory(self.ctx) + service.dispatch( + step_number=step_number if isinstance(step_number, int) else 0, + action_name=action_name, + action_args=action_args, + pre_img_bytes=pre_bytes, + post_img_bytes=post_bytes, + exec_outcome=exec_outcome, + data_engine_step_id=step_id, + ) + logger.info( + f"Dispatched visual transition summary for step {step_number} ({step_id})" + ) + + def _read_step_image(self, step_number: int | None, which: str) -> bytes | None: + if not isinstance(step_number, int): + return None + try: + path = self.ctx.data_engine.get_step_image_path(step_number, which) + if path is None: + return None + return path.read_bytes() + except Exception as e: + logger.warning(f"Could not read {which} image for step {step_number}: {e}") + return None + + @staticmethod + def _extract_action(record: Any, state: State) -> tuple[str, dict]: + actions = getattr(record, "action_taken", None) if record is not None else None + if not actions: + decisions = getattr(state, "structured_decisions", None) + if decisions: + try: + actions = json.loads(decisions) + except Exception: + actions = None + if isinstance(actions, dict): + actions = [actions] + if not isinstance(actions, list) or not actions or not isinstance(actions[0], dict): + return "no_op", {} + first = actions[0] + action_name = first.get("action") or "unknown" + action_args = {k: v for k, v in first.items() if k != "action"} + if len(actions) > 1: + action_args["additional_actions"] = actions[1:] + return action_name, action_args diff --git a/artemis/agents/validator/failure_analyzer.py b/artemis/agents/validator/failure_analyzer.py index 5f3f146..2938830 100644 --- a/artemis/agents/validator/failure_analyzer.py +++ b/artemis/agents/validator/failure_analyzer.py @@ -51,8 +51,8 @@ from artemis.services.llm import get_llm from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path from artemis.utils.ocr_xml_fusion import fuse_ocr_with_xml +from artemis.memory.context_policy import build_history_for from artemis.utils.task_tree import ( - build_plan_and_history, format_action_clean, get_active_subgoal_hashes, ) @@ -358,15 +358,11 @@ class FailureAnalyzer: def _build_plan_and_history(self, steps: list[dict], task_plan: str) -> str: active_subgoal_hash = self._get_active_subgoal_hash() - return build_plan_and_history( + return build_history_for( + "failure_analyzer", task_plan, steps, active_subgoal_hash, - last_n_detailed=1, - strict_milestone_pruning=True, - recent_window_size=3, - chronological_last_step=True, - for_failure_analyzer=True, ) def _prune_intermediate_screenshots(self, messages: list[Any]): diff --git a/artemis/config/__init__.py b/artemis/config/__init__.py index 68a8b1c..e1022e8 100644 --- a/artemis/config/__init__.py +++ b/artemis/config/__init__.py @@ -20,6 +20,10 @@ from artemis.config.agent import ( CommitteeConfig, ExplorerConfig, FlashProfileConfig, + MemoryChunkingConfig, + MemoryConfig, + MemoryRuntimeConfig, + MemoryTranscriptConfig, OutputterConfig, PlannerValidationConfig, ProProfileConfig, @@ -197,6 +201,10 @@ __all__ = [ "CommitteeConfig", "ExplorerConfig", "FlashProfileConfig", + "MemoryChunkingConfig", + "MemoryConfig", + "MemoryRuntimeConfig", + "MemoryTranscriptConfig", "OutputterConfig", "PlannerValidationConfig", "ProProfileConfig", diff --git a/artemis/config/agent.py b/artemis/config/agent.py index 0ec8fcc..b017e7f 100644 --- a/artemis/config/agent.py +++ b/artemis/config/agent.py @@ -289,6 +289,262 @@ class StepSummarizerConfig(BaseModel): default=True, description="Whether to prune outdated heavy UI XML trees from historical turns.", ) + retry_limit: int = Field( + default=3, + ge=0, + description=( + "Maximum retries per step summary after the first attempt; on" + " exhaustion the step is marked summary_status=failed instead of" + " retrying forever." + ), + ) + model_config = {"extra": "allow"} + + +class MemoryRuntimeConfig(BaseModel): + """Scheduling options for the shared step-memory runtime (agent.memory.runtime).""" + + max_concurrency: int = Field( + default=2, + ge=1, + le=16, + description=( + "Maximum background summary attempts running concurrently." + " Default 2 per the 2026-09-01 on-device baseline" + " (history-baseline-2026-09-01.md §6): the serial queue at 1 pushed" + " summary-ready P90 past the scrub-edge grace window; lens calls" + " are independent lightweight-model requests, so 2 is low-risk." + ), + ) + retry_limit: int = Field( + default=3, + ge=0, + description=( + "Maximum retries per summary job after the first attempt; on" + " exhaustion the job enters an explicit failed state." + ), + ) + flush_timeout_s: float = Field( + default=30.0, + gt=0, + description="Upper bound in seconds for draining in-flight summary jobs at shutdown.", + ) + model_config = {"extra": "allow"} + + +class MemoryTranscriptConfig(BaseModel): + """Scrub-edge options for the message transcript (agent.memory.transcript).""" + + enabled: bool = Field( + default=True, + description=( + "Whether the Pro operator builds its prompt from the session" + " transcript ledger (S/F/A regions + per-turn tail) instead of the" + " legacy per-turn 2-message rebuild. On by default since M5" + " (2026-09-01, per the on-device A/B in" + " history-baseline-2026-09-01.md); this is also the rollback" + " switch — setting it to false restores the legacy path" + " byte-for-byte, and L2/L3 chunk compression rides the same flag." + ), + ) + image_scrub_depth: int = Field( + default=3, + ge=1, + description=( + "Depth K of the screenshot scrub edge: the K-th most recent" + " historical screenshot is replaced by its visual summary." + ), + ) + pending_grace_steps: int = Field( + default=3, + ge=0, + description=( + "Extra image-depths a step may retain its screenshot past the" + " scrub edge while its summary is still pending; afterwards the" + " image is replaced by a placeholder referencing the DataEngine step." + ), + ) + xml_scrub_depth: int = Field( + default=1, + ge=1, + description=( + "How many of the most recent observations keep their UI Element" + " list; older lists are stripped (depth 1 matches the legacy" + " prune_history_xml semantics)." + ), + ) + context_budget_tokens: int = Field( + default=80000, + ge=1024, + description=( + "Measured-token budget for the transcript context (provisional" + " default pending the on-device baseline; §3.4). The soft/hard" + " ratios below are applied against this budget." + ), + ) + soft_ratio: float = Field( + default=0.7, + gt=0.0, + lt=1.0, + description=( + "Soft threshold: when the last measured prompt size reaches" + " budget*soft_ratio, the oldest open segment is chunk-compressed" + " (L2)." + ), + ) + hard_ratio: float = Field( + default=0.9, + gt=0.0, + le=1.0, + description=( + "Hard threshold: when the last measured prompt size reaches" + " budget*hard_ratio, the frozen region collapses to the L3 session" + " snapshot (chunk headers merged; per-step index retained)." + ), + ) + min_active_steps: int = Field( + default=5, + ge=1, + description=( + "Sliding-window floor: no compression trigger may shrink the" + " active region below this many most-recent raw turns." + ), + ) + similarity_hint: bool = Field( + default=True, + description=( + "Whether the operator prompt injects a local historical-state hint" + " when the current screen's perceptual hash closely matches a much" + " older step's post-action screen (no model call, no historical" + " image). Silent when the match is within the recent 3 steps —" + " that regime belongs to the pixel-level same-screen note." + ), + ) + similarity_max_distance: int = Field( + default=5, + ge=0, + le=64, + description=( + "Maximum Hamming distance between 64-bit dHash values for the" + " historical-state hint to fire. Calibrated 2026-09-01 from 460" + " dHash-stamped on-device steps (16 sessions): same-screen" + " re-captures cluster at distance <=4, genuinely different screens" + " mass at >=7 (valley at 5-6); cross-app pairs (guaranteed" + " different screens) score <=5 in only 0.14% of cases vs 0.24% at" + " the previous provisional 8." + ), + ) + model_config = {"extra": "allow"} + + +class MemoryRecallConfig(BaseModel): + """History recall tool options (agent.memory.recall).""" + + enabled: bool = Field( + default=True, + description=( + "Whether the Pro operator gets the recall_history tool for" + " deterministic lookups into cold (compressed/evicted) history." + ), + ) + max_results: int = Field( + default=5, + ge=1, + le=20, + description="Upper bound on results per recall_history call.", + ) + max_text_tokens: int = Field( + default=2000, + ge=128, + description=( + "Estimated-token cap (char/4) on one recall_history text response;" + " excerpts are truncated to fit." + ), + ) + max_image_steps: int = Field( + default=1, + ge=0, + le=1, + description=( + "Maximum number of steps whose real stored screenshots may be" + " returned as data URLs when include_images is requested." + ), + ) + model_config = {"extra": "allow"} + + +class MemoryChunkingConfig(BaseModel): + """Segment (L2) chunk compression options (agent.memory.chunking).""" + + max_steps: int = Field( + default=12, + ge=1, + description=( + "Size-threshold trigger: an open segment of this many steps is" + " chunk-compressed even without a milestone switch. Also the" + " maximum steps per chunk when a longer segment is split." + ), + ) + target_source_tokens: int = Field( + default=2000, + ge=128, + description=( + "Size-threshold trigger: estimated source tokens (char/4 over the" + " open segment's transcript text) that force a chunk compression." + ), + ) + model: str = Field( + default="gemini-3.7-flash", + description="Model used for the chunk-level StepCapsuleLens (bands ①+②).", + ) + max_chunks: int = Field( + default=8, + ge=1, + description=( + "Maximum full three-band chunk blocks kept in the frozen region;" + " older chunks merge into eras (headers set-merged, ledgers kept)." + ), + ) + max_eras: int | None = Field( + default=None, + ge=1, + description=( + "Maximum eras kept with their per-segment ledgers; older eras" + " collapse their ledgers to recall_history marker lines. None" + " (default) follows max_chunks, preserving the pre-M5 equal-value" + " behavior." + ), + ) + model_config = {"extra": "allow"} + + +class MemoryConfig(BaseModel): + """Unified step-memory configuration block (agent.memory).""" + + runtime: MemoryRuntimeConfig = Field( + default_factory=MemoryRuntimeConfig, + description="Shared background summarization runtime scheduling options.", + ) + transcript: MemoryTranscriptConfig = Field( + default_factory=MemoryTranscriptConfig, + description="Transcript scrub-edge options.", + ) + chunking: MemoryChunkingConfig = Field( + default_factory=MemoryChunkingConfig, + description="Segment chunk (L2) compression options.", + ) + recall: MemoryRecallConfig = Field( + default_factory=MemoryRecallConfig, + description="History recall tool options.", + ) + policies: dict[str, dict[str, Any]] = Field( + default_factory=dict, + description=( + "Per-agent overrides of the compiled-history ContextPolicy table" + " (artemis.memory.context_policy); keys are agent names, values" + " are policy field overrides (e.g. {'planner': {'last_n_detailed': 3}})." + ), + ) model_config = {"extra": "allow"} @@ -398,6 +654,10 @@ class AgentGlobalConfig(BaseModel): default_factory=VideoAnalyzerConfig, description="Video analyzer runtime options.", ) + memory: MemoryConfig = Field( + default_factory=MemoryConfig, + description="Unified step-memory runtime and transcript scrub options.", + ) model_config = {"extra": "allow"} @@ -452,6 +712,18 @@ class AgentGlobalConfig(BaseModel): ): data.setdefault("pro", {})["video_analyzer"] = data["video_analyzer"] + # 3b. Legacy alias: an explicit flash.step_summarizer.retry_limit keeps + # working by seeding memory.runtime.retry_limit when the new key is + # absent (an explicit new key wins over the legacy one). + flash_block = data.get("flash") + flash_ss = flash_block.get("step_summarizer") if isinstance(flash_block, dict) else None + if isinstance(flash_ss, dict) and "retry_limit" in flash_ss: + mem_block = data.setdefault("memory", {}) + if isinstance(mem_block, dict): + runtime_block = mem_block.setdefault("runtime", {}) + if isinstance(runtime_block, dict): + runtime_block.setdefault("retry_limit", flash_ss["retry_limit"]) + # 4. Environment variable overrides (highest precedence for runtime switches) def _parse_bool(var_name: str) -> bool | None: v = os.getenv(var_name) diff --git a/artemis/constants.py b/artemis/constants.py index 7af5245..dd60d48 100644 --- a/artemis/constants.py +++ b/artemis/constants.py @@ -13,7 +13,6 @@ # limitations under the License. RECURSION_LIMIT = 30000 -MAX_MESSAGES_IN_HISTORY = 25 # Optimization Constants CHECKER_MAX_ITERATIONS = 20 diff --git a/artemis/context.py b/artemis/context.py index 487f8f7..a82d297 100644 --- a/artemis/context.py +++ b/artemis/context.py @@ -232,6 +232,17 @@ class ArtemisContext(BaseModel): """Optional actuator backend override (artemis.mcp.actuators). ``None`` selects the default AdbActuator; installing e.g. a robot-arm backend happens here.""" + step_memory: Any | None = None + """Shared :class:`~artemis.memory.step_memory.StepMemoryService` composition + root (history redesign §6.2). Created on first use via + ``artemis.memory.ensure_step_memory``; Flash and Pro profiles share one + instance so summary jobs, retries, and flush semantics are unified.""" + + transcript_ledger: Any | None = None + """Session :class:`~artemis.memory.transcript.TranscriptLedger` for the Pro + operator's transcript prompt path (``agent.memory.transcript.enabled``). + Lives beside ``step_memory``; the State never carries message payloads.""" + async def __aenter__(self) -> ArtemisContext: return self @@ -246,6 +257,31 @@ class ArtemisContext(BaseModel): finally: self.action_session = None + # Drain in-flight step-summary jobs before the generic background-task + # sweep: the shared service owns its own bounded flush semantics. On an + # exceptional exit the flush is skipped and the jobs are cancelled below. + if self.step_memory is not None and callable(getattr(self.step_memory, "flush", None)): + try: + if exc_type is None: + await self.step_memory.flush() + else: + await self.step_memory.flush(timeout_seconds=0.0) + except Exception: + pass + + # Drain in-flight chunk capsule jobs the same way (M3): the chunk + # manager persists any harvested capsules so the DB copy is complete + # even though the frozen transcript text is not re-rendered. + chunker = getattr(self.transcript_ledger, "chunker", None) + if chunker is not None and callable(getattr(chunker, "flush", None)): + try: + if exc_type is None: + await chunker.flush() + else: + await chunker.flush(timeout_seconds=0.0) + except Exception: + pass + if self.background_tasks: tasks = list(self.background_tasks) pending_tasks = [task for task in tasks if not task.done()] diff --git a/artemis/data_engine/engine.py b/artemis/data_engine/engine.py index 49fe54f..a94ce29 100644 --- a/artemis/data_engine/engine.py +++ b/artemis/data_engine/engine.py @@ -14,6 +14,7 @@ import asyncio from collections.abc import Callable +import functools import hashlib import json import os @@ -29,6 +30,7 @@ from artemis.context import ArtemisContext from artemis.data_engine.models import ( BackgroundTaskRecord, FailedOutputRecord, + HistoryChunkRecord, ImageRecord, SessionMetadata, StepRecord, @@ -48,6 +50,43 @@ logger = get_logger(__name__) _CURRENT_DATA_ENGINE = None +# Overlay packages that never identify the app the user is working in. +_FOREGROUND_APP_IGNORED = {"com.android.systemui"} + + +def _derive_foreground_app(ui_tree: Any | None) -> str | None: + """Best-effort foreground package from a perception UI tree (M5). + + Counts ``package``/``packageName`` attributes across the node list + (recursing into ``children``), ignores system-overlay packages, and + returns the most frequent value. Pure string work — never raises, never + touches the device. + """ + counts: dict[str, int] = {} + + def _visit(node: Any, depth: int = 0) -> None: + if depth > 50: + return + if isinstance(node, dict): + pkg = node.get("package") or node.get("packageName") + if isinstance(pkg, str) and pkg and pkg not in _FOREGROUND_APP_IGNORED: + counts[pkg] = counts.get(pkg, 0) + 1 + children = node.get("children") + if isinstance(children, (list, tuple)): + for child in children: + _visit(child, depth + 1) + elif isinstance(node, (list, tuple)): + for child in node: + _visit(child, depth + 1) + + try: + _visit(ui_tree) + except Exception: + return None + if not counts: + return None + return max(counts.items(), key=lambda kv: kv[1])[0] + class DataEngine: """Core engine for managing runtime data, storage, and streaming.""" @@ -538,6 +577,33 @@ class DataEngine: if pre_image_name and post_image_name and pre_image_name == post_image_name: post_image_name = None + # M5: persist the foreground app for the recall search surface (the + # parameter was historically accepted but never stored). The explicit + # parameter wins; otherwise it is derived best-effort from the UI + # tree's package attributes — a pure string scan, no device call. + extra_metadata = dict(extra_metadata or {}) + try: + app = foreground_app or _derive_foreground_app(ui_tree) + if app: + extra_metadata.setdefault("foreground_app", app) + except Exception: + pass + + # M4: stamp perceptual hashes for the local screen-similarity hint + # (pure PIL dHash; best-effort, computed synchronously to keep the + # record immutable once handed to the background writer). + try: + from artemis.utils.image_hash import dhash_hex + + pre_dhash = dhash_hex(pre_screenshot_bytes) + if pre_dhash: + extra_metadata.setdefault("pre_image_dhash", pre_dhash) + post_dhash = dhash_hex(post_screenshot_bytes) if post_image_name else pre_dhash + if post_dhash: + extra_metadata.setdefault("post_image_dhash", post_dhash) + except Exception: + pass + step = StepRecord( step_id=step_id, session_id=self.current_session_id, @@ -550,7 +616,7 @@ class DataEngine: operator_raw_thinking=operator_raw_thinking, operator_native_thinking=operator_native_thinking, last_execution_result=last_execution_result, - extra_metadata=extra_metadata or {}, + extra_metadata=extra_metadata, ) def _run_background_storage(): @@ -971,8 +1037,24 @@ class DataEngine: path = self.get_image_path(image_name) return path if path.exists() else None - def update_step_summary(self, step_id: UUID | int | str, summary: str): - """Update the step with a concise summary in background.""" + def update_step_summary( + self, + step_id: UUID | int | str, + summary: str | None, + *, + source: str | None = None, + version: int | None = None, + model: str | None = None, + status: str = "ready", + ): + """Update the step summary in background, with status/version metadata. + + Writes ``summary_status`` / ``summary_source`` / ``summary_version`` / + ``summary_model`` into the step's ``extra_metadata`` alongside the + summary text; concurrent same-step writes are ordered by version in the + storage layer. ``status`` may be ``pending`` / ``ready`` / ``failed`` / + ``stale``; only ``ready`` writes carry summary text and publish SSE. + """ target_uuid = None if isinstance(step_id, int): if self.current_session_id: @@ -993,7 +1075,19 @@ class DataEngine: logger.debug(f"Could not resolve step_id for {step_id} to update summary.") return - self._run_in_background(self.storage.update_step_summary, target_uuid, summary) + self._run_in_background( + functools.partial( + self.storage.update_step_summary, + target_uuid, + summary, + source=source, + version=version, + model=model, + status=status, + ) + ) + if status != "ready" or summary is None: + return step_num = self.get_step_number(target_uuid if not isinstance(step_id, int) else step_id) payload = { "step_id": str(target_uuid), @@ -1004,6 +1098,60 @@ class DataEngine: payload["step_number"] = step_num self._publish("step_updated", payload) + def record_history_chunk( + self, + *, + start_step_number: int, + end_step_number: int, + version: int, + status: str, + start_step_id: str | None = None, + end_step_id: str | None = None, + source_step_ids: list[str] | None = None, + subgoal_hash: str | None = None, + band1: dict[str, Any] | None = None, + band2: str | None = None, + band3: str | None = None, + rendered_text: str | None = None, + ) -> UUID | None: + """Persist one history-chunk version row in the background (append-only). + + Chunk identity is the step range; callers own version numbering (a + newer version for the same range supersedes older ones at read time). + """ + if not self.current_session_id: + return None + chunk = HistoryChunkRecord( + chunk_id=uuid4(), + session_id=self.current_session_id, + start_step_id=start_step_id, + end_step_id=end_step_id, + start_step_number=start_step_number, + end_step_number=end_step_number, + source_step_ids=source_step_ids or [], + subgoal_hash=subgoal_hash, + version=version, + status=status, + band1=band1 or {}, + band2=band2, + band3=band3, + rendered_text=rendered_text, + ) + self._run_in_background(self.storage.create_history_chunk, chunk) + return chunk.chunk_id + + def get_history_chunks(self, *, all_versions: bool = False) -> list[HistoryChunkRecord]: + """History chunks of the active session (newest version per step range).""" + if not self.current_session_id: + return [] + try: + return self.storage.get_history_chunks( + self.current_session_id, all_versions=all_versions + ) + except Exception as e: + logger.error(f"Failed to fetch history chunks: {e}") + return [] + def update_step_thinking(self, step_id: UUID, operator_raw_thinking: str): """Update step's thinking in SQLite in background.""" self._run_in_background(self.storage.update_step_thinking, step_id, operator_raw_thinking) diff --git a/artemis/data_engine/models.py b/artemis/data_engine/models.py index 8725319..bd937e5 100644 --- a/artemis/data_engine/models.py +++ b/artemis/data_engine/models.py @@ -55,6 +55,33 @@ class StepRecord(BaseModel): extra_metadata: dict[str, Any] = Field(default_factory=dict) +class HistoryChunkRecord(BaseModel): + """One version of a compressed history segment (history redesign §3.3/§6.1). + + Identity is the step *range* ``(session_id, start_step_number, + end_step_number)``; the milestone ``subgoal_hash`` is label metadata only. + Rows are append-only: a later write for the same range carries a higher + ``version`` (capsule readiness, post-hoc checkpoint annotations). + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + chunk_id: UUID | str + session_id: UUID | str + start_step_id: str | None = None + end_step_id: str | None = None + start_step_number: int + end_step_number: int + source_step_ids: list[str] = Field(default_factory=list) + subgoal_hash: str | None = None + version: int = 1 + status: str = "pending" # pending (band ③ only), ready (bands ①②③) + band1: dict[str, Any] = Field(default_factory=dict) # synopsis + structured fields + band2: str | None = None # interval narrative + band3: str | None = None # mechanical per-step action ledger + rendered_text: str | None = None + created_at: float = Field(default_factory=time.time) + + class TraceRecord(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) trace_id: UUID | str diff --git a/artemis/data_engine/storage.py b/artemis/data_engine/storage.py index ccdfdc3..eaf0b61 100644 --- a/artemis/data_engine/storage.py +++ b/artemis/data_engine/storage.py @@ -27,6 +27,7 @@ from uuid import UUID from artemis.data_engine.models import ( BackgroundTaskRecord, FailedOutputRecord, + HistoryChunkRecord, ImageRecord, SessionMetadata, StepRecord, @@ -205,6 +206,26 @@ class StorageManager: FOREIGN KEY(session_id) REFERENCES sessions(session_id) ) """) + conn.execute(""" + CREATE TABLE IF NOT EXISTS history_chunks ( + chunk_id TEXT PRIMARY KEY, + session_id TEXT, + start_step_id TEXT, + end_step_id TEXT, + start_step_number INTEGER, + end_step_number INTEGER, + source_step_ids TEXT, + subgoal_hash TEXT, + version INTEGER, + status TEXT, + band1 TEXT, + band2 TEXT, + band3 TEXT, + rendered_text TEXT, + created_at REAL, + FOREIGN KEY(session_id) REFERENCES sessions(session_id) + ) + """) conn.execute(""" CREATE TABLE IF NOT EXISTS video_recordings ( video_id TEXT PRIMARY KEY, @@ -534,22 +555,179 @@ class StorageManager: ) conn.commit() - def update_step_summary(self, step_id: UUID, summary: str): - """Update the summary for a step.""" + def update_step_summary( + self, + step_id: UUID, + summary: str | None, + *, + source: str | None = None, + version: int | None = None, + model: str | None = None, + status: str = "ready", + ) -> bool: + """Versioned, status-carrying summary write (replaces the blind overwrite). + + Writes ``summary_status`` / ``summary_source`` / ``summary_version`` / + ``summary_model`` into the step's ``extra_metadata``. Concurrent writes + to the same step are ordered by version: an explicit ``version`` lower + than the stored one is dropped; ``version=None`` auto-increments the + stored version. Status downgrades are refused without a newer explicit + version: ``pending`` never overwrites ``ready``/``failed``, and + ``failed`` never overwrites ``ready``. The ``summary`` column itself is + only touched by a ``ready`` write with a non-None summary. + + Returns True when the write was applied, False when it was dropped as + stale (or the step does not exist). + """ + with self._get_connection() as conn: + cursor = conn.execute( + "SELECT extra_metadata FROM steps WHERE step_id = ?", + (str(step_id),), + ) + row = cursor.fetchone() + if row is None: + return False + + try: + meta = json.loads(row["extra_metadata"]) if row["extra_metadata"] else {} + if not isinstance(meta, dict): + meta = {} + except Exception: + meta = {} + + current_version = meta.get("summary_version") + current_version = current_version if isinstance(current_version, int) else 0 + current_status = meta.get("summary_status") + + if version is not None: + if version < current_version: + return False + new_version = version + is_newer = version > current_version + else: + new_version = current_version + 1 + is_newer = False + + # Status downgrades require an explicitly newer version. + if not is_newer: + if status == "pending" and current_status in ("ready", "failed"): + return False + if status == "failed" and current_status == "ready": + return False + + meta["summary_status"] = status + meta["summary_version"] = new_version + if source is not None: + meta["summary_source"] = source + if model is not None: + meta["summary_model"] = model + + if status == "ready" and summary is not None: + conn.execute( + "UPDATE steps SET summary = ?, extra_metadata = ? WHERE step_id = ?", + (summary, json.dumps(meta), str(step_id)), + ) + else: + conn.execute( + "UPDATE steps SET extra_metadata = ? WHERE step_id = ?", + (json.dumps(meta), str(step_id)), + ) + conn.commit() + return True + + def create_history_chunk(self, chunk: HistoryChunkRecord): + """Insert one history-chunk version row (append-only; never updates).""" with self._get_connection() as conn: conn.execute( """ - UPDATE steps - SET summary = ? - WHERE step_id = ? + INSERT INTO history_chunks ( + chunk_id, session_id, start_step_id, end_step_id, + start_step_number, end_step_number, source_step_ids, + subgoal_hash, version, status, band1, band2, band3, + rendered_text, created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( - summary, - str(step_id), + str(chunk.chunk_id), + str(chunk.session_id), + chunk.start_step_id, + chunk.end_step_id, + chunk.start_step_number, + chunk.end_step_number, + json.dumps(chunk.source_step_ids), + chunk.subgoal_hash, + chunk.version, + chunk.status, + json.dumps(chunk.band1, ensure_ascii=False, cls=SafeJSONEncoder), + chunk.band2, + chunk.band3, + chunk.rendered_text, + chunk.created_at, ), ) conn.commit() + def get_history_chunks( + self, session_id: UUID | str, *, all_versions: bool = False + ) -> list[HistoryChunkRecord]: + """History chunks of a session ordered by step range. + + By default only the newest version per step range is returned; pass + ``all_versions=True`` for the full append-only trail (audit/tests). + """ + with self._get_connection() as conn: + cursor = conn.execute( + """ + SELECT * FROM history_chunks WHERE session_id = ? + ORDER BY start_step_number ASC, version ASC, created_at ASC + """, + (str(session_id),), + ) + rows = cursor.fetchall() + + records: list[HistoryChunkRecord] = [] + for row in rows: + try: + source_ids = json.loads(row["source_step_ids"]) if row["source_step_ids"] else [] + except Exception: + source_ids = [] + try: + band1 = json.loads(row["band1"]) if row["band1"] else {} + if not isinstance(band1, dict): + band1 = {} + except Exception: + band1 = {} + records.append( + HistoryChunkRecord( + chunk_id=row["chunk_id"], + session_id=row["session_id"], + start_step_id=row["start_step_id"], + end_step_id=row["end_step_id"], + start_step_number=row["start_step_number"], + end_step_number=row["end_step_number"], + source_step_ids=source_ids, + subgoal_hash=row["subgoal_hash"], + version=row["version"], + status=row["status"], + band1=band1, + band2=row["band2"], + band3=row["band3"], + rendered_text=row["rendered_text"], + created_at=row["created_at"], + ) + ) + if all_versions: + return records + + latest: dict[tuple[int, int], HistoryChunkRecord] = {} + for rec in records: + key = (rec.start_step_number, rec.end_step_number) + current = latest.get(key) + if current is None or rec.version >= current.version: + latest[key] = rec + return sorted(latest.values(), key=lambda r: r.start_step_number) + def update_step_thinking(self, step_id: UUID, operator_raw_thinking: str): """Update the raw thinking process for a step.""" with self._get_connection() as conn: diff --git a/artemis/graph/checkpoints.py b/artemis/graph/checkpoints.py index ecdc46f..044584e 100644 --- a/artemis/graph/checkpoints.py +++ b/artemis/graph/checkpoints.py @@ -302,6 +302,28 @@ def _record_verdicts( ) +def _annotate_history_chunks(ctx, run: CheckpointRun, verdicts) -> None: + """Post-hoc chunk annotation (M3 §9): a verdict harvested after its subgoal's + segment was chunk-compressed is appended to that chunk and bumps its + version. Best-effort — history annotation never gates checkpoint booking.""" + try: + chunker = getattr(getattr(ctx, "transcript_ledger", None), "chunker", None) + if chunker is None: + return + payload = [ + { + "kind": getattr(v, "kind", "check"), + "item_text": getattr(v, "item_text", ""), + "status": getattr(v, "status", ""), + "evidence": getattr(v, "evidence", ""), + } + for v in verdicts + ] + chunker.annotate_from_checkpoint(run.checkpoint.checkpoint_id, payload) + except Exception as e: + logger.debug(f"History chunk annotation skipped: {e}") + + def _record_attempt_failure(ctx, run: CheckpointRun, status: str, evidence: str) -> None: base_dir = ctx.data_engine.base_dir for ci in run.checkpoint.check_items: @@ -367,6 +389,7 @@ def harvest_run( report = task.result() verdicts = list(getattr(report, "verdicts", []) or []) _record_verdicts(ctx, run, verdicts, anchor_step_id) + _annotate_history_chunks(ctx, run, verdicts) # Applicability gate for side effects: the verdict must belong to the # current attempt of its checkpoint AND the anchored subgoal's text must be diff --git a/artemis/graph/graph.py b/artemis/graph/graph.py index c44a80f..3da68c8 100644 --- a/artemis/graph/graph.py +++ b/artemis/graph/graph.py @@ -96,6 +96,20 @@ def convergence_node(state: State): return {} +def _notify_history_chunker(ctx: ArtemisContext, method: str, *args) -> None: + """Best-effort event feed to the transcript's L2/L3 chunk manager (M3). + + The chunker only exists when the transcript flag is on; every failure is + swallowed — history compression must never gate the execution path. + """ + try: + chunker = getattr(getattr(ctx, "transcript_ledger", None), "chunker", None) + if chunker is not None: + getattr(chunker, method)(*args) + except Exception as e: + logger.debug(f"History chunker notification '{method}' skipped: {e}") + + async def execution_check_node(state: State, ctx: ArtemisContext): state = strict_state(state, "execution_check") logger.info("Starting execution_check_node") @@ -126,6 +140,10 @@ async def execution_check_node(state: State, ctx: ArtemisContext): logger.warning(f"Failed to parse structured decisions: {e}") subgoal_hash_val, sub_subgoal_hash = _get_active_subgoal_hashes(ctx) + # The user's injected instruction is stamped verbatim on the step + # it reached: the chunk ledger (band ③) preserves it as a + # never-evicted line at every compression level (§3.3). + injected = getattr(state, "injected_instruction", None) step_id = ctx.data_engine.record_step( pre_screenshot_bytes=screenshot_bytes, ui_tree=xml_hierarchy, @@ -138,10 +156,14 @@ async def execution_check_node(state: State, ctx: ArtemisContext): "sub_subgoal_hash": sub_subgoal_hash, "width": raw_data.get("width"), "height": raw_data.get("height"), + **({"injected_instruction": injected} if injected else {}), **extra_metadata, }, ) logger.info(f"Recorded step in DataEngine: {step_id}") + # Milestone-switch fact source: the stamped subgoal hash of every + # executed step, compared consecutively inside the chunk manager. + _notify_history_chunker(ctx, "on_step_stamped", str(step_id), subgoal_hash_val) return str(step_id) except Exception as e: logger.error(f"Failed to record step in DataEngine: {e}") @@ -611,6 +633,10 @@ async def _process_plan_write( new_completions = new_top_level_completions(before, after) if new_completions: queue_checkpoints(ctx, state, after, new_completions, content_after) + # History chunking (M3): a top-level completion only queues an + # UNCONFIRMED boundary — the next stamped step confirms it via an + # actual subgoal-hash change (pseudo-switch protection, §3.3). + _notify_history_chunker(ctx, "queue_boundary_hint") return result @@ -823,11 +849,14 @@ async def get_graph(ctx: ArtemisContext) -> CompiledStateGraph: operator_native_tools.append(t) # Get specialized tools for Operator + from artemis.tools.history_recall import recall_history_wrapper + operator_specialized_wrappers = [ ask_diagnoser_wrapper, run_adb_command_wrapper, manage_task_wrapper, ask_explorer_wrapper, + recall_history_wrapper, ] if ctx.execution_setup and ctx.execution_setup.enable_committee: operator_specialized_wrappers.append(ask_committee_wrapper) diff --git a/artemis/graph/state.py b/artemis/graph/state.py index 24445b6..38a2e18 100644 --- a/artemis/graph/state.py +++ b/artemis/graph/state.py @@ -121,11 +121,6 @@ class State(BaseModel): "Native/implicit thinking process of the operator", take_last, ] = None - short_term_memory: Annotated[ - str | None, - "Short term memory / scratchpad for the operator", - take_last, - ] = None last_execution_result: Annotated[ dict | None, "Last execution result from validator", take_last ] = None diff --git a/artemis/graph/visibility.py b/artemis/graph/visibility.py index 69b2edb..290cd59 100644 --- a/artemis/graph/visibility.py +++ b/artemis/graph/visibility.py @@ -83,19 +83,20 @@ NODE_VISIBILITY: dict[str, NodeVisibility] = { "latest_screenshot", "indexed_points", "indexed_elements", - "short_term_memory", "subagent_calls", "current_step_id", "operator_tool_limit_exceeded", "structured_decisions", "operator_raw_thinking", "operator_native_thinking", + # Transcript path (M2): committing the previous turn into the + # ledger appends its validator result message. + "last_execution_result", }, writes={ "structured_decisions", "operator_raw_thinking", "operator_native_thinking", - "short_term_memory", "indexed_points", "indexed_elements", "current_step_id", @@ -111,6 +112,9 @@ NODE_VISIBILITY: dict[str, NodeVisibility] = { "operator_native_thinking", "user_stop_requested", "initial_goal", + # M3: the turn's injected instruction is stamped verbatim onto the + # step record so the chunk ledger can preserve it (never-evict). + "injected_instruction", }, writes={ "checker_success", diff --git a/artemis/mcp/action_executor.py b/artemis/mcp/action_executor.py index 93c55cf..54c1ec8 100644 --- a/artemis/mcp/action_executor.py +++ b/artemis/mcp/action_executor.py @@ -54,6 +54,7 @@ from artemis.utils.coordinates import ( compute_smart_swipe_coordinates, parse_swipe_parameters, ) +from artemis.utils.element_hit_test import hit_test_semantics from artemis.utils.logger import get_logger from artemis.utils.notes import ( format_list_notes_failure, @@ -139,11 +140,19 @@ class McpActionExecutor: img_bytes: bytes | None = None shot_path: str | None = None xml_list: str | None = None + target_semantics: dict[str, Any] | None = None with span: try: session = await self._session_or_start() wire_name, wire_args, finalize = self._translate(raw_name, args, state) + # Record-time enrichment: bare-coordinate targeted actions get + # best-effort element semantics from the pre-action frame (the + # post-action observe below overwrites state.indexed_elements, + # so this must happen before the device call). + target_semantics = self._target_semantics(raw_name, wire_args, state) + if target_semantics: + span.payload = {"args": {**args, **target_semantics}} extension = raw_name not in (REQUIRED_ACTIONS | OPTIONAL_ACTIONS) if extension: @@ -207,9 +216,69 @@ class McpActionExecutor: metadata={ "code": res.code.value, "normalized_coordinates": res.normalized_coordinates, + "target_semantics": target_semantics, }, ) + def _target_semantics( + self, raw_name: str, wire_args: dict[str, Any], state: Any + ) -> dict[str, Any] | None: + """Best-effort element semantics for a coordinate-targeted action. + + Hit tests the normalized (0-1000) target point(s) against the + pre-action frame's indexed elements. Single-target actions get the + flat ``target_*`` fields; multi-point actions (``click_sequence``, + ``swipe``) hit test every point best-effort (M5), hoisting the FIRST + point's semantics to the main ``target_*`` label and keeping the + per-point results alongside. Returns None for actions without a + resolvable coordinate target; degrades to + ``target_label_source: "none"`` when no element data covers the point. + """ + try: + elements = getattr(state, "indexed_elements", None) if state else None + + if raw_name in ("click", "long_press", "input_text"): + target = wire_args.get("target") + if not (isinstance(target, (list, tuple)) and len(target) == 2): + return None + return self._hit_test_point(elements, target) + + if raw_name == "click_sequence": + sequence = wire_args.get("sequence") + if not (isinstance(sequence, (list, tuple)) and sequence): + return None + points = [self._hit_test_point(elements, p) for p in sequence] + semantics: dict[str, Any] = dict(points[0]) + semantics["points_semantics"] = points + return semantics + + if raw_name == "swipe": + start = wire_args.get("start") + end = wire_args.get("end") + if not (isinstance(start, (list, tuple)) and len(start) == 2): + return None + semantics = dict(self._hit_test_point(elements, start)) + semantics["start_semantics"] = self._hit_test_point(elements, start) + if isinstance(end, (list, tuple)) and len(end) == 2: + semantics["end_semantics"] = self._hit_test_point(elements, end) + return semantics + + return None + except Exception as enrich_err: + logger.debug(f"Target semantics enrichment skipped for {raw_name}: {enrich_err}") + return {"target_label_source": "none"} + + def _hit_test_point(self, elements: Any, target: Any) -> dict[str, Any]: + """Hit tests one normalized (0-1000) point; never raises.""" + try: + width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080 + height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400 + x_px = int(max(0, min(width - 1, float(target[0]) * width / 1000))) + y_px = int(max(0, min(height - 1, float(target[1]) * height / 1000))) + return hit_test_semantics(elements, x_px, y_px) + except Exception: + return {"target_label_source": "none"} + @staticmethod def _observe_despite_failure(raw_name: str, res: ActionResult) -> bool: """Failures that still observe the screen, mirroring the legacy executor. diff --git a/artemis/mcp/action_manifest.py b/artemis/mcp/action_manifest.py index 99fc681..73885ff 100644 --- a/artemis/mcp/action_manifest.py +++ b/artemis/mcp/action_manifest.py @@ -122,6 +122,7 @@ BACKEND_INDEPENDENT_TOOLS: frozenset[str] = frozenset( "run_adb_command", "manage_task", "analyze_task_output", + "recall_history", "report_task_status", "report_failure_analysis", } diff --git a/artemis/memory/__init__.py b/artemis/memory/__init__.py new file mode 100644 index 0000000..1277bd3 --- /dev/null +++ b/artemis/memory/__init__.py @@ -0,0 +1,84 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared step-memory runtime for background summarization across profiles.""" + +from artemis.memory.chunking import ( + ChunkCapsuleService, + HistoryChunkManager, + StepCapsuleLens, + build_action_ledger, +) +from artemis.memory.context_policy import ( + CONTEXT_POLICIES, + ContextPolicy, + build_history_for, + resolve_policy, +) +from artemis.memory.step_memory import StepLens, StepMemoryService +from artemis.memory.transcript import TranscriptLedger, format_session_offset + + +def ensure_step_memory(ctx): + """Return the session's shared step-memory service, creating it on demand. + + Composition root for ``ctx.step_memory`` (history redesign §6.2): the + first caller — FlashRunner, the Pro SummarizerNode, or the Pro operator's + transcript path — instantiates the visual-transition lens service and + publishes it on the context so every profile shares one runtime. + """ + service = getattr(ctx, "step_memory", None) + if service is not None: + return service + + from artemis.agents.flash.summarizer import VisualStepSummarizer + + kwargs: dict = {} + model_name = None + try: + from artemis.config import load_agent_config + + cfg = load_agent_config() + model_name = cfg.flash.step_summarizer.model + kwargs = { + "retry_limit": cfg.memory.runtime.retry_limit, + "max_concurrency": cfg.memory.runtime.max_concurrency, + "flush_timeout_s": cfg.memory.runtime.flush_timeout_s, + } + except Exception: + pass + + service = VisualStepSummarizer(ctx, model_name=model_name, **kwargs) + try: + ctx.step_memory = service + except Exception: + pass + return service + + +__all__ = [ + "CONTEXT_POLICIES", + "ChunkCapsuleService", + "ContextPolicy", + "HistoryChunkManager", + "StepCapsuleLens", + "build_history_for", + "resolve_policy", + "StepLens", + "StepMemoryService", + "TranscriptLedger", + "build_action_ledger", + "ensure_step_memory", + "format_session_offset", +] diff --git a/artemis/memory/chunking.py b/artemis/memory/chunking.py new file mode 100644 index 0000000..08c4098 --- /dev/null +++ b/artemis/memory/chunking.py @@ -0,0 +1,1405 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""L2/L3 history compression: segment chunks, eras, and the emergency snapshot. + +Implements §3.3 of the history-module redesign (M3). A **HistoryChunk** is a +*segmented ledger*, never a fused capsule: compression shrinks each step's +width, never the step count. Every chunk renders as three bands: + +- **① Synopsis & effects** (LLM, :class:`StepCapsuleLens`): three-question + prose (doing / did / effect — the effect must name the notes left behind) + plus structured fields (verified_facts / unresolved / failed_paths / + important_entities / entry_state / exit_state). +- **② Compressed step summary** (same LLM call): interval narrative whose + union must cover the segment's step range seamlessly (machine-checked; a + gap forces regeneration, bounded retries degrade to pending). +- **③ Per-step action ledger** (mechanical, zero-distortion, + :func:`build_action_ledger`): one line per step — step number, ``T+mm:ss`` + session offset, ``format_action_clean`` semantic action, controller/ + validator result phrase; Failure-Analyzer recovery actions as indented + sub-lines; user-injected instructions verbatim on their own never-evicted + lines. + +Triggers (:class:`HistoryChunkManager`): milestone switch (sole fact source: +the *stamped* ``subgoal_hash`` changing between consecutive recorded steps; +plan-write completions only queue an unconfirmed boundary hint), segment size +(steps / estimated source tokens), and the measured-token soft threshold. +The hard threshold collapses the frozen region to the L3 session snapshot +(chunk headers set-merged; a minimal per-step index survives). Chunk-count +overflow folds the oldest chunks into eras (① set-merged, ② degraded to +segment title lines, ③ kept); era overflow compresses the oldest eras into +the extreme-layer **period paragraph** (§10 decision 5, final user review +2026-09-01): one header line carrying the step range and start/end session +offsets plus a mechanically assembled one-paragraph synopsis (from the member +chunks' band-① fields, no extra LLM call), a ``recall_history`` guidance +line, and the verbatim never-evict user-injection lines. Step-level +addressability degrades to *period* addressability at this layer only; the +full per-step ledger stays recallable from the DataEngine. + +Async discipline — **ready-gated swap** (user decision 2026-09-01, revising +the original M3 freeze-then-harvest form): a boundary/size/soft trigger only +*closes* a segment and dispatches its capsule; the original messages stay in +the transcript verbatim (lossless-pending, carried from the image layer to the +segment layer). The swap — freezing the segment's turns and rendering the +chunk block — happens at the first render at or after the capsule is ready. +Rationale: a header-less chunk has no ``verified_facts``, and a model that +sees "what happened" without "what was established" re-doubts verified facts +and loops. Degradation ladder: capsule generation failure (bounded retries +exhausted) keeps the original text and re-dispatches (the lens itself falls +back to a secondary model per attempt); soft-threshold pressure alone still +keeps the original text (correctness over tokens); only the hard threshold +force-swaps everything closed — pending chunks included — into the L3 +snapshot. Checkpoint annotations are independent of the header (they bump the +DB version immediately) and render in pending blocks too; frozen text still +only re-renders at swap events, so it stays byte-stable in between. +""" + +import asyncio +from dataclasses import dataclass, field +import json +from pathlib import Path +import re +from typing import Any, Callable + +from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage + +from artemis.memory.step_memory import JobKey, StepLens, StepMemoryService +from artemis.memory.transcript import format_session_offset +from artemis.utils.logger import get_logger + +logger = get_logger(__name__) + +#: Recall guidance line rendered under an extreme-layer period paragraph +#: (an era whose per-step ledger overflowed to recall-only; §3.3). +RECALL_GUIDANCE_TEMPLATE = ( + " (Step-level ledger via recall_history for steps {start}–{end})" +) + +CHUNK_PENDING_NOTE = ( + "①/② capsule pending (background generation); the mechanical" + " ledger below is complete. Original step records remain recallable" + " via recall_history." +) + +_NOTE_TOOLS = ("save_note", "update_note", "append_note") + + +# --------------------------------------------------------------------------- +# Band ③ — mechanical per-step action ledger +# --------------------------------------------------------------------------- + + +def _result_phrase(result: Any) -> str: + """Controller/validator result phrase for one ledger line (mechanical).""" + if not isinstance(result, dict) or not result: + return "no terminal action" + from artemis.utils.task_tree import format_result_clean + + detail = None + try: + detail = format_result_clean(result) + except Exception: + detail = None + if detail: + return detail + status = result.get("status") + if status == "success": + return "executed" + return str(status) if status else "executed" + + +def _action_phrase(step: dict) -> str: + """Semantic action phrase for one step (``format_action_clean``).""" + from artemis.utils.task_tree import format_action_clean + + action = step.get("action_taken") + text = format_action_clean(action) + if isinstance(action, list) and len(action) > 1: + text += f" (+{len(action) - 1} more actions)" + return text + + +def _fa_recovery_lines(step: dict) -> list[str]: + """Failure-Analyzer device actions of a step, rendered mechanically.""" + from artemis.utils.task_tree import format_tool_call_clean + + lines: list[str] = [] + for event in step.get("interleaved_events") or []: + name = event.get("name") or "" + if ( + event.get("type") == "tool_call" + and name.startswith("_exec_") + and name != "report_failure_analysis" + ): + try: + rendered = format_tool_call_clean( + name, event.get("args") or {}, event.get("result") + ) + except Exception: + rendered = None + if rendered: + lines.append(rendered) + return lines + + +def step_offset_label(step: dict, session_start: float | None) -> str: + """``T+mm:ss`` session-start offset of a step (byte-stable once frozen).""" + ts = step.get("timestamp") + if session_start is None or not isinstance(ts, (int, float)): + return "T+??:??" + return format_session_offset(float(ts) - float(session_start)) + + +def injected_instruction_line(step: dict) -> str | None: + """The never-evict verbatim user-injection line for a step, if any.""" + instr = (step.get("extra_metadata") or {}).get("injected_instruction") + if not instr: + return None + return f' User @ Step {step.get("step_number")}: "{instr}"' + + +def build_action_ledger( + steps: list[dict], + session_start: float | None, + *, + minimal: bool = False, +) -> str: + """Assemble band ③ mechanically (1:1, never elided, no LLM involved). + + ``minimal=True`` renders the L3 minimum-width index (step number + offset + + action phrase). User-injected instruction lines are preserved verbatim + at every width — they are never evicted at any compression level. + """ + lines: list[str] = [] + for step in steps: + number = step.get("step_number") + offset = step_offset_label(step, session_start) + action = _action_phrase(step) + if minimal: + lines.append(f"- Step {number} ({offset}): {action}") + else: + lines.append(f"- Step {number} ({offset}): {action} -> {_result_phrase(step.get('last_execution_result'))}") + for recovery in _fa_recovery_lines(step): + lines.append(f" FA: {recovery}") + user_line = injected_instruction_line(step) + if user_line: + lines.append(user_line) + return "\n".join(lines) + + +def extract_note_writes(step: dict) -> list[dict[str, str]]: + """Notes written during a step (band ① input; task_plan writes excluded).""" + writes: list[dict[str, str]] = [] + for event in step.get("interleaved_events") or []: + if event.get("type") != "tool_call" or event.get("name") not in _NOTE_TOOLS: + continue + args = event.get("args") or {} + key = args.get("key") + if not key or key == "task_plan": + continue + gist = args.get("content") or args.get("replacement") or "" + writes.append({"tool": event["name"], "key": str(key), "gist": str(gist)[:300]}) + return writes + + +# --------------------------------------------------------------------------- +# Band ①+② — StepCapsuleLens (single LLM call) + machine-checked coverage +# --------------------------------------------------------------------------- + + +def validate_interval_coverage(intervals: Any, start_step: int, end_step: int) -> bool: + """Band ② hard constraint: the interval union covers [start, end] seamlessly.""" + if not isinstance(intervals, list) or not intervals: + return False + expected = start_step + for interval in intervals: + if not isinstance(interval, dict): + return False + try: + s = int(interval.get("start_step")) + e = int(interval.get("end_step")) + except (TypeError, ValueError): + return False + if s != expected or e < s or not str(interval.get("text") or "").strip(): + return False + expected = e + 1 + return expected == end_step + 1 + + +_CAPSULE_REQUIRED_KEYS = ("doing", "did", "effect", "intervals") +_CAPSULE_LIST_KEYS = ("verified_facts", "unresolved", "failed_paths", "important_entities") + + +def collect_note_keys(payload: dict[str, Any]) -> list[str]: + """Deduplicated note keys written during the segment (payload order).""" + keys: list[str] = [] + seen: set[str] = set() + for step in payload.get("steps") or []: + for write in step.get("note_writes") or []: + key = str(write.get("key") or "").strip() + if key and key not in seen: + seen.add(key) + keys.append(key) + return keys + + +def _strip_code_fences(text: str) -> str: + text = text.strip() + match = re.search(r"```(?:json)?\s*(.*?)\s*```", text, flags=re.DOTALL) + return match.group(1) if match else text + + +class StepCapsuleLens(StepLens): + """Chunk-level lens producing bands ①+② in one call (§5). + + The payload carries the segment's per-step visual summaries, exact + actions, validator results, reasoning excerpts, notes writes, and + injected instructions. The rendered summary string is the JSON capsule; + an attempt whose interval union fails the mechanical coverage check + returns ``None`` so the bounded service retry regenerates it, and on + exhaustion the chunk simply stays pending (③ is independently usable). + """ + + name = "step_capsule" + + _PROMPT_PATH = Path(__file__).parent / "step_capsule.md" + + def __init__( + self, + model_name: str | None = None, + llm: Any | None = None, + *, + ctx: Any = None, + fallback_model_name: str | None = None, + fallback_llm: Any | None = None, + ): + self._model_name = model_name or "gemini-3.7-flash" + self._llm = llm + self._ctx = ctx + # Availability hardening: `chunking.model` is a dedicated model with no + # gateway behind it — when that one endpoint is down (e.g. a day-long + # 503), every capsule dies and chunk headers stay pending forever. A + # configured fallback model turns a provider outage into a degraded + # attempt instead of a dead loop. + self._fallback_model_name = ( + fallback_model_name if fallback_model_name != self._model_name else None + ) + self._fallback_llm = fallback_llm + try: + self._prompt = self._PROMPT_PATH.read_text(encoding="utf-8") + except Exception: + self._prompt = ( + "You compress one segment of executed steps into a JSON capsule with" + " keys doing/did/effect/entry_state/exit_state/verified_facts/" + "unresolved/failed_paths/important_entities/intervals. Never use" + " verdict words (successfully, completed, failed, ...); intervals" + " must cover the step range seamlessly. Return only JSON." + ) + + def _get_llm(self): + if self._llm is None: + from artemis.services.llm import get_google_llm + + self._llm = get_google_llm(model_name=self._model_name, temperature=0.0) + return self._llm + + def _get_fallback_llm(self): + if self._fallback_llm is None and self._fallback_model_name: + from artemis.services.llm import get_google_llm + + self._fallback_llm = get_google_llm( + model_name=self._fallback_model_name, temperature=0.0 + ) + return self._fallback_llm + + @property + def has_fallback(self) -> bool: + return self._fallback_llm is not None or bool(self._fallback_model_name) + + @property + def model_name(self) -> str: + return self._model_name + + def build_messages(self, payload: dict[str, Any]) -> list[BaseMessage]: + start = payload.get("start_step") + end = payload.get("end_step") + label = payload.get("milestone_label") + header = [f"SEGMENT: Steps {start}–{end}"] + if label: + header.append(f"MILESTONE: {label}") + + blocks: list[str] = ["\n".join(header)] + for step in payload.get("steps", []): + part = [ + f"## Step {step.get('step_number')} ({step.get('offset')})", + f"Action: {step.get('action')}", + f"Recorded result: {step.get('outcome')}", + ] + if step.get("visual_summary"): + part.append(f"Visual transition: {step['visual_summary']}") + if step.get("thinking_excerpt"): + part.append(f"Reasoning excerpt: {step['thinking_excerpt']}") + for write in step.get("note_writes", []): + part.append( + f"Note written ({write['tool']} -> {write['key']}): {write['gist']}" + ) + if step.get("injected_instruction"): + part.append( + f'User injected instruction: "{step["injected_instruction"]}"' + ) + for fa in step.get("fa_actions", []): + part.append(f"Failure-Analyzer recovery: {fa}") + blocks.append("\n".join(part)) + + return [ + SystemMessage(content=self._prompt), + HumanMessage(content="\n\n".join(blocks)), + ] + + def parse_capsule(self, text: str, payload: dict[str, Any]) -> dict[str, Any] | None: + """Parse + machine-check one capsule; None on any violation.""" + try: + parsed = json.loads(_strip_code_fences(text)) + except Exception: + return None + if not isinstance(parsed, dict): + return None + for key in _CAPSULE_REQUIRED_KEYS: + if key not in parsed: + return None + for key in _CAPSULE_LIST_KEYS: + value = parsed.get(key) + parsed[key] = [str(v) for v in value] if isinstance(value, list) else [] + parsed.setdefault("entry_state", "") + parsed.setdefault("exit_state", "") + if not validate_interval_coverage( + parsed.get("intervals"), int(payload["start_step"]), int(payload["end_step"]) + ): + logger.warning( + "StepCapsuleLens: interval union does not cover Steps" + f" {payload['start_step']}–{payload['end_step']}; regenerating." + ) + return None + # Notes linkage is a machine check, same rank as ② coverage (user + # decision 2026-09-01): every note key written during the segment must + # surface in the band-① text, or the capsule regenerates. + note_keys = collect_note_keys(payload) + if note_keys: + searchable = " ".join( + [ + str(parsed.get(field) or "") + for field in ("doing", "did", "effect", "entry_state", "exit_state") + ] + + [v for key in _CAPSULE_LIST_KEYS for v in parsed.get(key) or []] + ) + missing = [key for key in note_keys if key not in searchable] + if missing: + logger.warning( + f"StepCapsuleLens: capsule omits note key(s) {missing}" + " written during the segment; regenerating." + ) + return None + return parsed + + async def _invoke(self, llm: Any, messages: list[BaseMessage], model_label: str): + response = await asyncio.wait_for(llm.ainvoke(messages), timeout=90.0) + # Raw-model bypass metering (gateway-wrapped models meter themselves); + # capsule prompts are small and must not clobber the session's + # last_prompt_tokens (the compaction thresholds' live context base). + try: + from artemis.services.llm import RobustChatModelWrapper + from artemis.services.token_meter import record_llm_usage + + if not isinstance(llm, RobustChatModelWrapper): + engine = getattr(self._ctx, "data_engine", None) if self._ctx else None + record_llm_usage( + engine, + response, + source=f"lens:step_capsule:{model_label}", + update_last_prompt=False, + ) + except Exception: + pass + return response + + async def render(self, key: JobKey, payload: dict[str, Any]) -> str | None: + messages = self.build_messages(payload) + if self.has_fallback: + from artemis.services.llm import with_fallback + + response = await with_fallback( + lambda: self._invoke(self._get_llm(), messages, self._model_name), + lambda: self._invoke( + self._get_fallback_llm(), + messages, + self._fallback_model_name or "fallback", + ), + none_should_fallback=False, + ) + else: + response = await self._invoke(self._get_llm(), messages, self._model_name) + content = getattr(response, "content", "") + if isinstance(content, list): + content = "".join( + b.get("text", "") for b in content if isinstance(b, dict) and "text" in b + ) + parsed = self.parse_capsule(str(content), payload) + if parsed is None: + return None + return json.dumps(parsed, ensure_ascii=False) + + +class ChunkCapsuleService(StepMemoryService): + """Background runtime for chunk capsules (bands ①②) on the shared skeleton.""" + + def __init__(self, ctx: Any, lens: StepCapsuleLens, **kwargs): + super().__init__(ctx, lens=lens, **kwargs) + + @property + def lens(self) -> StepCapsuleLens: + return self._lens # type: ignore[return-value] + + +# --------------------------------------------------------------------------- +# In-memory chunk / era mirrors +# --------------------------------------------------------------------------- + + +@dataclass +class ChunkState: + """In-memory mirror of one chunk (authoritative for rendering & versions).""" + + ordinal: int + start_step_number: int + end_step_number: int + start_step_id: str | None + end_step_id: str | None + source_step_ids: list[str] + subgoal_hash: str | None + milestone_label: str | None + start_offset: str + end_offset: str + band3: str + minimal_index: str + user_lines: list[str] + version: int = 1 + status: str = "pending" + band1: dict[str, Any] = field(default_factory=dict) + band2: str | None = None + annotations: list[dict[str, Any]] = field(default_factory=list) + + @property + def capsule_key(self) -> str: + return f"chunk:{self.start_step_number}-{self.end_step_number}" + + @property + def step_range_label(self) -> str: + return f"Steps {self.start_step_number}–{self.end_step_number}" + + +@dataclass +class EraState: + """A group of merged chunks: ① set-merged, ② title lines, ③ retained.""" + + ordinal: int + chunks: list[ChunkState] + recall_only: bool = False + + @property + def start_step_number(self) -> int: + return self.chunks[0].start_step_number + + @property + def end_step_number(self) -> int: + return self.chunks[-1].end_step_number + + +def merge_structured_fields(band1_list: list[dict[str, Any]]) -> dict[str, list[str]]: + """Order-preserving set-merge of the chunks' structured fields (no re-summarizing).""" + merged: dict[str, list[str]] = {key: [] for key in _CAPSULE_LIST_KEYS} + for band1 in band1_list: + for key in _CAPSULE_LIST_KEYS: + for value in band1.get(key) or []: + if value not in merged[key]: + merged[key].append(value) + return merged + + +# --------------------------------------------------------------------------- +# Rendering +# --------------------------------------------------------------------------- + + +def render_band2(band1: dict[str, Any]) -> str: + lines = [] + for interval in band1.get("intervals") or []: + s, e = interval.get("start_step"), interval.get("end_step") + span = f"Step {s}" if s == e else f"Steps {s}–{e}" + lines.append(f" - {span}: {interval.get('text')}") + return "\n".join(lines) + + +def _render_annotations(chunk: ChunkState) -> list[str]: + if not chunk.annotations: + return [] + lines = [" Post-hoc check results:"] + for ann in chunk.annotations: + lines.append( + f" - [{ann.get('kind', 'check')}] '{ann.get('item_text', '')}'" + f" → {ann.get('status', '?')} ({ann.get('evidence', '')})" + ) + return lines + + +def render_chunk_block(chunk: ChunkState) -> str: + """Full three-band chunk block, strictly in §3.3 order (① → ② → ③).""" + milestone = f' | Milestone "{chunk.milestone_label}"' if chunk.milestone_label else "" + header = ( + f"[Chunk {chunk.ordinal}{milestone} | {chunk.step_range_label}" + f" | {chunk.start_offset} → {chunk.end_offset}]" + ) + parts = [header, ""] + if chunk.status == "ready" and chunk.band1: + b1 = chunk.band1 + parts.append("① Synopsis & effects") + parts.append(f" What this segment was doing: {b1.get('doing', '')}") + parts.append(f" What was actually done: {b1.get('did', '')}") + parts.append(f" Effects / left behind: {b1.get('effect', '')}") + parts.append( + f" Entry: {b1.get('entry_state', '')} Exit: {b1.get('exit_state', '')}" + ) + parts.append(f" Verified: {'; '.join(b1.get('verified_facts') or []) or '-'}") + parts.append(f" Unresolved: {'; '.join(b1.get('unresolved') or []) or '-'}") + parts.append(f" Failed paths: {'; '.join(b1.get('failed_paths') or []) or '-'}") + parts.append(f" Entities: {'; '.join(b1.get('important_entities') or []) or '-'}") + parts.extend(_render_annotations(chunk)) + parts.append("") + parts.append("② Compressed step summary") + parts.append(chunk.band2 or render_band2(chunk.band1)) + else: + parts.append(CHUNK_PENDING_NOTE) + parts.extend(_render_annotations(chunk)) + parts.append("") + parts.append("③ Step action ledger") + parts.append(chunk.band3) + return "\n".join(parts) + + +_SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?。！？])\s+") + +#: Note references inside a band-① ``effect`` (note keys are file-shaped: +#: ``notes/`` paths or ``*.md`` files) — mechanical extraction only. +_NOTE_REF_RE = re.compile(r"\bnotes/[\w\-.]+|\b[\w\-./]+\.md\b") + +#: Cap on merged verified_facts quoted inside a period paragraph. +_PERIOD_FACTS_LIMIT = 4 + + +def _first_sentence(text: str) -> str: + text = (text or "").strip() + if not text: + return "" + return _SENTENCE_SPLIT_RE.split(text, 1)[0].strip().rstrip(".。") + + +def render_era_period_paragraph(era: EraState) -> str: + """One-paragraph period synopsis for a recall-only era (§3.3 extreme layer). + + Mechanically assembled from the member chunks' band-① fields — never an + extra LLM call: the first non-empty ``doing``, each chunk's first ``did`` + sentence chained in order (a band-①-less/pending chunk falls back to its + milestone label or step range), the merged note references found in the + ``effect`` texts, and the first few merged ``verified_facts``. When no + chunk has a band ① at all, it degrades to the milestone-label list with + step ranges — the paragraph is never empty. + """ + ready = [c for c in era.chunks if c.band1] + if not ready: + labels = [ + f"{c.milestone_label or 'segment'} ({c.step_range_label})" + for c in era.chunks + ] + return "Milestones: " + "; ".join(labels) + "." + + sentences: list[str] = [] + doing = next( + ( + str(c.band1.get("doing") or "").strip() + for c in era.chunks + if c.band1 and str(c.band1.get("doing") or "").strip() + ), + "", + ) + if doing: + sentences.append(doing if doing.endswith((".", "!", "?", "。", "！", "？")) else doing + ".") + + did_parts: list[str] = [] + for chunk in era.chunks: + did = _first_sentence(str(chunk.band1.get("did") or "")) if chunk.band1 else "" + if not did: + did = chunk.milestone_label or ( + f"steps {chunk.start_step_number}–{chunk.end_step_number}" + ) + did_parts.append(did) + sentences.append("Did: " + "; ".join(did_parts) + ".") + + note_refs: list[str] = [] + for chunk in ready: + for ref in _NOTE_REF_RE.findall(str(chunk.band1.get("effect") or "")): + if ref not in note_refs: + note_refs.append(ref) + if note_refs: + sentences.append("Notes left: " + ", ".join(note_refs) + ".") + + facts = merge_structured_fields([c.band1 for c in ready])["verified_facts"] + if facts: + sentences.append("Verified: " + "; ".join(facts[:_PERIOD_FACTS_LIMIT]) + ".") + return " ".join(sentences) + + +def render_era_block(era: EraState) -> str: + """Era block: ① merged headers, ② degraded to titles, ③ per-chunk ledgers. + + A recall-only era instead renders as the extreme-layer *period paragraph* + (§10 decision 5, final review 2026-09-01): step range + start/end session + offsets in the header (period addressability — the coarse foreign key for + video alignment), a mechanically assembled synopsis, the recall guidance + line, and the verbatim never-evict user-injection lines. + """ + if era.recall_only: + lines = [ + ( + f"[Era {era.ordinal} | Steps {era.start_step_number}–{era.end_step_number}" + f" | {era.chunks[0].start_offset} → {era.chunks[-1].end_offset}]" + f" {render_era_period_paragraph(era)}" + ), + RECALL_GUIDANCE_TEMPLATE.format( + start=era.start_step_number, end=era.end_step_number + ), + ] + # Never-evict: user-injected instruction lines survive even recall-only. + for chunk in era.chunks: + lines.extend(chunk.user_lines) + return "\n".join(lines) + + merged = merge_structured_fields([c.band1 for c in era.chunks if c.band1]) + entry = next((c.band1.get("entry_state") for c in era.chunks if c.band1), "") or "" + exit_state = "" + for chunk in reversed(era.chunks): + if chunk.band1: + exit_state = chunk.band1.get("exit_state") or "" + break + + parts = [ + ( + f"[Era {era.ordinal} | Steps {era.start_step_number}–{era.end_step_number}" + f" | {era.chunks[0].start_offset} → {era.chunks[-1].end_offset}" + f" | merged from {len(era.chunks)} chunks]" + ), + "", + "① Merged synopsis (structured fields, set-merged)", + f" Entry: {entry} Exit: {exit_state}", + f" Verified: {'; '.join(merged['verified_facts']) or '-'}", + f" Unresolved: {'; '.join(merged['unresolved']) or '-'}", + f" Failed paths: {'; '.join(merged['failed_paths']) or '-'}", + f" Entities: {'; '.join(merged['important_entities']) or '-'}", + "", + "② Segment titles", + ] + for chunk in era.chunks: + title = chunk.milestone_label + if not title and chunk.band1: + title = str(chunk.band1.get("did") or "").split(".")[0] + parts.append(f" - {chunk.step_range_label}: {title or 'segment'}") + parts.extend(_render_annotations(chunk)) + parts.append("") + parts.append("③ Step action ledger") + for chunk in era.chunks: + parts.append(chunk.band3) + return "\n".join(parts) + + +def render_l3_snapshot( + eras: list[EraState], + chunks: list[ChunkState], + *, + goal: str | None, + plan_text: str | None, +) -> str: + """L3 emergency snapshot: merged knowledge plane + minimal per-step index. + + The chunk headers are mechanically set-merged (no summary-of-summary); the + per-step skeleton survives at minimal width so step-level time perception + never breaks (§3.3 L3). + """ + all_chunks = [c for era in eras for c in era.chunks] + list(chunks) + merged = merge_structured_fields([c.band1 for c in all_chunks if c.band1]) + exit_state = "" + for chunk in reversed(all_chunks): + if chunk.band1: + exit_state = chunk.band1.get("exit_state") or "" + break + + start = all_chunks[0].start_step_number if all_chunks else 0 + end = all_chunks[-1].end_step_number if all_chunks else 0 + parts = [ + f"[Session snapshot | Steps {start}–{end} | hard-threshold fallback]", + f"Overall goal: {goal or '-'}", + "Plan state:", + (plan_text or "-").rstrip(), + f"Verified facts: {'; '.join(merged['verified_facts']) or '-'}", + f"Unresolved: {'; '.join(merged['unresolved']) or '-'}", + f"Failed paths: {'; '.join(merged['failed_paths']) or '-'}", + f"Important entities: {'; '.join(merged['important_entities']) or '-'}", + f"Device/app state: {exit_state or '-'}", + "", + "--- Step index (minimal width, per chunk) ---", + ] + for chunk in all_chunks: + parts.append(f"[{chunk.step_range_label}]") + parts.append(chunk.minimal_index) + return "\n".join(parts) + + +# --------------------------------------------------------------------------- +# HistoryChunkManager — triggers, compression events, F-region rendering +# --------------------------------------------------------------------------- + + +class HistoryChunkManager: + """Owner of L2/L3 compression policy over a :class:`TranscriptLedger`. + + Fed by the graph (stamped subgoal hashes, plan-write boundary hints, + checkpoint harvests) and consulted by the ledger at render time + (:meth:`on_render`). All deep mutations of the frozen region happen inside + a compression event; between events the frozen blocks are byte-stable. + """ + + def __init__( + self, + *, + engine: Any = None, + ctx: Any = None, + chunking_config: Any = None, + transcript_config: Any = None, + capsule_service: StepMemoryService | None = None, + meter_getter: Callable[[], int | None] | None = None, + goal: str | None = None, + ): + self._engine = engine + self._ctx = ctx + self._goal = goal + + cc = chunking_config + self._max_steps = int(getattr(cc, "max_steps", 12) or 12) + self._target_source_tokens = int(getattr(cc, "target_source_tokens", 2000) or 2000) + self._model_name = getattr(cc, "model", None) or "gemini-3.7-flash" + self._max_chunks = int(getattr(cc, "max_chunks", 8) or 8) + # Independent era cap (M5): None follows max_chunks (pre-M5 behavior). + self._max_eras = int(getattr(cc, "max_eras", None) or self._max_chunks) + + tc = transcript_config + self._budget = int(getattr(tc, "context_budget_tokens", 80000) or 80000) + self._soft_ratio = float(getattr(tc, "soft_ratio", 0.7) or 0.7) + self._hard_ratio = float(getattr(tc, "hard_ratio", 0.9) or 0.9) + self._min_active_steps = int(getattr(tc, "min_active_steps", 5) or 5) + + self._meter_getter = meter_getter + self._capsule_service = capsule_service or self._build_capsule_service(ctx) + + # Trigger state (fed from the graph). + self._step_hashes: dict[str, str] = {} + self._last_hash: str | None = None + self._boundary_hint_pending = False + + # Mirrors. ``_chunks``/``_eras`` hold SWAPPED chunks (their turns are + # frozen out of the transcript); ``_awaiting`` holds closed segments + # whose original turns still live in the transcript until their + # capsule header is ready (ready-gated swap). An entry's ``chunk`` is + # None when the segment had no step records — it can then only leave + # the queue through the hard-threshold emergency swap. + self._chunks: list[ChunkState] = [] + self._eras: list[EraState] = [] + self._awaiting: list[dict[str, Any]] = [] + self._chunk_counter = 0 + self._era_counter = 0 + + def _build_capsule_service(self, ctx: Any) -> StepMemoryService: + kwargs: dict[str, Any] = {} + try: + from artemis.config import load_agent_config + + runtime = load_agent_config().memory.runtime + kwargs = { + "retry_limit": runtime.retry_limit, + "max_concurrency": runtime.max_concurrency, + "flush_timeout_s": runtime.flush_timeout_s, + } + except Exception: + pass + lens = StepCapsuleLens( + self._model_name, + ctx=ctx, + fallback_model_name=self._resolve_capsule_fallback_model(ctx), + ) + return ChunkCapsuleService(ctx, lens, **kwargs) + + def _resolve_capsule_fallback_model(self, ctx: Any) -> str | None: + """Fallback model for capsule generation when `chunking.model` is down. + + Resolved from the LLM config's summarizer role (which inherits the + global default fallback unless overridden). Only same-provider (google) + fallbacks apply — the capsule lens rides the raw google model path. + """ + try: + llm_cfg = getattr(ctx, "llm_config", None) if ctx is not None else None + if llm_cfg is None: + from artemis.config.llm import get_default_llm_config + + llm_cfg = get_default_llm_config() + fallback = getattr(getattr(llm_cfg, "summarizer", None), "fallback", None) + provider = str(getattr(fallback, "provider", "") or "") + model = getattr(fallback, "model", None) + if model and provider in ("google", "gemini") and model != self._model_name: + return str(model) + except Exception: + pass + return None + + @property + def capsule_service(self) -> StepMemoryService: + return self._capsule_service + + @property + def chunks(self) -> tuple[ChunkState, ...]: + return tuple(self._chunks) + + @property + def eras(self) -> tuple[EraState, ...]: + return tuple(self._eras) + + @property + def awaiting_chunks(self) -> tuple[ChunkState, ...]: + """Closed-but-unswapped chunks (their original turns are still live).""" + return tuple(e["chunk"] for e in self._awaiting if e["chunk"] is not None) + + @property + def boundary_hint_pending(self) -> bool: + return self._boundary_hint_pending + + # ------------------------------------------------------------------ + # Graph-fed trigger events + # ------------------------------------------------------------------ + + def queue_boundary_hint(self) -> None: + """A plan write completed a top-level milestone: queue an *unconfirmed* + boundary. Only the next stamped step confirms it (a hash change); a + stamp without a hash change (e.g. the write was vetoed and rolled + back) discards the hint — pseudo-switch protection (§3.3).""" + self._boundary_hint_pending = True + + def on_step_stamped(self, step_id: str, subgoal_hash: str | None) -> None: + """Record one executed step's stamped subgoal hash (the sole milestone + fact source: a change between consecutive stamps IS the switch).""" + stamped = subgoal_hash or "default" + self._step_hashes[str(step_id)] = stamped + if self._last_hash is not None and stamped != self._last_hash: + if self._boundary_hint_pending: + logger.info("HistoryChunkManager: queued boundary confirmed by stamp change.") + self._boundary_hint_pending = False + elif self._boundary_hint_pending: + logger.info( + "HistoryChunkManager: queued boundary NOT confirmed by the next" + " stamped step (plan write likely rolled back); discarding." + ) + self._boundary_hint_pending = False + self._last_hash = stamped + + def annotate_from_checkpoint( + self, checkpoint_id: str, verdicts: list[dict[str, Any]] + ) -> bool: + """Post-hoc annotation: a harvested checkpoint verdict landed after its + segment was chunked. The matching chunk gains an annotation and a + version bump (DB immediately; the frozen text re-renders only at the + next compression event).""" + if not verdicts: + return False + aliases = self._subgoal_aliases(checkpoint_id) + annotated = False + for chunk in self._all_chunks(): + if chunk.subgoal_hash and chunk.subgoal_hash in aliases: + chunk.annotations.extend(verdicts) + chunk.version += 1 + self._persist(chunk) + annotated = True + return annotated + + def _subgoal_aliases(self, checkpoint_id: str) -> set[str]: + aliases = {checkpoint_id} + try: + from artemis.utils.task_tree import get_all_subgoal_aliases + + base_dir = getattr(self._engine, "base_dir", None) + if base_dir: + aliases |= set(get_all_subgoal_aliases(checkpoint_id, base_dir)) + except Exception: + pass + return aliases + + def _all_chunks(self) -> list[ChunkState]: + return ( + [c for era in self._eras for c in era.chunks] + + self._chunks + + [e["chunk"] for e in self._awaiting if e["chunk"] is not None] + ) + + # ------------------------------------------------------------------ + # Render-time compression events + # ------------------------------------------------------------------ + + def on_render(self, ledger) -> None: + """Evaluate triggers and, when one fires, run one compression event.""" + try: + self._on_render_inner(ledger) + except Exception as e: + logger.error(f"History chunk compression event failed: {e}") + + def _on_render_inner(self, ledger) -> None: + base_tokens = self._context_base_tokens() + soft = base_tokens is not None and base_tokens >= self._budget * self._soft_ratio + hard = base_tokens is not None and base_tokens >= self._budget * self._hard_ratio + + closed_any = self._close_new_segments(ledger, soft) + if closed_any or soft or hard: + # Failure rung of the degradation ladder: exhausted capsules are + # re-dispatched (the lens retries a fallback model per attempt); + # the original text stays until one attempt lands. Cadence is + # bounded to trigger/pressure renders, never every render. + self._redispatch_failed_capsules() + self._harvest_capsules() + self._swap_ready_segments(ledger, hard=hard) + + def _close_new_segments(self, ledger, soft: bool) -> bool: + """Trigger evaluation: close due segments and dispatch their capsules. + + Closing NEVER freezes turns (ready-gated swap): the segment's original + messages stay live in the transcript; a closed segment joins the + ``_awaiting`` queue until its capsule header is ready. + """ + turns = ledger.unchunked_turns() + if len(turns) <= self._min_active_steps: + return False # sliding-window floor: nothing is eligible + eligible_count = len(turns) - self._min_active_steps + claimed = sum(len(e["turns"]) for e in self._awaiting) + if eligible_count <= claimed: + return False + remaining = turns[claimed:] + remaining_eligible = eligible_count - claimed + + segments = self._partition(remaining) + if not segments: + return False + + # Milestone trigger: a *complete* closed segment — one that ended (a + # later segment follows, or its hash differs from the current stamp) + # and has fully aged past the sliding-window floor — closes whole + # (§3.3: the previous segment becomes ONE HistoryChunk; the floor only + # delays the event, it never splits the segment). The trailing segment + # is subject to the size/soft triggers over its eligible portion only. + selected: list[tuple[str | None, list[dict]]] = [] + open_hash = self._last_hash + + consumed_prefix = 0 + tail_segment: tuple[str | None, list[dict]] | None = None + for idx, segment in enumerate(segments): + seg_hash, seg_turns = segment + is_last = idx == len(segments) - 1 + closed = (not is_last) or seg_hash != open_hash + if closed and consumed_prefix + len(seg_turns) <= remaining_eligible: + selected.append(segment) + consumed_prefix += len(seg_turns) + continue + tail_segment = segment + break + + milestone_event = bool(selected) + + tail_portion: list[dict] = [] + if tail_segment is not None: + tail_portion = tail_segment[1][: max(0, remaining_eligible - consumed_prefix)] + + size_event = False + if tail_segment is not None and tail_portion: + tail_chars = ledger.turn_text_chars(tail_portion) + size_event = ( + len(tail_segment[1]) >= self._max_steps + or (tail_chars // 4) >= self._target_source_tokens + ) + + if size_event: + selected.append((tail_segment[0], tail_portion)) + elif not milestone_event and soft and tail_portion: + # Soft threshold with nothing else due: close the oldest open + # segment's eligible portion (bounded by the chunk size cap). + selected.append((tail_segment[0], tail_portion[: self._max_steps])) + + if not selected: + return False + + steps_by_id = self._load_steps_by_id() + for seg_hash, seg_turns in selected: + for slice_turns in self._slices(seg_turns): + chunk = self._create_chunk(slice_turns, seg_hash, steps_by_id) + self._awaiting.append({"chunk": chunk, "turns": list(slice_turns)}) + logger.info( + f"History segments closed (ready-gated): {len(self._awaiting)} awaiting" + " capsule headers; original turns retained until ready." + ) + return True + + def _redispatch_failed_capsules(self) -> None: + for entry in self._awaiting: + chunk = entry["chunk"] + if chunk is None or chunk.status != "pending": + continue + key = chunk.capsule_key + try: + if not self._capsule_service.has_failed(key): + continue + payload = self._capsule_service.get_job_payload(key) + if payload is None: + continue + logger.info( + f"Re-dispatching failed capsule {key}; original text is" + " retained until a capsule lands." + ) + self._capsule_service.submit(key, payload) + except Exception as e: + logger.error(f"Capsule re-dispatch for {key} failed: {e}") + + def _swap_ready_segments(self, ledger, *, hard: bool) -> None: + """Swap the ready prefix of awaiting segments into the frozen region. + + Swaps consume the transcript's oldest unchunked turns, so only a + contiguous READY prefix may swap — an older pending segment keeps its + (and every younger segment's) original text live. The hard threshold + is the sole exception: everything closed force-swaps, pending chunks + included, and the frozen region renders as the L3 snapshot. + """ + if not self._awaiting: + return + swap: list[dict[str, Any]] = [] + if hard: + swap, self._awaiting = self._awaiting, [] + else: + while self._awaiting: + chunk = self._awaiting[0]["chunk"] + if chunk is None or chunk.status != "ready": + break + swap.append(self._awaiting.pop(0)) + if not swap: + return + + for entry in swap: + if entry["chunk"] is not None: + self._chunks.append(entry["chunk"]) + self._fold_eras() + blocks = self._render_frozen_blocks(hard=hard) + consumed = sum(len(e["turns"]) for e in swap) + ledger.freeze_turns(consumed, blocks) + logger.info( + f"History compression swap: froze {consumed} turns" + f" ({len(self._chunks)} chunks, {len(self._eras)} eras," + f" {len(self._awaiting)} still awaiting, hard={hard})." + ) + + def _partition(self, eligible: list[dict]) -> list[tuple[str | None, list[dict]]]: + """Split eligible turns into consecutive same-hash segments.""" + segments: list[tuple[str | None, list[dict]]] = [] + current_hash: str | None = None + current: list[dict] = [] + for turn in eligible: + key = turn.get("step_key") + stamped = self._step_hashes.get(str(key)) if key is not None else None + if stamped is None: + stamped = current_hash # unknown stamps join the running segment + if current and stamped == current_hash: + current.append(turn) + else: + if current: + segments.append((current_hash, current)) + current_hash = stamped + current = [turn] + if current: + segments.append((current_hash, current)) + return segments + + def _slices(self, seg_turns: list[dict]) -> list[list[dict]]: + return [ + seg_turns[i : i + self._max_steps] + for i in range(0, len(seg_turns), self._max_steps) + ] + + def _context_base_tokens(self) -> int | None: + if self._meter_getter is not None: + try: + return self._meter_getter() + except Exception: + return None + try: + from artemis.services.token_meter import get_meter + + session_id = getattr(self._engine, "current_session_id", None) + if not session_id: + return None + last = get_meter(session_id).last_prompt_tokens + return last or None + except Exception: + return None + + # ------------------------------------------------------------------ + # Chunk creation + # ------------------------------------------------------------------ + + def _load_steps_by_id(self) -> dict[str, dict]: + steps: list[dict] = [] + try: + if self._engine is not None: + steps = self._engine.get_agent_friendly_steps() or [] + except Exception as e: + logger.error(f"Failed to load steps for chunking: {e}") + return {str(s.get("step_id")): s for s in steps} + + def _session_start(self) -> float | None: + return getattr(self._engine, "session_start_time", None) + + def _create_chunk( + self, + slice_turns: list[dict], + seg_hash: str | None, + steps_by_id: dict[str, dict], + ) -> ChunkState | None: + step_keys = [str(t.get("step_key")) for t in slice_turns if t.get("step_key")] + steps = [steps_by_id[k] for k in step_keys if k in steps_by_id] + steps.sort(key=lambda s: s.get("step_number") or 0) + if not steps: + logger.warning( + "History chunk skipped: no DataEngine step records for the" + f" selected turns ({len(slice_turns)} turns)." + ) + return None + + session_start = self._session_start() + start_step, end_step = steps[0], steps[-1] + self._chunk_counter += 1 + chunk = ChunkState( + ordinal=self._chunk_counter, + start_step_number=int(start_step.get("step_number")), + end_step_number=int(end_step.get("step_number")), + start_step_id=str(start_step.get("step_id")), + end_step_id=str(end_step.get("step_id")), + source_step_ids=[str(s.get("step_id")) for s in steps], + subgoal_hash=seg_hash, + milestone_label=self._resolve_label(seg_hash), + start_offset=step_offset_label(start_step, session_start), + end_offset=step_offset_label(end_step, session_start), + band3=build_action_ledger(steps, session_start), + minimal_index=build_action_ledger(steps, session_start, minimal=True), + user_lines=[ + line for line in (injected_instruction_line(s) for s in steps) if line + ], + ) + # Ready gating: the caller queues the chunk as awaiting — it only + # enters self._chunks (and the frozen region) once its capsule is + # ready, or through the hard-threshold emergency swap. + self._persist(chunk) + self._dispatch_capsule(chunk, steps, session_start) + return chunk + + def _dispatch_capsule( + self, chunk: ChunkState, steps: list[dict], session_start: float | None + ) -> None: + payload_steps = [] + for step in steps: + payload_steps.append( + { + "step_number": step.get("step_number"), + "offset": step_offset_label(step, session_start), + "action": _action_phrase(step), + "outcome": _result_phrase(step.get("last_execution_result")), + "visual_summary": step.get("summary"), + "thinking_excerpt": (step.get("operator_raw_thinking") or "")[:1200] + or None, + "note_writes": extract_note_writes(step), + "injected_instruction": (step.get("extra_metadata") or {}).get( + "injected_instruction" + ), + "fa_actions": _fa_recovery_lines(step), + } + ) + payload = { + "start_step": chunk.start_step_number, + "end_step": chunk.end_step_number, + "step_number": chunk.start_step_number, # service log display + "milestone_label": chunk.milestone_label, + "steps": payload_steps, + } + try: + self._capsule_service.submit(chunk.capsule_key, payload) + except Exception as e: + logger.error(f"Failed to dispatch chunk capsule {chunk.capsule_key}: {e}") + + def _resolve_label(self, seg_hash: str | None) -> str | None: + if not seg_hash or seg_hash == "default": + return None + try: + from artemis.utils.notes import get_note_file_path + from artemis.utils.plan_grammar import parse_plan, subgoal_hash + from artemis.utils.task_tree import get_all_subgoal_aliases + + base_dir = getattr(self._engine, "base_dir", None) + if not base_dir: + return None + plan_path = get_note_file_path(base_dir, "task_plan") + if not plan_path.exists(): + return None + for item in parse_plan(plan_path.read_text(encoding="utf-8")).items: + item_hash = subgoal_hash(item.text) + if item_hash == seg_hash or seg_hash in get_all_subgoal_aliases( + item_hash, base_dir + ): + return item.text + except Exception: + return None + return None + + # ------------------------------------------------------------------ + # Capsule harvest / persistence + # ------------------------------------------------------------------ + + def _harvest_capsules(self) -> None: + """Fold ready capsule results into pending chunk mirrors (event-time).""" + for chunk in self._all_chunks(): + if chunk.status != "pending": + continue + raw = self._capsule_service.get_summary(chunk.capsule_key) + if not raw: + continue + try: + band1 = json.loads(raw) + except Exception: + continue + chunk.band1 = band1 + chunk.band2 = render_band2(band1) + chunk.status = "ready" + chunk.version += 1 + self._persist(chunk) + + def _persist(self, chunk: ChunkState) -> None: + try: + if self._engine is None or not hasattr(self._engine, "record_history_chunk"): + return + band1 = dict(chunk.band1) + if chunk.annotations: + band1["annotations"] = chunk.annotations + self._engine.record_history_chunk( + start_step_number=chunk.start_step_number, + end_step_number=chunk.end_step_number, + version=chunk.version, + status=chunk.status, + start_step_id=chunk.start_step_id, + end_step_id=chunk.end_step_id, + source_step_ids=chunk.source_step_ids, + subgoal_hash=chunk.subgoal_hash, + band1=band1, + band2=chunk.band2, + band3=chunk.band3, + rendered_text=render_chunk_block(chunk), + ) + except Exception as e: + logger.error(f"Failed to persist history chunk: {e}") + + # ------------------------------------------------------------------ + # Era folding + frozen-region rendering + # ------------------------------------------------------------------ + + def _fold_eras(self) -> None: + overflow = len(self._chunks) - self._max_chunks + if overflow > 0: + folded, self._chunks = self._chunks[:overflow], self._chunks[overflow:] + self._era_counter += 1 + self._eras.append(EraState(ordinal=self._era_counter, chunks=folded)) + era_overflow = len(self._eras) - self._max_eras + if era_overflow > 0: + for era in self._eras[:era_overflow]: + if not era.recall_only: + era.recall_only = True + logger.info( + f"Era {era.ordinal} overflowed to the recall-only period" + f" paragraph (Steps {era.start_step_number}–" + f"{era.end_step_number})." + ) + + def _render_frozen_blocks(self, *, hard: bool) -> list[BaseMessage]: + if not self._chunks and not self._eras: + return [] + if hard: + plan_text = self._read_plan_text() + text = render_l3_snapshot( + self._eras, self._chunks, goal=self._goal, plan_text=plan_text + ) + return [HumanMessage(content=[{"type": "text", "text": text}])] + blocks: list[BaseMessage] = [] + for era in self._eras: + blocks.append( + HumanMessage(content=[{"type": "text", "text": render_era_block(era)}]) + ) + for chunk in self._chunks: + blocks.append( + HumanMessage(content=[{"type": "text", "text": render_chunk_block(chunk)}]) + ) + return blocks + + def _read_plan_text(self) -> str | None: + try: + from artemis.utils.notes import get_note_file_path + + base_dir = getattr(self._engine, "base_dir", None) + if not base_dir: + return None + path = get_note_file_path(base_dir, "task_plan") + return path.read_text(encoding="utf-8") if path.exists() else None + except Exception: + return None + + # ------------------------------------------------------------------ + # Draining + # ------------------------------------------------------------------ + + async def flush(self, timeout_seconds: float | None = None) -> None: + """Drain in-flight capsule jobs, then persist any harvested results. + + Frozen transcript text is deliberately NOT re-rendered here (deep + mutations only happen at compression events); this keeps the DB copy + complete at session end. + """ + try: + await self._capsule_service.flush(timeout_seconds) + except Exception: + pass + try: + self._harvest_capsules() + except Exception as e: + logger.debug(f"Chunk capsule harvest at flush skipped: {e}") diff --git a/artemis/memory/context_policy.py b/artemis/memory/context_policy.py new file mode 100644 index 0000000..d6fafe9 --- /dev/null +++ b/artemis/memory/context_policy.py @@ -0,0 +1,248 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ContextPolicy: the declarative per-agent compiled-history policy table (M4). + +Every production caller of :func:`artemis.utils.task_tree.build_plan_and_history` +used to hard-code its visibility kwargs at the call site. This module gathers +those call sites into one declaration per agent — the rendered output for each +agent is byte-identical to the previous hard-coded call (pinned by the golden +tests in ``tests/unit/memory/test_context_policy.py``) — and adds the M4 chunk +hand-off: when the transcript flag is on and history chunks exist, the +compiled view replaces chunked-away step lines with chunk blocks. + +Chunk view per policy (§10 decision 4): + +- ``"full"`` (outputter / history_analyzer): the chunk's stored three-band + rendering including the ③ full-width per-step ledger; +- ``"digest"`` (everything else): bands ① + ② with the ③ ledger replaced by + a ``ledger via recall_history`` marker line. + +Because chunks only exist when ``agent.memory.transcript.enabled`` is on, the +flag-off output of every agent stays byte-for-byte unchanged. + +``agent.memory.policies`` allows per-agent field overrides, e.g.:: + + "memory": {"policies": {"planner": {"last_n_detailed": 3}}} +""" + +from dataclasses import dataclass, fields, replace +from typing import Any + +from artemis.utils.logger import get_logger + +logger = get_logger(__name__) + +#: Marker appended to a digest-view chunk block in place of the ③ ledger. +DIGEST_LEDGER_MARKER_TEMPLATE = ( + "③ Step action ledger: available via recall_history (Steps {start}–{end})" +) + +#: The band-③ heading inside a chunk's stored ``rendered_text``. +_BAND3_HEADING = "③ Step action ledger" + + +@dataclass(frozen=True) +class ContextPolicy: + """One agent's compiled-history rendering declaration. + + Field semantics map 1:1 onto ``build_plan_and_history`` kwargs, with two + conveniences: ``uncompressed=True`` renders every step (the historical + ``min_summaries=len(steps)`` idiom) and ``whitelist=True`` marks policies + whose caller supplies ``keep_subgoal_hashes`` at runtime. + """ + + strategy: str = "sliding_window" # documentation only + strict_milestone_pruning: bool = False + recent_window_size: int = 3 + min_summaries: int = 5 + uncompressed: bool = False + last_n_detailed: int = 1 + all_detailed: bool = False + for_failure_analyzer: bool = False + whitelist: bool = False + chunk_view: str | None = "digest" # "full" | "digest" | None (never chunks) + + +#: The eight production call sites, verbatim (kwargs recorded 2026-09-01). +CONTEXT_POLICIES: dict[str, ContextPolicy] = { + # operator.py:996 — last_n_detailed comes from the node constructor. + "operator": ContextPolicy( + strategy="strict_milestone", + strict_milestone_pruning=True, + recent_window_size=3, + ), + # operator.py:341 — transcript F-region cold-start restore. + "operator_cold_start": ContextPolicy( + strategy="full", + uncompressed=True, + last_n_detailed=1, + chunk_view=None, + ), + # failure_analyzer.py:361 + "failure_analyzer": ContextPolicy( + strategy="strict_milestone", + strict_milestone_pruning=True, + recent_window_size=3, + last_n_detailed=1, + for_failure_analyzer=True, + ), + # planner.py:133 — caller passes an empty plan and hash "default". + "planner": ContextPolicy( + strategy="strict_milestone", + strict_milestone_pruning=True, + recent_window_size=5, + last_n_detailed=2, + ), + # outputter.py:81 + "outputter": ContextPolicy( + strategy="full", + uncompressed=True, + last_n_detailed=0, + chunk_view="full", + ), + # history_analyzer.py:113 + "history_analyzer": ContextPolicy( + strategy="full", + uncompressed=True, + last_n_detailed=1, + chunk_view="full", + ), + # diagnoser.py:166 — every non-whitelist kwarg is the historical default. + "diagnoser": ContextPolicy(strategy="milestone_whitelist", whitelist=True), + # committee_tool.py:189 + "committee": ContextPolicy( + strategy="milestone_whitelist", + whitelist=True, + last_n_detailed=3, + ), +} + +_POLICY_FIELDS = {f.name for f in fields(ContextPolicy)} + + +def resolve_policy(agent: str) -> ContextPolicy: + """The agent's policy with any ``agent.memory.policies`` overrides applied.""" + try: + policy = CONTEXT_POLICIES[agent] + except KeyError: + raise KeyError( + f"No ContextPolicy declared for agent '{agent}';" + f" known agents: {sorted(CONTEXT_POLICIES)}" + ) from None + + overrides: dict[str, Any] = {} + try: + from artemis.config import load_agent_config + + configured = (load_agent_config().memory.policies or {}).get(agent) or {} + overrides = {k: v for k, v in configured.items() if k in _POLICY_FIELDS} + unknown = set(configured) - _POLICY_FIELDS + if unknown: + logger.warning( + f"agent.memory.policies['{agent}'] ignores unknown fields: {sorted(unknown)}" + ) + except Exception: + overrides = {} + return replace(policy, **overrides) if overrides else policy + + +def _chunk_digest_text(rendered_text: str, start: int, end: int) -> str: + """Bands ① + ② of a stored chunk rendering, ③ replaced by the recall marker.""" + marker = DIGEST_LEDGER_MARKER_TEMPLATE.format(start=start, end=end) + head, sep, _ = rendered_text.partition(f"{_BAND3_HEADING}\n") + if not sep: + return f"{rendered_text.rstrip()}\n{marker}" + return f"{head.rstrip()}\n\n{marker}" + + +def load_chunk_blocks(engine: Any, view: str | None) -> list[dict] | None: + """The persisted history chunks rendered for the compiled view, or None. + + Returns None (→ byte-identical legacy rendering) unless the transcript + flag is on, the engine exposes chunks, and at least one chunk exists. + """ + if engine is None or view not in ("full", "digest"): + return None + try: + from artemis.config import load_agent_config + + if not load_agent_config().memory.transcript.enabled: + return None + except Exception: + return None + try: + rows = engine.get_history_chunks() or [] + except Exception as e: + logger.debug(f"load_chunk_blocks: no chunks available: {e}") + return None + + blocks: list[dict] = [] + for row in rows: + start = getattr(row, "start_step_number", None) + end = getattr(row, "end_step_number", None) + rendered = getattr(row, "rendered_text", None) + if not isinstance(start, int) or not isinstance(end, int) or not rendered: + continue + text = ( + rendered + if view == "full" + else _chunk_digest_text(rendered, start, end) + ) + blocks.append( + {"start_step_number": start, "end_step_number": end, "text": text} + ) + return blocks or None + + +def build_history_for( + agent: str, + task_plan: str, + steps: list, + current_subgoal_hash: str, + *, + keep_subgoal_hashes: set | None = None, + last_n_detailed: int | None = None, + engine: Any = None, +) -> str: + """Render the agent's compiled plan-and-history view per its policy. + + Args: + agent: Key into :data:`CONTEXT_POLICIES`. + keep_subgoal_hashes: Runtime whitelist (whitelist policies only). + last_n_detailed: Runtime override (the operator's constructor knob). + engine: DataEngine handing chunks to the compiled view; omit at call + sites that must never receive chunk blocks. + """ + from artemis.utils.task_tree import build_plan_and_history + + policy = resolve_policy(agent) + kwargs: dict[str, Any] = { + "min_summaries": len(steps) if policy.uncompressed else policy.min_summaries, + "last_n_detailed": ( + policy.last_n_detailed if last_n_detailed is None else last_n_detailed + ), + "all_detailed": policy.all_detailed, + "strict_milestone_pruning": policy.strict_milestone_pruning, + "recent_window_size": policy.recent_window_size, + "for_failure_analyzer": policy.for_failure_analyzer, + } + if policy.whitelist: + kwargs["keep_subgoal_hashes"] = keep_subgoal_hashes + + chunks = load_chunk_blocks(engine, policy.chunk_view) + if chunks: + kwargs["chunks"] = chunks + + return build_plan_and_history(task_plan, steps, current_subgoal_hash, **kwargs) diff --git a/artemis/memory/step_capsule.md b/artemis/memory/step_capsule.md new file mode 100644 index 0000000..1df5d58 --- /dev/null +++ b/artemis/memory/step_capsule.md @@ -0,0 +1,57 @@ +# ROLE +You are the Segment Capsule writer for a mobile automation agent's history compression. You receive one contiguous segment of executed steps (per-step visual transition summaries, exact actions, controller/validator results, operator reasoning excerpts, notes written, and user-injected instructions) and produce the two LLM-authored bands of the segment's history chunk: + +- Band ① — Synopsis & effects: what this segment was doing, what was actually done, and what effects/artifacts it left behind, plus structured fields. +- Band ② — Compressed step summary: an interval narrative ("Steps xx–xx did ..., Step xx did ...") that seamlessly covers every step of the segment. + +The mechanical per-step action ledger (band ③) is assembled outside of you; do NOT reproduce it. + +# NEUTRAL-WORDING CONTRACT (STRICT) +1. **Zero subjective validation**: never declare semantic success, completion, or final failure. Subjective verdicts in compressed history cause future turns to falsely believe a goal was definitively met or permanently broken. + - **BANNED WORDS**: "successfully", "completed", "achieved", "entered", "navigated to", "arrived at", "failed", "unsuccessful", "could not", "impossible". + - Describe observable transitions and recorded results instead (e.g. "the settings page was displayed", "the controller reported an execution error"). +2. **First-person perspective**: write from the agent's perspective using "I". Never "the agent" / "the operator". +3. **Planned iterations are not anomalies**: systematic processing of different candidate items, or periodic polling cycles under a `[Loop]` / `[Loop:continuous]` milestone, is planned progress — record it objectively (e.g. "polling round #2"). Only flag a loop when the exact same target is repeated without progress or states oscillate blindly; then record the pattern and count factually. +4. **Preserve, don't abstract**: be deliberately detailed and stay close to the source wording. Prefer longer output over abstracting facts beyond recognition. Faithfully carry visible text, codes, prices, account names, file names, error banners. +5. **Fact priority**: user instructions > validator/controller recorded results > UI/OCR deltas > sourced visual summaries > reasoning excerpts. + +# BAND ① REQUIREMENTS +Answer three questions in detailed prose (field `doing` / `did` / `effect`): +- `doing`: what this segment was trying to do and why (strategic context within the plan). +- `did`: what was actually carried out, including detours, retries, recoveries, and pivots. +- `effect`: what state/effects/artifacts the segment left behind. **This MUST include every note left during the segment**: for each notes write in the input, name the target note file and the gist of what was recorded. Every note's target key is machine-checked against your effect field — an omitted note key forces regeneration. + +Structured fields (arrays of short strings, may be empty): +- `verified_facts`: states/values actually observed or verified on screen. +- `unresolved`: open questions, unverified assumptions, missing prerequisites. +- `failed_paths`: approaches attempted in this segment whose recorded results were errors (state the recorded error factually). +- `important_entities`: accounts, files, order/tracking numbers, package names, key UI entry points. +- `entry_state` / `exit_state` (single strings): the observable screen/app state at segment start and end. + +# BAND ② REQUIREMENTS +Field `intervals`: an ordered array of `{"start_step": N, "end_step": M, "text": "..."}` objects. +- Every line's text must reference concrete behavior for those steps. +- **The union of intervals MUST cover the segment's step range seamlessly** — no gaps, no overlaps, in ascending order. This is machine-checked; a gap forces regeneration. +- Group only homogeneous consecutive actions into one interval; a heterogeneous action gets its own single-step interval (`start_step == end_step`). +- Keep the neutral-wording contract in every line. + +# OUTPUT FORMAT +Return ONLY one JSON object (no markdown fences, no commentary): + +```json +{ + "doing": "...", + "did": "...", + "effect": "...", + "entry_state": "...", + "exit_state": "...", + "verified_facts": ["..."], + "unresolved": ["..."], + "failed_paths": ["..."], + "important_entities": ["..."], + "intervals": [ + {"start_step": 18, "end_step": 20, "text": "..."}, + {"start_step": 21, "end_step": 21, "text": "..."} + ] +} +``` diff --git a/artemis/memory/step_memory.py b/artemis/memory/step_memory.py new file mode 100644 index 0000000..c47899f --- /dev/null +++ b/artemis/memory/step_memory.py @@ -0,0 +1,254 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared background step-summary runtime (StepMemoryService). + +Hosts the scheduling skeleton lifted from the Flash VisualStepSummarizer: +zero-blocking dispatch, bounded retry, bounded flush, canonical step_id keying +with alias resolution (tool_call_id / legacy ordinals), and status queries. + +What a summary *contains* (the lens semantics) lives in subclasses — the +service only guarantees how jobs are scheduled, retried, keyed, and drained. +Both Flash and (from M2 on) Pro profiles share this runtime. +""" + +import asyncio +from typing import Any + +from artemis.utils.logger import get_logger + +logger = get_logger(__name__) + +JobKey = int | str + + +class StepLens: + """Formal lens interface: what one summary attempt *produces*. + + A lens turns one job payload into summary text; the surrounding service + owns scheduling, retries, keying, and draining. Implementations must be + side-effect-tolerant under retries (an attempt may run again after a + failure) and return ``None``/raise to signal a failed attempt. + + Two lens altitudes exist in the redesign (§5): the step-level visual + transition lens (:class:`~artemis.agents.flash.summarizer.VisualStepSummarizer` + keeps it inlined as its ``_attempt`` for compatibility) and the chunk-level + :class:`~artemis.memory.chunking.StepCapsuleLens`. + """ + + name = "lens" + + async def render(self, key: JobKey, payload: dict[str, Any]) -> str | None: + """Produce the summary text for one payload, or None on failure.""" + raise NotImplementedError + + +class StepMemoryService: + """Non-blocking background runtime for per-step summary jobs. + + Key design properties: + 1. Zero-blocking dispatch: callers submit and proceed immediately. + 2. Bounded retry: each job attempts at most ``1 + retry_limit`` times and + then enters an explicit failed state — never an unbounded loop. + 3. Bounded flush: ``flush()`` waits up to the configured timeout, then + cancels whatever is still in flight. + 4. Canonical keying with aliases: jobs are keyed by the DataEngine step id + where available; tool_call_ids (and legacy ordinals) remain queryable + through an alias map so message-level consumers need no migration. + 5. Bounded concurrency: at most ``max_concurrency`` attempts run at once. + + Subclasses implement :meth:`_attempt` (one summarization attempt for a + payload) and may override :meth:`_on_status` to persist pending/failed + status transitions. + """ + + def __init__( + self, + ctx: Any, + *, + lens: StepLens | None = None, + max_concurrency: int = 1, + retry_limit: int = 3, + flush_timeout_s: float = 30.0, + ): + self.ctx = ctx + self._lens = lens + self._summaries: dict[JobKey, str] = {} + self._failed: set[JobKey] = set() + self._pending_tasks: dict[JobKey, asyncio.Task] = {} + self._step_inputs: dict[JobKey, dict[str, Any]] = {} + self._retry_counts: dict[JobKey, int] = {} + self._retry_delays = (0.0, 0.5, 1.0, 2.0, 3.0) + self._retry_limit = max(0, retry_limit) + self._flush_timeout_s = max(0.0, flush_timeout_s) + self._aliases: dict[JobKey, JobKey] = {} + self._semaphore = asyncio.Semaphore(max(1, max_concurrency)) + + # ------------------------------------------------------------------ + # Keying + # ------------------------------------------------------------------ + + def resolve_key(self, key: JobKey) -> JobKey: + """Resolve an alias (tool_call_id / legacy ordinal) to its canonical key.""" + return self._aliases.get(key, key) + + def submit( + self, + key: JobKey, + payload: dict[str, Any], + aliases: tuple[JobKey, ...] = (), + ) -> None: + """Submit one summary job without blocking the caller. + + Args: + key: Canonical job key (DataEngine step id where available). + payload: Lens-specific input data; stored until the job resolves. + aliases: Additional keys (e.g. tool_call_id) that resolve to ``key``. + """ + for alias in aliases: + if alias is not None and alias != key: + self._aliases[alias] = key + + self._step_inputs[key] = payload + self._failed.discard(key) + self._on_status(key, "pending") + + task = asyncio.create_task(self._run_until_ready(key)) + self._pending_tasks[key] = task + + # ------------------------------------------------------------------ + # Scheduling core + # ------------------------------------------------------------------ + + async def _run_until_ready(self, key: JobKey) -> None: + """Retry one job independently, bounded by the configured retry limit. + + Attempts at most ``1 + retry_limit`` times; on exhaustion the job + enters an explicit failed state (surfaced via :meth:`has_failed` and + the ``_on_status`` hook). Retry backoff sleeps happen outside the + concurrency semaphore so a waiting job never starves its peers. + """ + while key not in self._summaries: + async with self._semaphore: + succeeded = await self._attempt(key) + if succeeded: + return + + retry_count = self._retry_counts.get(key, 0) + 1 + self._retry_counts[key] = retry_count + payload = self._step_inputs.get(key) + display_step = payload.get("step_number") if payload else key + + if retry_count > self._retry_limit: + self._failed.add(key) + logger.warning( + f"StepMemoryService: Giving up on Step {display_step} after" + f" {retry_count} failed attempts (retry_limit={self._retry_limit})." + ) + self._on_status(key, "failed") + return + + delay = self._retry_delays[min(retry_count - 1, len(self._retry_delays) - 1)] + logger.info( + f"StepMemoryService: Retrying Step {display_step} " + f"(attempt {retry_count + 1}) after {delay:.1f}s" + ) + if delay: + await asyncio.sleep(delay) + + async def _attempt(self, key: JobKey) -> bool: + """Run one summarization attempt for ``key``; True when a summary landed. + + The default implementation delegates to the configured :class:`StepLens`; + subclasses without a lens override this method directly (the Flash + visual summarizer keeps its inlined attempt for compatibility). + """ + if self._lens is None: + raise NotImplementedError + payload = self._step_inputs.get(key) + if payload is None: + return False + try: + summary = await self._lens.render(key, payload) + except Exception as e: + logger.warning(f"StepMemoryService: lens '{self._lens.name}' attempt failed: {e}") + return False + if not summary: + return False + self._summaries[key] = summary + self._on_ready(key, summary) + return True + + def _on_ready(self, key: JobKey, summary: str) -> None: + """Hook invoked when a lens-produced summary lands; default is a no-op.""" + + def _on_status(self, key: JobKey, status: str) -> None: + """Hook invoked on pending/failed transitions; default is a no-op.""" + + # ------------------------------------------------------------------ + # Status queries (alias-resolving) + # ------------------------------------------------------------------ + + def get_summary(self, key: JobKey, fallback_text: str | None = None) -> str | None: + """Retrieve the summary for a job key (or alias), or fallback text.""" + return self._summaries.get(self.resolve_key(key), fallback_text) + + def has_summary(self, key: JobKey) -> bool: + """Check whether a job's summary has completed.""" + return self.resolve_key(key) in self._summaries + + def has_job(self, key: JobKey) -> bool: + """Check whether a job has been submitted for summarization.""" + return self.resolve_key(key) in self._step_inputs + + def has_failed(self, key: JobKey) -> bool: + """Check whether a job permanently failed (retries exhausted).""" + return self.resolve_key(key) in self._failed + + def is_pending(self, key: JobKey) -> bool: + """Check whether a job still needs a summary.""" + return self.has_job(key) and not self.has_summary(key) + + def get_job_payload(self, key: JobKey) -> dict[str, Any] | None: + """Return the stored payload for a job key (or alias), if any.""" + return self._step_inputs.get(self.resolve_key(key)) + + def get_step_number(self, key: JobKey) -> int | None: + """Return the human-facing step ordinal recorded for a job, if known.""" + payload = self.get_job_payload(key) + if payload is None: + return None + step_number = payload.get("step_number") + return step_number if isinstance(step_number, int) else None + + # ------------------------------------------------------------------ + # Draining + # ------------------------------------------------------------------ + + async def flush(self, timeout_seconds: float | None = None) -> None: + """Wait for active jobs up to a bound, then cancel remaining retry loops.""" + if timeout_seconds is None: + timeout_seconds = self._flush_timeout_s + tasks = [t for t in self._pending_tasks.values() if not t.done()] + if tasks: + logger.info(f"StepMemoryService: Flushing {len(tasks)} pending summary tasks...") + _, pending = await asyncio.wait(tasks, timeout=timeout_seconds) + if pending: + logger.warning( + f"StepMemoryService: Cancelling {len(pending)} unfinished summary tasks " + f"after {timeout_seconds:.1f}s flush timeout." + ) + for task in pending: + task.cancel() + await asyncio.gather(*pending, return_exceptions=True) diff --git a/artemis/memory/transcript.py b/artemis/memory/transcript.py new file mode 100644 index 0000000..7a83dcb --- /dev/null +++ b/artemis/memory/transcript.py @@ -0,0 +1,364 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Session-level message transcript ledger for the Pro operator (M2). + +Implements the four-region discipline of the history-module redesign §3.2: + +- **S (stable prefix)**: the static system message — byte-identical for the + whole session (the plan+history section is moved out by the template split). +- **F (frozen history)**: in M2 only the cold-start restored-history block; L2 + chunks arrive in M3. +- **A (active window)**: raw per-turn messages, append-only. Each committed + turn contributes its tail observation HumanMessage, the operator AIMessages + (tool_calls and native thinking preserved by reference), the in-turn + ToolMessages, and the turn's validator result message. The scrub edge + (:class:`~artemis.agents.flash.context_compressor.ScrubEdgeCompressor`, + generalized in M2) strips old UI lists / plan recitations at depth 1 and + resolves screenshots to visual summaries at depth K — messages are never + removed or reordered, so tool-call/response pairs are never split. +- **T (current tail)**: built fresh every turn by the operator; passed to + :meth:`render` and only enters A when the turn is committed. + +All timestamps inside the transcript use the session-start offset ``T+mm:ss`` +(byte-stable once frozen); "ago" wording is reserved for the auxiliary agents' +per-call compiled views. +""" + +import json +import time +from typing import Any, Callable + +from langchain_core.messages import BaseMessage, HumanMessage + +from artemis.agents.flash.context_compressor import ScrubEdgeCompressor +from artemis.memory.step_memory import StepMemoryService +from artemis.utils.logger import get_logger + +logger = get_logger(__name__) + +#: Marker for the per-turn task-plan recitation block in the observation tail. +PLAN_RECITATION_MARKER = "--- Task Plan (recited) ---" + +#: Marker of the Pro observation's UI element list block (see +#: ``ObservationPromptComponent``). +PRO_UI_LIST_MARKER = "--- Visible UI Elements ---" + +#: Marker prefix of a committed turn's validator result message. +EXECUTION_RESULT_MARKER = "--- Action Execution Result" + +#: Header prefix of the cold-start restored-history block. +RESTORED_HISTORY_HEADER = "[Restored history]" + + +def format_session_offset(seconds: float) -> str: + """Render a session-start offset as ``T+mm:ss`` (minutes never wrap).""" + total = max(0, int(seconds)) + return f"T+{total // 60:02d}:{total % 60:02d}" + + +class TranscriptLedger: + """Append-only four-region message ledger for one Pro session. + + The ledger owns no LLM calls: summary lookups go through the shared + :class:`StepMemoryService` (``ctx.step_memory``), whose visual-transition + lens is fed by the Pro SummarizerNode each step. + """ + + def __init__( + self, + *, + step_memory: StepMemoryService | None = None, + prune_history_xml: bool = True, + image_scrub_depth: int = 3, + pending_grace_steps: int = 3, + xml_scrub_depth: int = 1, + clock: Callable[[], float] | None = None, + ): + self._clock = clock or time.monotonic + self._session_start = self._clock() + + self._static: list[BaseMessage] = [] + self._restored: list[BaseMessage] = [] + self._active: list[BaseMessage] = [] + self._staged: list[BaseMessage] | None = None + self._turn_count = 0 + + # L2/L3 chunk compression (M3). ``_turns`` records each committed + # turn's message span in ``_active``; a compression event advances + # ``_active_start`` past whole turns (never splitting tool-call pairs) + # and replaces the frozen chunk blocks wholesale. The underlying + # ``_active`` list stays append-only so scrub-edge indices stay valid. + self._turns: list[dict] = [] + self._chunked_turn_count = 0 + self._active_start = 0 + self._frozen_blocks: list[BaseMessage] = [] + self._chunker: Any | None = None + + # id(message) -> summary-job key (DataEngine step id). A side map keeps + # the key out of the serialized message payload entirely. + self._step_keys: dict[int, str] = {} + + self._compressor = ScrubEdgeCompressor( + summarizer=step_memory, + prune_history_xml=prune_history_xml, + image_scrub_depth=image_scrub_depth, + pending_grace_steps=pending_grace_steps, + xml_scrub_depth=xml_scrub_depth, + summary_key_getter=lambda msg: self._step_keys.get(id(msg)), + strip_markers=(PRO_UI_LIST_MARKER, PLAN_RECITATION_MARKER), + tail_offset=1, + ) + + # ------------------------------------------------------------------ + # Session clock + # ------------------------------------------------------------------ + + def elapsed_seconds(self) -> float: + return max(0.0, self._clock() - self._session_start) + + def elapsed_label(self) -> str: + """The current session offset as ``T+mm:ss``.""" + return format_session_offset(self.elapsed_seconds()) + + # ------------------------------------------------------------------ + # S region + # ------------------------------------------------------------------ + + @property + def has_static_prefix(self) -> bool: + return bool(self._static) + + def set_static_prefix(self, messages: list[BaseMessage]) -> None: + """Install the byte-stable system prefix; only settable once.""" + if self._static: + raise RuntimeError("TranscriptLedger static prefix is already set.") + self._static = list(messages) + + # ------------------------------------------------------------------ + # F region (M2: cold-start restored history only) + # ------------------------------------------------------------------ + + @property + def has_restored_history(self) -> bool: + return bool(self._restored) + + def set_restored_history(self, text: str) -> None: + """Install the cold-start frozen history block (empty ledger only).""" + if self._restored: + raise RuntimeError("TranscriptLedger restored history is already set.") + if self._active or self._staged or self._turn_count: + raise RuntimeError( + "Restored history can only seed an empty ledger (cold start)." + ) + self._restored = [ + HumanMessage(content=[{"type": "text", "text": text}]) + ] + + # ------------------------------------------------------------------ + # A region (append-only turn commits) + # ------------------------------------------------------------------ + + @property + def turn_count(self) -> int: + return self._turn_count + + @property + def has_staged_turn(self) -> bool: + return self._staged is not None + + @property + def active_messages(self) -> tuple[BaseMessage, ...]: + return tuple(self._active) + + def stage_turn(self, messages: list[BaseMessage]) -> None: + """Hold a finished turn's messages until the next build commits them. + + Committing is deferred because the turn's DataEngine step id and its + validator result only exist after the operator returns. + """ + if self._staged is not None: + logger.warning( + "TranscriptLedger: previous staged turn was never committed;" + " committing it without step metadata." + ) + self.commit_staged() + self._staged = [m for m in messages if isinstance(m, BaseMessage)] + + def commit_staged( + self, + *, + step_key: str | None = None, + validator_result: Any | None = None, + ) -> None: + """Move the staged turn into the active region. + + Args: + step_key: The DataEngine step id recorded for this turn; keys the + turn's observation screenshot to its visual-transition summary + job for the depth-K scrub. + validator_result: The turn's validator report (``None`` when the + turn executed no terminal action); appended as a frozen result + message carrying the ``T+mm:ss`` session offset. + """ + if self._staged is None: + return + staged = self._staged + self._staged = None + + if step_key is not None: + observation = self._first_image_message(staged) + if observation is not None: + self._step_keys[id(observation)] = str(step_key) + + span_start = len(self._active) + self._active.extend(staged) + + if validator_result is not None: + result_message = self._build_result_message(validator_result) + if result_message is not None: + self._active.append(result_message) + + self._turns.append( + { + "step_key": str(step_key) if step_key is not None else None, + "start": span_start, + "end": len(self._active), + } + ) + self._turn_count += 1 + + @staticmethod + def _first_image_message(messages: list[BaseMessage]) -> BaseMessage | None: + for msg in messages: + content = getattr(msg, "content", None) + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") in ("image_url", "image"): + return msg + return None + + def _build_result_message(self, result: Any) -> HumanMessage | None: + rendered = None + if isinstance(result, dict): + status = result.get("status") or "unknown" + detail = None + try: + from artemis.utils.task_tree import format_result_clean + + # format_result_clean reports only errors/repairs; success + # renders as the bare status line. + detail = format_result_clean(result) + except Exception: + detail = None + rendered = f"Status: {status}" + (f"\n{detail}" if detail else "") + else: + try: + rendered = json.dumps(result, ensure_ascii=False, default=str) + except Exception: + rendered = str(result) + if not rendered: + return None + return HumanMessage( + content=[ + { + "type": "text", + "text": ( + f"{EXECUTION_RESULT_MARKER} ({self.elapsed_label()}) ---\n{rendered}" + ), + } + ] + ) + + # ------------------------------------------------------------------ + # F region (M3): chunk compression primitives + # ------------------------------------------------------------------ + + def attach_chunker(self, chunker: Any) -> None: + """Install the L2/L3 :class:`HistoryChunkManager` consulted at render.""" + self._chunker = chunker + + @property + def chunker(self) -> Any | None: + return self._chunker + + @property + def frozen_blocks(self) -> tuple[BaseMessage, ...]: + return tuple(self._frozen_blocks) + + def unchunked_turns(self) -> list[dict]: + """Committed turns not yet consumed by a compression event (copies).""" + return [dict(t) for t in self._turns[self._chunked_turn_count :]] + + def turn_text_chars(self, turns: list[dict]) -> int: + """Total text characters of the given turns' messages (size trigger).""" + total = 0 + for turn in turns: + for msg in self._active[turn["start"] : turn["end"]]: + content = getattr(msg, "content", None) + if isinstance(content, str): + total += len(content) + elif isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + total += len(block.get("text", "")) + return total + + def freeze_turns(self, turn_count: int, frozen_blocks: list[BaseMessage]) -> None: + """Compression event: consume the oldest ``turn_count`` unchunked turns + and replace the frozen chunk blocks wholesale. + + The boundary always advances to a committed turn's end, so a tool-call + message and its responses are never split across the F/A boundary. + """ + if turn_count <= 0: + self._frozen_blocks = list(frozen_blocks) + return + available = self._turns[self._chunked_turn_count :] + if turn_count > len(available): + raise ValueError( + f"freeze_turns({turn_count}) exceeds the {len(available)}" + " unchunked committed turns." + ) + self._chunked_turn_count += turn_count + self._active_start = self._turns[self._chunked_turn_count - 1]["end"] + self._frozen_blocks = list(frozen_blocks) + + # ------------------------------------------------------------------ + # Rendering + # ------------------------------------------------------------------ + + def render(self, tail: list[BaseMessage]) -> list[BaseMessage]: + """Advance compression and return ``S + F + A + tail``. + + Order per turn: the chunker runs first (a triggered compression event + deep-mutates the frozen region and advances the F/A boundary), then + the scrub edge advances over the active window. The returned list is a + fresh container: appending to it (the operator's in-turn tool loop) + never mutates the ledger regions. + """ + if self._chunker is not None: + try: + self._chunker.on_render(self) + except Exception as e: + logger.error(f"History chunker render hook failed: {e}") + self._compressor.compress(self._active) + return [ + *self._static, + *self._restored, + *self._frozen_blocks, + *self._active[self._active_start :], + *tail, + ] diff --git a/artemis/services/llm.py b/artemis/services/llm.py index 547552d..59664c3 100644 --- a/artemis/services/llm.py +++ b/artemis/services/llm.py @@ -56,6 +56,7 @@ from artemis.llm.reliability import ( retry_policy_for, ) from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider +from artemis.services.token_meter import record_llm_usage from artemis.llm.structured import ( ParseFailure, StructuredOutputError, @@ -632,13 +633,22 @@ class RobustChatModelWrapper: args, kwargs = _inject_parent_trace_id(trace_id, *args, **kwargs) return args, kwargs, trace_id + def _meter_usage(self, response) -> None: + """Record measured token usage for this call (M0 metering, best-effort).""" + engine = self.ctx.data_engine if self.ctx else None + if engine is None: + engine = _get_current_data_engine() + record_llm_usage(engine, response, source=self._endpoint_key()) + async def ainvoke(self, *args, **kwargs): args, kwargs, _ = self._traced_call(args, kwargs) - return await _run_with_recovery( + response = await _run_with_recovery( functools.partial(self.base_model.ainvoke, *args, **kwargs), provider=self._provider_value(), endpoint_key=self._endpoint_key(), ) + self._meter_usage(response) + return response async def complete(self, *args, **kwargs): """Single completion entry point: returns the full final message. @@ -652,11 +662,13 @@ class RobustChatModelWrapper: """ args, kwargs, _ = self._traced_call(args, kwargs) emit_deltas = bool(self.ctx and self.ctx.data_engine) - return await _run_with_recovery( + response = await _run_with_recovery( functools.partial(self._complete_attempt, emit_deltas, *args, **kwargs), provider=self._provider_value(), endpoint_key=self._endpoint_key(), ) + self._meter_usage(response) + return response def _emit_stream_delta(self, stream_exec_id, chunk) -> None: content = getattr(chunk, "content", None) diff --git a/artemis/services/token_meter.py b/artemis/services/token_meter.py new file mode 100644 index 0000000..2f8f364 --- /dev/null +++ b/artemis/services/token_meter.py @@ -0,0 +1,202 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Session-level LLM token metering (M0: pure recording, no decisions). + +Every gateway-managed LLM call reports its real ``usage_metadata`` here. The +meter accumulates per-session totals and records one ``llm_usage`` trace per +call into the DataEngine, carrying: + +- the call's measured ``prompt_tokens`` — the last call's measured input is the + best available estimate of the *current context base* for the next call + (the compaction thresholds of M2/M3 will consume this, never char-count + heuristics); +- provider-reported cache hits (Gemini ``cached_content_token_count`` / + LangChain ``input_token_details.cache_read``) to establish the cache-hit-rate + baseline; +- running per-session totals. + +This module never raises into the LLM call path and makes no decisions. +""" + +import threading +from typing import Any + +from artemis.utils.logger import get_logger + +logger = get_logger(__name__) + +_LOCK = threading.Lock() +_METERS: dict[str, "SessionTokenMeter"] = {} + + +def extract_usage(response: Any) -> dict[str, int] | None: + """Extracts unified token usage from an LLM response message, best-effort. + + Prefers LangChain's normalized ``usage_metadata``; falls back to raw + provider shapes in ``response_metadata``. Returns None when the response + carries no usable usage numbers. + """ + usage: dict[str, Any] | None = None + raw = getattr(response, "usage_metadata", None) + if isinstance(raw, dict) and raw: + usage = raw + else: + meta = getattr(response, "response_metadata", None) + if isinstance(meta, dict): + candidate = meta.get("usage_metadata") or meta.get("token_usage") + if isinstance(candidate, dict) and candidate: + usage = candidate + if usage is None: + return None + + def _int(value: Any) -> int: + try: + return int(value) + except (TypeError, ValueError): + return 0 + + prompt = _int( + usage.get("input_tokens") or usage.get("prompt_tokens") or usage.get("prompt_token_count") + ) + completion = _int( + usage.get("output_tokens") + or usage.get("completion_tokens") + or usage.get("candidates_token_count") + ) + total = _int(usage.get("total_tokens") or usage.get("total_token_count")) or ( + prompt + completion + ) + + cached = 0 + details = usage.get("input_token_details") + if isinstance(details, dict): + cached = _int(details.get("cache_read")) + if not cached: + cached = _int( + usage.get("cached_content_token_count") + or usage.get("cachedContentTokenCount") + or usage.get("cached_tokens") + ) + + if total <= 0: + return None + return { + "prompt_tokens": prompt, + "completion_tokens": completion, + "total_tokens": total, + "cached_tokens": cached, + } + + +class SessionTokenMeter: + """Accumulates measured LLM usage for one session.""" + + def __init__(self, session_id: str): + self.session_id = session_id + self._lock = threading.Lock() + self.llm_calls = 0 + self.prompt_tokens = 0 + self.completion_tokens = 0 + self.cached_tokens = 0 + self.cache_hit_calls = 0 + self.last_prompt_tokens = 0 + + def record( + self, usage: dict[str, int], *, update_last_prompt: bool = True + ) -> dict[str, int]: + """Accumulates one call's usage; returns a session snapshot. + + ``update_last_prompt=False`` accumulates the totals without touching + ``last_prompt_tokens`` — background lens calls carry tiny prompts that + must not masquerade as the live context base consumed by the L2/L3 + compaction thresholds. + """ + with self._lock: + self.llm_calls += 1 + self.prompt_tokens += usage.get("prompt_tokens", 0) + self.completion_tokens += usage.get("completion_tokens", 0) + cached = usage.get("cached_tokens", 0) + self.cached_tokens += cached + if cached > 0: + self.cache_hit_calls += 1 + if update_last_prompt: + self.last_prompt_tokens = usage.get("prompt_tokens", 0) + return self.snapshot() + + def snapshot(self) -> dict[str, int]: + return { + "session_llm_calls": self.llm_calls, + "session_prompt_tokens": self.prompt_tokens, + "session_completion_tokens": self.completion_tokens, + "session_cached_tokens": self.cached_tokens, + "session_cache_hit_calls": self.cache_hit_calls, + } + + +def get_meter(session_id: Any) -> SessionTokenMeter: + """Returns (creating on first use) the meter for a session id.""" + key = str(session_id) + with _LOCK: + meter = _METERS.get(key) + if meter is None: + meter = SessionTokenMeter(key) + _METERS[key] = meter + return meter + + +def record_llm_usage( + engine: Any, + response: Any, + *, + source: str | None = None, + update_last_prompt: bool = True, +) -> dict | None: + """Meters one LLM response and records an ``llm_usage`` trace, best-effort. + + ``context_base_tokens`` in the payload is this call's measured prompt size — + the running estimate of the live context for threshold decisions in later + milestones. Returns the recorded payload, or None when nothing was + recorded (no engine/session/usage). Never raises. + """ + try: + if engine is None: + return None + session_id = getattr(engine, "current_session_id", None) + usage = extract_usage(response) + if not session_id or usage is None: + return None + + snapshot = get_meter(session_id).record( + usage, update_last_prompt=update_last_prompt + ) + payload: dict[str, Any] = { + **usage, + "context_base_tokens": usage["prompt_tokens"], + **snapshot, + } + if source: + payload["source"] = source + + engine.record_trace( + type="llm_call", + name="llm_usage", + payload=payload, + step_id=getattr(engine, "current_step_id", None), + status="success", + ) + return payload + except Exception as meter_err: + logger.debug(f"Token metering skipped: {meter_err}") + return None diff --git a/artemis/tools/committee_tool.py b/artemis/tools/committee_tool.py index fb73b61..a964f72 100644 --- a/artemis/tools/committee_tool.py +++ b/artemis/tools/committee_tool.py @@ -48,8 +48,8 @@ from artemis.tools.types import CyFunctionDetector from artemis.tools.video_tool import get_video_analyzer_tool from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path, get_notes_dir +from artemis.memory.context_policy import build_history_for from artemis.utils.task_tree import ( - build_plan_and_history, get_active_subgoal_hashes, get_recent_subgoal_hashes, ) @@ -186,12 +186,13 @@ async def _execute_committee( logger.error(f"Failed to parse active subgoal in committee: {e}") try: - plan_and_history = build_plan_and_history( + plan_and_history = build_history_for( + "committee", task_plan, steps, active_subgoal_hash, keep_subgoal_hashes=keep_hashes, - last_n_detailed=3, + engine=ctx.data_engine, ) history_summary += plan_and_history except Exception as e: # pylint: disable=broad-exception-caught diff --git a/artemis/tools/history_recall.py b/artemis/tools/history_recall.py new file mode 100644 index 0000000..40ca8bf --- /dev/null +++ b/artemis/tools/history_recall.py @@ -0,0 +1,564 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""recall_history: deterministic lossless lookup into cold history (M4). + +Implements the sister design's §10 as adopted by the history-module redesign +§3.5: one unified tool over the DataEngine's raw records — step summaries, +history chunks (bands ①②③), exact actions and validator results, operator +thinking, notes, XML/OCR text (which also carries package/activity strings), +Failure-Analyzer traces, and image references. Phase one is keyword + step +range filtering over SQLite-backed records in Python — no vector database, no +model call. + +Hard boundaries (config ``agent.memory.recall``): + +- at most ``max_results`` results (default 5); +- one response is capped at ``max_text_tokens`` estimated tokens (char/4); +- every result carries a step number / step id; +- ``include_details`` output is clamped per field; +- ``include_images`` returns only the screenshots that actually exist on disk + for a single step (data URLs), never more than ``max_image_steps`` steps; +- large raw sources return only excerpts around the match plus a reference. + +An era whose ledger overflowed to a ``[Era N | Steps a–b: ledger via +recall_history]`` marker line is re-entered here: recalling that step range +returns the full-width ``build_action_ledger`` rows for it. +""" + +import base64 +import json +from typing import Any + +from pydantic import BaseModel, Field + +from artemis.data_engine.trace import trace_langchain_tool +from artemis.tools.base import ArtemisTool, ToolRegistry +from artemis.tools.tool_wrapper import ToolWrapper +from artemis.utils.logger import get_logger + +logger = get_logger(__name__) + +#: How many most-recent steps get their screenshot XML/OCR blobs scanned +#: (per-image JSON loads are the expensive part of the sweep). +XML_SCAN_CAP = 150 + +#: Character clamp for one matched excerpt. +EXCERPT_CHARS = 240 +DETAIL_EXCERPT_CHARS = 600 + + +class RecallHistoryArgs(BaseModel): + """Arguments schema for the recall_history tool.""" + + query: str = Field( + ..., + description=( + "Keywords to search for across the whole execution history" + " (summaries, actions, validator results, reasoning, notes," + " on-screen XML/OCR text, package/activity names). Case-" + "insensitive; every whitespace-separated term is matched" + " independently. May be empty when step_range is given (returns" + " the per-step action ledger of that range)." + ), + ) + step_range: list[int] | None = Field( + default=None, + description=( + "Optional [start, end] step-number range (inclusive) to restrict" + " the search — e.g. the range shown in a '[... ledger via" + " recall_history]' marker line. Also returns the full per-step" + " action ledger of the range." + ), + ) + include_details: bool = Field( + default=False, + description=( + "Include clamped per-result details (exact action, validator" + " result, reasoning excerpt) instead of the one-line form." + ), + ) + include_images: bool = Field( + default=False, + description=( + "Return the stored screenshots of the single best-matching step" + " (only images that actually exist; at most one step)." + ), + ) + max_results: int = Field( + default=5, + ge=1, + description="Maximum results to return (server-side cap still applies).", + ) + + +def _terms(query: str) -> list[str]: + return [t for t in (query or "").lower().split() if t] + + +def _estimate_tokens(text: str) -> int: + return len(text) // 4 + + +def _clamp(text: str, limit: int) -> str: + text = (text or "").strip() + if len(text) <= limit: + return text + return text[: limit - 1].rstrip() + "…" + + +def _excerpt(text: str, terms: list[str], limit: int) -> str: + """A window of ``text`` around the first term hit (flattened whitespace).""" + flat = " ".join((text or "").split()) + lower = flat.lower() + pos = -1 + for term in terms: + pos = lower.find(term) + if pos >= 0: + break + if pos < 0: + return _clamp(flat, limit) + start = max(0, pos - limit // 3) + window = flat[start : start + limit] + prefix = "…" if start > 0 else "" + suffix = "…" if start + limit < len(flat) else "" + return f"{prefix}{window.strip()}{suffix}" + + +def _score(haystack: str, terms: list[str]) -> int: + lower = haystack.lower() + return sum(1 for term in terms if term in lower) + + +def _events_text(step: dict) -> str: + parts: list[str] = [] + for event in step.get("interleaved_events") or []: + etype = event.get("type") + if etype == "tool_call": + parts.append(str(event.get("name") or "")) + parts.append(json.dumps(event.get("args") or {}, ensure_ascii=False, default=str)) + result = event.get("result") + if result is not None: + parts.append(str(result)[:2000]) + elif etype and "thought" in etype: + parts.append(str(event.get("content") or "")) + return "\n".join(parts) + + +def _step_haystack(step: dict) -> str: + fields = [ + str(step.get("summary") or ""), + json.dumps(step.get("action_taken"), ensure_ascii=False, default=str) + if step.get("action_taken") + else "", + json.dumps(step.get("last_execution_result"), ensure_ascii=False, default=str) + if step.get("last_execution_result") + else "", + str(step.get("operator_raw_thinking") or ""), + _events_text(step), + str((step.get("extra_metadata") or {}).get("injected_instruction") or ""), + # M5: the foreground app package stamped by record_step joins the + # search surface, so package-name queries no longer depend on the + # XML-blob scan (capped to recent steps) alone. + str((step.get("extra_metadata") or {}).get("foreground_app") or ""), + ] + return "\n".join(f for f in fields if f) + + +def _screen_text(engine: Any, step: dict) -> str: + """Raw XML/OCR text of a step's pre-action screenshot (package/activity + strings live inside the XML blob).""" + image_name = step.get("pre_image_name") or step.get("post_image_name") + if not image_name: + return "" + try: + record = engine.storage.get_image(image_name) + except Exception: + return "" + if record is None: + return "" + parts = [] + for blob in (record.ui_tree, record.ocr_result): + if blob: + parts.append(blob if isinstance(blob, str) else json.dumps(blob, ensure_ascii=False, default=str)) + return "\n".join(parts) + + +def _session_start(engine: Any) -> float | None: + return getattr(engine, "session_start_time", None) + + +def _step_label(step: dict, session_start: float | None) -> str: + from artemis.memory.chunking import step_offset_label + + return f"Step {step.get('step_number')} ({step_offset_label(step, session_start)})" + + +def _note_writer_steps(steps: list[dict], key: str) -> list[int]: + writers: list[int] = [] + for step in steps: + for event in step.get("interleaved_events") or []: + if ( + event.get("type") == "tool_call" + and event.get("name") in ("save_note", "update_note", "append_note") + and (event.get("args") or {}).get("key") == key + ): + number = step.get("step_number") + if isinstance(number, int): + writers.append(number) + return writers + + +def search_history( + engine: Any, + *, + query: str, + step_range: list[int] | None = None, + include_details: bool = False, + include_images: bool = False, + max_results: int = 5, + recall_config: Any = None, +) -> str | list[dict]: + """Deterministic history search; returns text or content blocks (images).""" + cfg_max_results = int(getattr(recall_config, "max_results", 5) or 5) + cfg_max_tokens = int(getattr(recall_config, "max_text_tokens", 2000) or 2000) + cfg_max_image_steps = int(getattr(recall_config, "max_image_steps", 1)) + + max_results = max(1, min(int(max_results or cfg_max_results), cfg_max_results)) + char_budget = cfg_max_tokens * 4 + + try: + steps: list[dict] = engine.get_agent_friendly_steps() or [] + except Exception as e: + return f"recall_history failed to load history: {e}" + + range_start = range_end = None + if step_range: + try: + range_start, range_end = int(step_range[0]), int(step_range[-1]) + if range_start > range_end: + range_start, range_end = range_end, range_start + except (TypeError, ValueError, IndexError): + range_start = range_end = None + + def in_range(number: Any) -> bool: + if range_start is None: + return True + return isinstance(number, int) and range_start <= number <= range_end + + scoped_steps = [s for s in steps if in_range(s.get("step_number"))] + terms = _terms(query) + session_start = _session_start(engine) + + if not steps: + return "recall_history: no recorded steps yet." + if step_range and not scoped_steps: + return ( + f"recall_history: no recorded steps in range" + f" {range_start}–{range_end}." + ) + + sections: list[str] = [] + + # --- Range ledger: the explicit re-entry point for 'ledger via + # recall_history' guidance lines and recall-only era period paragraphs — + # full-width build_action_ledger rows. + if range_start is not None and scoped_steps: + from artemis.memory.chunking import build_action_ledger + + ledger = build_action_ledger(scoped_steps, session_start) + sections.append( + f"Action ledger for Steps {range_start}–{range_end}" + f" ({len(scoped_steps)} recorded steps):\n{ledger}" + ) + + # --- Keyword search over steps. + results: list[tuple[int, int, str]] = [] # (score, step_number, rendered) + if terms: + xml_eligible = { + id(s) for s in scoped_steps[-XML_SCAN_CAP:] + } + for step in scoped_steps: + haystack = _step_haystack(step) + screen = _screen_text(engine, step) if id(step) in xml_eligible else "" + score = _score(haystack, terms) + screen_score = _score(screen, terms) if screen else 0 + total = score + screen_score + if total <= 0: + continue + excerpt_limit = DETAIL_EXCERPT_CHARS if include_details else EXCERPT_CHARS + source = haystack if score >= screen_score else screen + body = _excerpt(source, terms, excerpt_limit) + lines = [f"[{_step_label(step, session_start)} | id {step.get('step_id')}]"] + summary = step.get("summary") + if summary: + lines.append(f" Summary: {_clamp(str(summary), EXCERPT_CHARS)}") + lines.append(f" Match: {body}") + if include_details: + action = step.get("action_taken") + if action: + lines.append( + " Action: " + + _clamp(json.dumps(action, ensure_ascii=False, default=str), DETAIL_EXCERPT_CHARS) + ) + result = step.get("last_execution_result") + if result: + lines.append( + " Result: " + + _clamp(json.dumps(result, ensure_ascii=False, default=str), DETAIL_EXCERPT_CHARS) + ) + thinking = step.get("operator_raw_thinking") + if thinking: + lines.append(f" Reasoning: {_clamp(str(thinking), DETAIL_EXCERPT_CHARS)}") + number = step.get("step_number") or 0 + results.append((total, number, "\n".join(lines))) + + # --- History chunks (bands ①②③ / rendered text). + try: + chunk_rows = engine.get_history_chunks() or [] + except Exception: + chunk_rows = [] + for row in chunk_rows: + start_n = getattr(row, "start_step_number", None) + end_n = getattr(row, "end_step_number", None) + if range_start is not None and not ( + isinstance(start_n, int) + and isinstance(end_n, int) + and start_n <= range_end + and end_n >= range_start + ): + continue + text = getattr(row, "rendered_text", None) or "\n".join( + filter(None, [getattr(row, "band2", None), getattr(row, "band3", None)]) + ) + score = _score(text or "", terms) + if score <= 0: + continue + body = _excerpt(text, terms, DETAIL_EXCERPT_CHARS if include_details else EXCERPT_CHARS) + rendered = ( + f"[History chunk | Steps {start_n}–{end_n} | status" + f" {getattr(row, 'status', '?')}]\n Match: {body}" + ) + results.append((score, int(end_n or 0), rendered)) + + # --- Notes (filesystem; attributed to their writing steps). + try: + from artemis.utils.notes import list_notes_keys, read_note_content + + base_dir = getattr(engine, "base_dir", None) + note_keys = list_notes_keys(base_dir) if base_dir else [] + except Exception: + note_keys = [] + for key in note_keys: + try: + content = read_note_content(base_dir, key) + except Exception: + continue + score = _score(content, terms) + _score(key, terms) + if score <= 0: + continue + writers = _note_writer_steps(steps, key) + if range_start is not None and writers and not any( + range_start <= n <= range_end for n in writers + ): + continue + anchor = writers[-1] if writers else (steps[-1].get("step_number") or 0) + origin = ( + f"last written at Step {writers[-1]}" + if writers + else f"as of Step {anchor}" + ) + body = _excerpt(content, terms, DETAIL_EXCERPT_CHARS if include_details else EXCERPT_CHARS) + results.append( + (score, int(anchor), f"[Note '{key}' | {origin}]\n Match: {body}") + ) + + results.sort(key=lambda r: (-r[0], -r[1])) + kept = results[:max_results] + dropped = len(results) - len(kept) + + if kept: + header = f"Top {len(kept)} match(es) for {terms}:" + if dropped > 0: + header += f" ({dropped} more not shown; narrow the query or step_range)" + sections.append(header + "\n\n" + "\n".join(r[2] for r in kept)) + elif terms: + sections.append(f"No matches for {terms} in the searched history.") + + text = "\n\n".join(sections) + if _estimate_tokens(text) > cfg_max_tokens: + text = ( + text[:char_budget].rstrip() + + "\n… (response truncated at the recall token budget; narrow the" + " query or step_range)" + ) + + if not include_images or cfg_max_image_steps < 1: + return text + + # --- Image recall: real stored screenshots of ONE step only. + target_step: dict | None = None + if kept: + # The best step-shaped result; fall back to the newest scoped step. + best_numbers = [r[1] for r in kept] + for number in best_numbers: + target_step = next( + (s for s in scoped_steps if s.get("step_number") == number), None + ) + if target_step: + break + if target_step is None and scoped_steps: + target_step = scoped_steps[-1] + if target_step is None: + return text + + blocks: list[dict] = [{"type": "text", "text": text}] + added = 0 + for which, name in ( + ("pre-action", target_step.get("pre_image_name")), + ("post-action", target_step.get("post_image_name")), + ): + if not name: + continue + try: + path = engine.get_image_path(name) + if not path or not path.exists(): + continue + encoded = base64.b64encode(path.read_bytes()).decode("utf-8") + except Exception: + continue + blocks.append( + { + "type": "text", + "text": ( + f"--- {which} screenshot of Step" + f" {target_step.get('step_number')} ---" + ), + } + ) + blocks.append( + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{encoded}"}, + } + ) + added += 1 + if added == 0: + blocks.append( + { + "type": "text", + "text": ( + f"(no stored screenshots exist for Step" + f" {target_step.get('step_number')})" + ), + } + ) + return blocks + + +RECALL_HISTORY_DOCSTRING = ( + "[SHELL] Deterministic lookup into older execution history that has been" + " compressed out of the visible context. Searches step summaries, exact" + " actions and validator results, reasoning, notes, on-screen XML/OCR text" + " and history-chunk ledgers by keywords and/or a step-number range;" + " a step_range also returns that range's full per-step action ledger" + " (the entry point for 'ledger via recall_history' guidance/marker lines" + " and for recall-only '[Era ...]' period paragraphs)." + " Results are bounded and every result carries its step number." + " Do not call it speculatively on ordinary steps." +) + + +class RecallHistoryTool(ArtemisTool): + """Unified cold-history recall over the DataEngine (design §10).""" + + def __init__(self): + super().__init__( + name="recall_history", + description=RECALL_HISTORY_DOCSTRING, + args_schema=RecallHistoryArgs, + category="memory", + ) + + def is_available(self, ctx: Any = None) -> bool: + if ctx is None or getattr(ctx, "data_engine", None) is None: + return False + return bool(getattr(_recall_config(warn=False), "enabled", True)) + + # pylint: disable=too-many-arguments + async def execute( + self, + driver: Any = None, # pylint: disable=unused-argument + ctx: Any = None, + query: str = "", + step_range: list[int] | None = None, + include_details: bool = False, + include_images: bool = False, + max_results: int = 5, + **kwargs: Any, + ) -> Any: + engine = getattr(ctx, "data_engine", None) if ctx else None + if engine is None: + return "recall_history unavailable: no active DataEngine session." + if not query and not step_range: + return ( + "recall_history needs a query and/or a step_range — e.g." + ' recall_history(query="login timeout") or' + " recall_history(query=\"\", step_range=[1, 40])." + ) + try: + return search_history( + engine, + query=query or "", + step_range=step_range, + include_details=bool(include_details), + include_images=bool(include_images), + max_results=int(max_results or 5), + recall_config=_recall_config(), + ) + except Exception as e: + logger.error(f"recall_history failed: {e}") + return f"recall_history failed: {e}" + + +def _recall_config(warn: bool = True) -> Any: + try: + from artemis.config import load_agent_config + + return load_agent_config().memory.recall + except Exception as e: + if warn: + logger.debug(f"recall config unavailable, using defaults: {e}") + return None + + +recall_history = RecallHistoryTool() +ToolRegistry.register(recall_history) + + +def get_recall_history_tool(ctx: Any): + """Exports recall_history as a traced LangChain BaseTool.""" + return trace_langchain_tool(recall_history.to_langchain_tool(ctx), ctx) + + +def _recall_available(ctx: Any) -> bool: + return recall_history.is_available(ctx) + + +recall_history_wrapper = ToolWrapper( + tool_fn_getter=get_recall_history_tool, + on_success_fn=lambda *a, **k: "Recalled history", + on_failure_fn=lambda err: f"recall_history failed: {err}", + is_available_fn=_recall_available, +) diff --git a/artemis/utils/element_hit_test.py b/artemis/utils/element_hit_test.py new file mode 100644 index 0000000..c167b61 --- /dev/null +++ b/artemis/utils/element_hit_test.py @@ -0,0 +1,123 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Record-time action semantics enrichment via element hit testing. + +An action addressed by bare coordinates carries no element semantics +("tap(632,1180)" says nothing about what was tapped). Before such an action is +persisted to the DataEngine, these helpers resolve the smallest indexed element +covering the tap point on the pre-action frame and attach best-effort +``target_text`` / ``target_class`` / ``target_resource_id`` fields, plus a +``target_label_source`` marker distinguishing model-named targets from +after-the-fact inference: + +- ``"index"``: the model addressed the element by its perception index — the + semantics come straight from the indexed element (highest confidence). +- ``"hit_test"``: inferred by hit testing the coordinates against XML-derived + indexed elements on the pre-action frame. +- ``"ocr"``: inferred by hit testing against OCR-derived indexed elements + (used only when no XML-derived element covers the point). +- ``"none"``: no element covers the point (or no perception data was + available) — the action stays a bare-coordinate action. + +Enrichment is strictly best-effort: missing/malformed element data degrades to +``"none"`` and never raises. +""" + +from typing import Any + +from artemis.utils.visualization import parse_bounds + +_LABEL_SOURCE_KEY = "target_label_source" + + +def _element_bounds(element: dict[str, Any]) -> tuple[int, int, int, int] | None: + """Returns (left, top, right, bottom) pixel bounds of an indexed element. + + Accepts both list bounds (``[l, t, r, b]``, the shape produced by + ``format_minimal_list_with_elements``) and Android bounds strings + (``"[l,t][r,b]"``). Returns None for missing or malformed bounds. + """ + bounds = element.get("bounds") + if isinstance(bounds, str): + return parse_bounds(bounds) + if isinstance(bounds, (list, tuple)) and len(bounds) == 4: + try: + left, top, right, bottom = (int(v) for v in bounds) + except (TypeError, ValueError): + return None + return left, top, right, bottom + return None + + +def find_element_at_point( + elements: list[dict[str, Any]] | None, x: int, y: int +) -> tuple[dict[str, Any] | None, str]: + """Finds the smallest indexed element whose bounds contain pixel point (x, y). + + XML-derived elements are preferred over OCR-derived ones (OCR boxes carry + text but weaker structure); within each group the smallest covering area + wins. Returns ``(element, source)`` where source is ``"hit_test"``, + ``"ocr"``, or ``"none"`` (with element None). + """ + best_xml: tuple[int, dict[str, Any]] | None = None + best_ocr: tuple[int, dict[str, Any]] | None = None + + for element in elements or []: + if not isinstance(element, dict): + continue + rect = _element_bounds(element) + if rect is None: + continue + left, top, right, bottom = rect + if not (left <= x <= right and top <= y <= bottom): + continue + area = max(1, (right - left)) * max(1, (bottom - top)) + if element.get("is_ocr"): + if best_ocr is None or area < best_ocr[0]: + best_ocr = (area, element) + else: + if best_xml is None or area < best_xml[0]: + best_xml = (area, element) + + if best_xml is not None: + return best_xml[1], "hit_test" + if best_ocr is not None: + return best_ocr[1], "ocr" + return None, "none" + + +def hit_test_semantics( + elements: list[dict[str, Any]] | None, x: int, y: int +) -> dict[str, Any]: + """Best-effort semantic fields for a bare-coordinate action at pixel (x, y). + + Always returns a dict containing ``target_label_source``; the + ``target_text`` / ``target_class`` / ``target_resource_id`` fields are only + present when the covering element carries them. + """ + try: + element, source = find_element_at_point(elements, int(x), int(y)) + except Exception: + return {_LABEL_SOURCE_KEY: "none"} + semantics: dict[str, Any] = {_LABEL_SOURCE_KEY: source} + if element is None: + return semantics + if element.get("text"): + semantics["target_text"] = element["text"] + if element.get("class"): + semantics["target_class"] = element["class"] + if element.get("resource_id"): + semantics["target_resource_id"] = element["resource_id"] + return semantics diff --git a/artemis/utils/image_hash.py b/artemis/utils/image_hash.py new file mode 100644 index 0000000..6abe8b4 --- /dev/null +++ b/artemis/utils/image_hash.py @@ -0,0 +1,81 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pure-PIL perceptual hashing (dHash) for screen-similarity hints (M4). + +A 64-bit difference hash: grayscale, resize to 9x8, compare each pixel to its +right neighbor. Robust to JPEG re-encoding and mild rendering noise, cheap to +compute (one small resize), and comparable with an integer Hamming distance — +no model call, no historical image bytes needed at compare time. + +Hashes are stored per step in ``extra_metadata`` at record time +(``pre_image_dhash`` / ``post_image_dhash``, 16 hex chars) and compared +against the current screenshot before prompt construction. +""" + +import io + +from artemis.utils.logger import get_logger + +logger = get_logger(__name__) + +#: Grid width of the dHash (the hash carries HASH_SIZE * HASH_SIZE bits). +HASH_SIZE = 8 + + +def dhash_hex(image_bytes: bytes | None, hash_size: int = HASH_SIZE) -> str | None: + """64-bit dHash of an encoded image, as a 16-char hex string. + + Best-effort: returns ``None`` on any failure (missing/corrupt bytes, + Pillow unavailable) — perceptual hashing is an enhancement, never a + dependency. + """ + if not image_bytes: + return None + try: + from PIL import Image + + with Image.open(io.BytesIO(image_bytes)) as img: + # JPEG fast-path: DCT-scaled draft decode keeps the sync cost of + # hashing at record time to a few milliseconds. The draft is only + # an intermediate resolution; the final resize below fixes the + # grid, so hashes stay comparable across code paths. + try: + img.draft("L", (hash_size * 8, hash_size * 8)) + except Exception: + pass + small = img.convert("L").resize( + (hash_size + 1, hash_size), Image.Resampling.LANCZOS + ) + pixels = list(small.getdata()) + bits = 0 + for row in range(hash_size): + for col in range(hash_size): + left = pixels[row * (hash_size + 1) + col] + right = pixels[row * (hash_size + 1) + col + 1] + bits = (bits << 1) | (1 if left > right else 0) + return f"{bits:0{hash_size * hash_size // 4}x}" + except Exception as e: + logger.debug(f"dhash_hex failed: {e}") + return None + + +def hamming_distance_hex(hash_a: str | None, hash_b: str | None) -> int | None: + """Bit-level Hamming distance between two hex hashes; None if incomparable.""" + if not hash_a or not hash_b or len(hash_a) != len(hash_b): + return None + try: + return bin(int(hash_a, 16) ^ int(hash_b, 16)).count("1") + except ValueError: + return None diff --git a/artemis/utils/task_tree.py b/artemis/utils/task_tree.py index 5d23df2..7a8bddc 100644 --- a/artemis/utils/task_tree.py +++ b/artemis/utils/task_tree.py @@ -801,14 +801,22 @@ def build_plan_and_history( keep_subgoal_hashes: set | None = None, min_summaries: int = 5, last_n_detailed: int = 1, - detailed_subgoal_hashes: set | None = None, all_detailed: bool = False, strict_milestone_pruning: bool = False, recent_window_size: int = 3, - chronological_last_step: bool = False, for_failure_analyzer: bool = False, + chunks: list | None = None, ) -> str: - """Builds a clean plan list and separate flat chronological execution history with adaptive compression.""" + """Builds a clean plan list and separate flat chronological execution history with adaptive compression. + + ``chunks`` (optional, M4): pre-rendered history-chunk blocks — each a dict + with ``start_step_number``/``end_step_number``/``text``. When provided, + the chunk blocks are emitted right after the history header and the + per-step lines inside any chunked range are dropped (the chunk block + replaces them), except steps that must render detailed (the recent + window / the most-recent step stay untouched). When ``chunks`` is + ``None`` or empty the output is byte-identical to the pre-M4 rendering. + """ # 1. Plan Checklist Section output_parts = [] @@ -900,6 +908,26 @@ def build_plan_and_history( # Ensure the last step is always in detailed_step_ids detailed_step_ids.add(full_info_step_id) + # M4: chunk blocks replace the per-step lines of already-chunked + # ranges. Chunks only ever cover frozen (old) turns, so they are + # emitted first, in chronological order; steps that must render + # detailed (recent window / most-recent step) are never suppressed. + chunked_ranges: list[tuple[int, int]] = [] + if chunks: + for chunk in sorted( + chunks, key=lambda c: c.get("start_step_number") or 0 + ): + text = chunk.get("text") + if not text: + continue + try: + start_n = int(chunk.get("start_step_number")) + end_n = int(chunk.get("end_step_number")) + except (TypeError, ValueError): + continue + chunked_ranges.append((start_n, end_n)) + output_parts.append(text) + for step in steps: step_id = step["step_id"] @@ -907,6 +935,13 @@ def build_plan_and_history( if step_id not in visible_step_ids: continue + if chunked_ranges and step_id not in detailed_step_ids and step_id != full_info_step_id: + step_number = step.get("step_number") + if isinstance(step_number, int) and any( + start_n <= step_number <= end_n for start_n, end_n in chunked_ranges + ): + continue # represented by its chunk block above + relative_time = step.get("relative_time") or "N/A" summary = step.get("summary") action = step.get("action_taken") diff --git a/config/artemis.jsonc b/config/artemis.jsonc index d2f368b..874fc73 100644 --- a/config/artemis.jsonc +++ b/config/artemis.jsonc @@ -190,6 +190,50 @@ // ========================================================================== // 🌐 4.3 Shared Capabilities & Global Overrides // ========================================================================== + // 🧠 Unified Step-Memory Runtime (history-module redesign v3, M0–M5) + // Transcript ledger + layered compression for the Pro operator. + "memory": { + // Background summarization runtime (visual/capsule lenses) + "runtime": { + "max_concurrency": 2, // 1→2 per 2026-09-01 baseline: serial queue pushed summary-ready P90 past the scrub window + "retry_limit": 3, + "flush_timeout_s": 30 + }, + "transcript": { + // ROLLBACK SWITCH: set false to restore the legacy per-turn 2-message + // prompt path byte-for-byte (L2/L3 chunk compression rides this same + // flag). Default on since M5 (2026-09-01) per the on-device A/B: + // turn latency −77% P50, cache-hit calls 43%→88%, output tokens −74%. + "enabled": true, + "context_budget_tokens": 80000, + "soft_ratio": 0.7, + "hard_ratio": 0.9, + "min_active_steps": 5, + "image_scrub_depth": 3, + "pending_grace_steps": 3, + "xml_scrub_depth": 1, + // Local historical-state hint (no model call); distance calibrated + // 2026-09-01 from 460 dHash-stamped on-device steps (same screen <=4, + // different screens >=7). + "similarity_hint": true, + "similarity_max_distance": 5 + }, + // Segment (L2) chunk compression: three-band chunks, era folding + "chunking": { + "max_steps": 12, + "target_source_tokens": 2000, + "model": "gemini-3.7-flash", + "max_chunks": 8 + // "max_eras": 8 // independent era cap; omitted = follows max_chunks + }, + // Deterministic cold-history recall tool (Pro operator) + "recall": { + "enabled": true, + "max_results": 5, + "max_text_tokens": 2000, + "max_image_steps": 1 + } + }, // 🎬 Screen Recording & Video Analyzer Configuration "video_analyzer": { "enabled": true, // Whether continuous screen recording via scrcpy is active (default: auto-detected) diff --git a/tests/unit/agents/test_flash_scrub_edge.py b/tests/unit/agents/test_flash_scrub_edge.py new file mode 100644 index 0000000..2b5e6e8 --- /dev/null +++ b/tests/unit/agents/test_flash_scrub_edge.py @@ -0,0 +1,325 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the Flash scrub-edge compressor (history redesign §3.2, M1). + +Covers the M1 acceptance items: the legacy-equivalent success-path product +(asserted directly since the legacy ``compress_flash_messages`` reference was +removed in M5), the three race regressions (grace recovery, grace exhaustion +without backfill, failed-summary placeholder), the freeze invariant (bytes +behind the scrub edge never change), and depth-1 XML stripping equivalence. +""" + +import json +from unittest.mock import Mock + +import pytest +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage + +from artemis.agents.flash.context_compressor import ScrubEdgeCompressor +from artemis.agents.flash.summarizer import VisualStepSummarizer +from artemis.context import ArtemisContext + + +@pytest.fixture +def mock_context(): + ctx = Mock(spec=ArtemisContext) + ctx.data_engine = None + return ctx + + +def _initial_observation(): + return HumanMessage( + content=[ + {"type": "text", "text": "Task: Search flights"}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_0"}}, + {"type": "text", "text": "--- UI Element List ---\nTree 0"}, + ] + ) + + +def _step(i: int) -> ToolMessage: + return ToolMessage( + tool_call_id=f"tc{i}", + name="click", + content=[ + {"type": "text", "text": f"Action {i} completed."}, + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG_{i}"}}, + {"type": "text", "text": f"--- UI Element List ---\nTree {i}"}, + ], + ) + + +def _simulate_turns(compressor, messages, new_messages): + """Mimic FlashRunner's ordering: compress at turn start, then append.""" + for msg in new_messages: + compressor.compress(messages) + messages.append(msg) + compressor.compress(messages) + + +def _content_fingerprint(msg) -> str: + return json.dumps(msg.content, sort_keys=True, default=str) + + +def _has_image(msg) -> bool: + return any( + isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content + ) + + +def test_success_path_product_behind_the_scrub_edge(mock_context): + """With summaries ready in time, every message behind the scrub edge is the + legacy-shaped product: action text kept, XML stripped, screenshot replaced + by the ``--- Historical Visual Transition ---`` summary block. + + (Asserted directly; the legacy ``compress_flash_messages`` reference this + was originally diffed against was removed in M5.) Intended M1 timing + change: the swap happens at depth K, so the newest K-1 historical steps + still carry their screenshots. + """ + summarizer = VisualStepSummarizer(mock_context) + for i in range(1, 7): + summarizer._summaries[f"tc{i}"] = f"Visual summary {i}." + + new_messages: list = [] + compressor = ScrubEdgeCompressor( + summarizer=summarizer, + prune_history_xml=True, + image_scrub_depth=3, + pending_grace_steps=3, + ) + _simulate_turns( + compressor, new_messages, [_initial_observation()] + [_step(i) for i in range(1, 7)] + ) + + # Initial observation behind the edge: image silently dropped (no summary + # job), task text kept, XML stripped. + assert new_messages[0].content == [{"type": "text", "text": "Task: Search flights"}] + + # Steps 1-4 behind the scrub edge: exact legacy-shaped swap product. + for i in range(1, 5): + assert new_messages[i].content == [ + {"type": "text", "text": f"Action {i} completed."}, + { + "type": "text", + "text": f"--- Historical Visual Transition ---\nVisual summary {i}.", + }, + ], f"message {i} diverged from the legacy-shaped product" + + # The live observation is intact. + assert _has_image(new_messages[6]) + assert "Tree 6" in str(new_messages[6].content) + + # Intended timing change: step 5 (depth 2 < K) keeps its screenshot under + # the scrub edge; its XML was still stripped at depth 1. + assert _has_image(new_messages[5]) + assert "Visual summary 5." not in str(new_messages[5].content) + assert "Tree 5" not in str(new_messages[5].content) + + +def test_race_pending_at_edge_recovers_within_grace(mock_context): + """A step whose summary is late keeps its image at the scrub edge and is + swapped to the summary once it arrives within the grace window.""" + summarizer = VisualStepSummarizer(mock_context) + summarizer._step_inputs["tc1"] = {"step_number": 1} # job pending + + messages: list = [] + compressor = ScrubEdgeCompressor( + summarizer=summarizer, + image_scrub_depth=3, + pending_grace_steps=3, + ) + _simulate_turns( + compressor, messages, [_initial_observation()] + [_step(i) for i in range(1, 4)] + ) + + # Step 1 sits at the scrub edge with a pending summary: image retained. + assert _has_image(messages[1]) + + # Summary arrives within grace; the next pass applies it. + summarizer._summaries["tc1"] = "Recovered visual summary 1." + _simulate_turns(compressor, messages, [_step(4)]) + + assert not _has_image(messages[1]) + assert { + "type": "text", + "text": "--- Historical Visual Transition ---\nRecovered visual summary 1.", + } in messages[1].content + + +def test_race_grace_exhausted_placeholder_never_backfilled(mock_context): + """After the grace window a pending step becomes a placeholder, and a late + summary never mutates the frozen message again.""" + summarizer = VisualStepSummarizer(mock_context) + summarizer._step_inputs["tc1"] = {"step_number": 1} # pending forever (for now) + + messages: list = [] + compressor = ScrubEdgeCompressor( + summarizer=summarizer, + image_scrub_depth=2, + pending_grace_steps=1, + ) + _simulate_turns(compressor, messages, [_step(i) for i in range(1, 5)]) + + # Depth 4 > K + grace = 3: grace exhausted, placeholder applied. + step1_text = str(messages[0].content) + assert not _has_image(messages[0]) + assert "[visual summary pending; evidence at DataEngine step 1]" in step1_text + frozen_fingerprint = _content_fingerprint(messages[0]) + + # The summary arriving later must NOT backfill the frozen message. + summarizer._summaries["tc1"] = "Too late summary." + _simulate_turns(compressor, messages, [_step(5)]) + compressor.compress(messages) + + assert _content_fingerprint(messages[0]) == frozen_fingerprint + assert "Too late summary." not in str(messages[0].content) + + +def test_race_failed_summary_becomes_unavailable_placeholder(mock_context): + """A failed summary is replaced by an unavailable placeholder at the scrub + edge without waiting out the grace window.""" + summarizer = VisualStepSummarizer(mock_context) + summarizer._step_inputs["tc1"] = {"step_number": 1} + summarizer._failed.add("tc1") + + messages: list = [] + compressor = ScrubEdgeCompressor( + summarizer=summarizer, + image_scrub_depth=2, + pending_grace_steps=5, + ) + _simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)]) + + assert not _has_image(messages[0]) + assert "[visual summary unavailable; evidence at DataEngine step 1]" in str( + messages[0].content + ) + + +def test_freeze_invariant_bytes_never_change(mock_context): + """Once a message crosses the scrub edge its content bytes never change, + across many turns and late-arriving summaries.""" + summarizer = VisualStepSummarizer(mock_context) + # Odd steps have summaries ready from the start; even steps stay pending. + for i in range(1, 11): + summarizer._step_inputs[f"tc{i}"] = {"step_number": i} + if i % 2 == 1: + summarizer._summaries[f"tc{i}"] = f"Ready summary {i}." + + messages: list = [] + compressor = ScrubEdgeCompressor( + summarizer=summarizer, + image_scrub_depth=2, + pending_grace_steps=1, + ) + + frozen_fingerprints: dict[int, str] = {} + + def check_frozen(): + for idx in compressor._frozen: + fingerprint = _content_fingerprint(messages[idx]) + if idx in frozen_fingerprints: + assert frozen_fingerprints[idx] == fingerprint, ( + f"frozen message {idx} mutated" + ) + else: + frozen_fingerprints[idx] = fingerprint + + for i in range(1, 11): + compressor.compress(messages) + check_frozen() + messages.append(AIMessage(content=f"Thinking about step {i}.")) + messages.append(_step(i)) + if i == 6: + # Late summaries for already-frozen even steps: must be ignored. + summarizer._summaries["tc2"] = "Late summary 2." + summarizer._summaries["tc4"] = "Late summary 4." + compressor.compress(messages) + check_frozen() + + assert len(frozen_fingerprints) >= 6 + # A frozen pending step carries the placeholder, not the late summary. + assert "Late summary 2." not in " ".join(str(m.content) for m in messages) + + +def test_xml_depth1_strip_matches_legacy_semantics(mock_context): + """Depth-1 XML stripping keeps combined-block prefixes and the live list.""" + messages: list = [] + compressor = ScrubEdgeCompressor(summarizer=None, prune_history_xml=True) + + combined = ToolMessage( + tool_call_id="tc1", + name="click", + content=[ + { + "type": "text", + "text": "Tapped Settings.\n--- UI Element List ---\n[huge tree]", + }, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}}, + ], + ) + _simulate_turns(compressor, messages, [combined, _step(2)]) + + # Historical combined block: prefix retained, tree stripped (the exact + # legacy ``compress_flash_messages`` product for this input). + assert messages[0].content[0] == {"type": "text", "text": "Tapped Settings."} + # Live observation keeps its UI list untouched. + assert "Tree 2" in str(messages[1].content) + + +def test_prune_history_xml_disabled_keeps_ui_lists(mock_context): + """With the switch off, UI lists survive both depth-1 and freeze scrubs.""" + summarizer = VisualStepSummarizer(mock_context) + summarizer._summaries["tc1"] = "Summary 1." + + messages: list = [] + compressor = ScrubEdgeCompressor( + summarizer=summarizer, + prune_history_xml=False, + image_scrub_depth=2, + pending_grace_steps=0, + ) + _simulate_turns(compressor, messages, [_step(1), _step(2), _step(3)]) + + # Step 1 froze (summary applied) but its UI list is preserved. + assert not _has_image(messages[0]) + assert "Tree 1" in str(messages[0].content) + assert "Summary 1." in str(messages[0].content) + + +def test_untracked_messages_are_never_touched(mock_context): + """Injected instructions and AI turns pass through untouched; a frozen + image-only tool message falls back to non-empty content.""" + messages: list = [ + HumanMessage(content="[REAL-TIME INJECTED INSTRUCTION from user]: stay put"), + ] + compressor = ScrubEdgeCompressor(summarizer=None, image_scrub_depth=2) + + image_only = ToolMessage( + tool_call_id="tc1", + name="click", + content=[{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}}], + ) + _simulate_turns( + compressor, + messages, + [image_only, AIMessage(content="thinking"), _step(2), _step(3)], + ) + + assert messages[0].content == "[REAL-TIME INJECTED INSTRUCTION from user]: stay put" + assert messages[2].content == "thinking" + # Image-only tool message froze with the defensive fallback text. + assert messages[1].content == [{"type": "text", "text": "Action completed."}] diff --git a/tests/unit/agents/test_flash_step_summarizer.py b/tests/unit/agents/test_flash_step_summarizer.py index bcd85f0..6ee7de1 100644 --- a/tests/unit/agents/test_flash_step_summarizer.py +++ b/tests/unit/agents/test_flash_step_summarizer.py @@ -14,12 +14,12 @@ """Unit tests for VisualStepSummarizer and Context Compressor in Flash profile.""" -from unittest.mock import AsyncMock, Mock, call +from unittest.mock import ANY, AsyncMock, Mock, call from uuid import uuid4 import pytest from langchain_core.messages import AIMessage, HumanMessage, ToolMessage -from artemis.agents.flash.context_compressor import compress_flash_messages +from artemis.agents.flash.context_compressor import ScrubEdgeCompressor from artemis.agents.flash.summarizer import VisualStepSummarizer from artemis.context import ArtemisContext from artemis.sdk.builders import Builders @@ -100,8 +100,94 @@ async def test_summarizer_retries_independently(mock_context): assert attempt_count == 2 +@pytest.mark.asyncio +async def test_summarizer_bounded_retry_marks_failed(mock_context): + """Persistent failures exit after 1 + retry_limit attempts with an explicit failed state.""" + mock_context.data_engine = Mock() + summarizer = VisualStepSummarizer(mock_context, retry_limit=2) + summarizer._retry_delays = (0.0,) + attempt_count = 0 + + async def mock_ainvoke(messages): + nonlocal attempt_count + attempt_count += 1 + raise TimeoutError("Simulated permanent failure") + + summarizer._llm = Mock() + summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke) + + step_id = uuid4() + summarizer.dispatch( + step_number=1, + action_name="click", + action_args={"target": [100, 100]}, + pre_img_bytes=b"pre1", + post_img_bytes=b"post1", + exec_outcome="Outcome 1", + action_key="tc-fail", + data_engine_step_id=step_id, + ) + await summarizer.flush() + + # Bounded exit: initial attempt + retry_limit retries, then explicit failure. + assert attempt_count == 3 + assert summarizer.has_failed("tc-fail") + assert not summarizer.has_summary("tc-fail") + # Lossless semantics: a failed step still reads as pending so the + # compressor keeps its source image instead of dropping evidence. + assert summarizer.is_pending("tc-fail") + + # DataEngine observability: pending on dispatch, failed on exhaustion. + mock_context.data_engine.update_step_summary.assert_has_calls( + [ + call(step_id, None, status="pending", source="visual_transition", model=ANY), + call(step_id, None, status="failed", source="visual_transition", model=ANY), + ] + ) + + +@pytest.mark.asyncio +async def test_summarizer_success_within_retry_budget_not_failed(mock_context): + """A summary that succeeds on a retry within the budget never enters failed state.""" + summarizer = VisualStepSummarizer(mock_context, retry_limit=3) + summarizer._retry_delays = (0.0,) + attempt_count = 0 + + async def mock_ainvoke(messages): + nonlocal attempt_count + attempt_count += 1 + if attempt_count < 3: + raise TimeoutError("Transient failure") + return AIMessage(content="[Step 1 State: Finally worked.]") + + summarizer._llm = Mock() + summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke) + + summarizer.dispatch( + step_number=1, + action_name="click", + action_args={"target": [100, 100]}, + pre_img_bytes=b"pre1", + post_img_bytes=b"post1", + exec_outcome="Outcome 1", + action_key="tc-ok", + ) + await summarizer.flush() + + assert attempt_count == 3 + assert summarizer.has_summary("tc-ok") + assert not summarizer.has_failed("tc-ok") + + def test_context_compressor_with_ready_summaries(mock_context): - """Verify that ready summaries replace past image blocks and prune outdated XML.""" + """Ready summaries replace past image blocks and outdated XML is pruned. + + Rewritten against ``ScrubEdgeCompressor`` (M5; the legacy + ``compress_flash_messages`` was removed) with ``image_scrub_depth=1`` so + every historical message is scrubbed immediately, mirroring the legacy + timing this test was written for. Summaries are keyed by legacy ordinals + to keep the older-caller fallback path covered. + """ summarizer = VisualStepSummarizer(mock_context) summarizer._summaries[1] = "[Step 1 State: Tapped Search bar. Keyboard appeared.]" summarizer._summaries[2] = "[Step 2 State: Typed 'flights'. Suggestions displayed.]" @@ -134,7 +220,13 @@ def test_context_compressor_with_ready_summaries(mock_context): ), ] - compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True) + compressor = ScrubEdgeCompressor( + summarizer=summarizer, + prune_history_xml=True, + image_scrub_depth=1, + pending_grace_steps=0, + ) + compressor.compress(messages) # Initial HumanMessage: Historical initial image pruned, task text preserved assert len(messages[0].content) == 1 @@ -155,88 +247,15 @@ def test_context_compressor_with_ready_summaries(mock_context): assert "Live Latest Tree 3" in messages[2].content[2]["text"] -def test_context_compressor_graceful_fallback(mock_context): - """Verify that when a summary is not ready yet, it cleanly falls back to standard pruned state.""" - summarizer = VisualStepSummarizer(mock_context) - # Step 1 has no summary in cache - - messages = [ - HumanMessage( - content=[ - {"type": "text", "text": "Task: Search"}, - {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}}, - ] - ), - ToolMessage( - tool_call_id="tc1", - name="click", - content=[ - {"type": "text", "text": "Action 'click' completed."}, - {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}}, - ], - ), - ] - - compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True) - - # Turn 1 should simply have image omitted (standard fallback) - assert len(messages[0].content) == 1 - assert messages[0].content[0]["text"] == "Task: Search" - - # Turn 2 (Latest) preserves image - assert len(messages[1].content) == 2 - assert messages[1].content[1]["type"] == "image_url" - - -def test_context_compressor_backfills_late_summary_idempotently(mock_context): - """A pending image remains until its late summary replaces it.""" - summarizer = VisualStepSummarizer(mock_context) - summarizer._step_inputs["action-1"] = {"step_number": 1} - messages = [ - HumanMessage( - content=[ - {"type": "text", "text": "Task: Search"}, - {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}}, - ] - ), - ToolMessage( - tool_call_id="action-1", - name="click", - content=[ - {"type": "text", "text": "Tapped Search."}, - {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}}, - ], - ), - ToolMessage( - tool_call_id="action-2", - name="input_text", - content=[ - {"type": "text", "text": "Typed query."}, - {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_3"}}, - ], - ), - ] - - compress_flash_messages(messages, summarizer=summarizer) - assert any(block["type"] == "image_url" for block in messages[1].content) - assert len(messages[1].content) == 2 - - summarizer._summaries["action-1"] = "Search field focused; keyboard appeared." - compress_flash_messages(messages, summarizer=summarizer) - compress_flash_messages(messages, summarizer=summarizer) - - generated_blocks = [ - block - for block in messages[1].content - if block.get("text", "").startswith("--- Historical Visual Transition ---") - ] - assert len(generated_blocks) == 1 - assert generated_blocks[0]["text"].endswith("Search field focused; keyboard appeared.") - assert all(block["type"] != "image_url" for block in messages[1].content) - - def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context): - """Generic tool messages cannot shift action summaries onto the wrong result.""" + """Generic tool messages cannot shift action summaries onto the wrong result. + + (Rewritten against ``ScrubEdgeCompressor`` in M5. The legacy tests for + pending-summary fallback and late-summary backfill were removed with + ``compress_flash_messages``: the scrub edge covers pending handling in + ``test_flash_scrub_edge.py``, and late backfill is now forbidden by the + freeze invariant rather than supported.) + """ summarizer = VisualStepSummarizer(mock_context) summarizer._summaries["click-id"] = "Click-specific visual memory." summarizer._summaries["type-id"] = "Typing-specific visual memory." @@ -265,7 +284,12 @@ def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context) ), ] - compress_flash_messages(messages, summarizer=summarizer) + compressor = ScrubEdgeCompressor( + summarizer=summarizer, + image_scrub_depth=1, + pending_grace_steps=0, + ) + compressor.compress(messages) assert "Click-specific visual memory." in str(messages[0].content) assert "Typing-specific visual memory." not in str(messages[0].content) @@ -274,35 +298,6 @@ def test_context_compressor_uses_tool_call_id_over_message_ordinal(mock_context) assert messages[2].content[-1]["type"] == "image_url" -def test_context_compressor_preserves_text_before_embedded_ui_list(mock_context): - """Pruning a combined XML block must retain its action result prefix.""" - messages = [ - ToolMessage( - tool_call_id="old", - name="click", - content=[ - { - "type": "text", - "text": "Tapped Settings.\n--- UI Element List ---\n[huge tree]", - }, - {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,A"}}, - ], - ), - ToolMessage( - tool_call_id="live", - name="click", - content=[ - {"type": "text", "text": "Opened page."}, - {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,B"}}, - ], - ), - ] - - compress_flash_messages(messages, prune_history_xml=True) - - assert messages[0].content == [{"type": "text", "text": "Tapped Settings."}] - - @pytest.mark.asyncio async def test_summarizer_keys_actions_independently_of_turn_number(mock_context): """Multiple actions from one LLM turn keep independent inputs and summaries.""" @@ -344,17 +339,229 @@ async def test_summarizer_keys_actions_independently_of_turn_number(mock_context assert summarizer.get_summary("tc-1") == "First action memory." assert summarizer.get_summary("tc-2") == "Second action memory." - assert summarizer._step_inputs["tc-1"]["data_engine_step_id"] == first_step_id - assert summarizer._step_inputs["tc-2"]["data_engine_step_id"] == second_step_id + # M1 keying: jobs are canonical under the DataEngine step id; the + # tool_call_id stays queryable as an alias. + assert summarizer.get_job_payload("tc-1")["data_engine_step_id"] == first_step_id + assert summarizer.get_job_payload("tc-2")["data_engine_step_id"] == second_step_id + assert summarizer.resolve_key("tc-1") == str(first_step_id) mock_context.data_engine.update_step_summary.assert_has_calls( [ - call(first_step_id, "First action memory."), - call(second_step_id, "Second action memory."), + call( + first_step_id, + "First action memory.", + status="ready", + source="visual_transition", + model=ANY, + ), + call( + second_step_id, + "Second action memory.", + status="ready", + source="visual_transition", + model=ANY, + ), ], any_order=True, ) +@pytest.mark.asyncio +async def test_single_frame_dispatch_uses_single_image_prompt_variant(mock_context): + """A missing after-frame (every Pro step, by design) selects the + single-image prompt: decision frame + red marker, no AFTER ACTION section.""" + summarizer = VisualStepSummarizer(mock_context) + captured: list = [] + + async def mock_ainvoke(messages): + captured.append(messages) + return AIMessage( + content=( + "In Step 1, I tapped the 'Battery' row marked in red in the" + " Settings list, which showed 'Network & internet' and 'Display'." + ) + ) + + summarizer._llm = Mock() + summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke) + + summarizer.dispatch( + step_number=1, + action_name="click", + action_args={"target": [100, 200]}, + pre_img_bytes=b"decision_frame", + post_img_bytes=None, + exec_outcome="executed", + ) + await summarizer.flush() + + assert summarizer.has_summary(1) + system_text = str(captured[0][0].content) + human_text = str(captured[0][1].content) + assert "Exactly ONE screenshot" in system_text + assert "no independent post-action evidence" in system_text + assert "DECISION FRAME" in human_text + assert "AFTER ACTION SCREEN" not in human_text + assert "--- [2]" not in human_text + + +@pytest.mark.asyncio +async def test_dual_frame_dispatch_keeps_transition_prompt(mock_context): + """Both frames present (Flash path): the classic BEFORE/AFTER transition + prompt and section labels are unchanged.""" + summarizer = VisualStepSummarizer(mock_context) + captured: list = [] + + async def mock_ainvoke(messages): + captured.append(messages) + return AIMessage( + content=( + "In Step 1, I tapped the 'Settings' gear icon marked in red;" + " the main Settings menu opened showing 'Apps'." + ) + ) + + summarizer._llm = Mock() + summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke) + + summarizer.dispatch( + step_number=1, + action_name="click", + action_args={"target": [100, 200]}, + pre_img_bytes=b"pre", + post_img_bytes=b"post", + exec_outcome="executed", + ) + await summarizer.flush() + + assert summarizer.has_summary(1) + system_text = str(captured[0][0].content) + human_text = str(captured[0][1].content) + assert "Before/After screenshots" in system_text + assert "--- [1] BEFORE ACTION SCREEN (Action Marked Visually in Red) ---" in human_text + assert "--- [2] AFTER ACTION SCREEN ---" in human_text + assert "DECISION FRAME" not in human_text + + +@pytest.mark.asyncio +async def test_echo_output_fails_attempt_then_recovers_on_retry(mock_context): + """An input-marker echo is a failed attempt (not a summary): the bounded + retry regenerates and the recovered text lands normally.""" + summarizer = VisualStepSummarizer(mock_context) + summarizer._retry_delays = (0.0,) + attempts = 0 + + async def mock_ainvoke(messages): + nonlocal attempts + attempts += 1 + if attempts == 1: + return AIMessage(content="--- [2] AFTER ACTION SCREEN ---") + return AIMessage( + content=( + "In Step 1, I tapped the search field marked in red; the field" + " showed placeholder text 'Search here'." + ) + ) + + summarizer._llm = Mock() + summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke) + + summarizer.dispatch( + step_number=1, + action_name="click", + action_args={"target": [1, 2]}, + pre_img_bytes=b"pre", + post_img_bytes=b"post", + exec_outcome="executed", + ) + await summarizer.flush() + + assert attempts == 2 + assert summarizer.has_summary(1) + assert "Search here" in summarizer.get_summary(1) + + +@pytest.mark.asyncio +async def test_degenerate_outputs_exhaust_to_failed(mock_context): + """Echo/too-short/too-long outputs never land as summaries; persistent + degeneration exhausts the bounded retries into the explicit failed state.""" + summarizer = VisualStepSummarizer(mock_context, retry_limit=1) + summarizer._retry_delays = (0.0,) + + async def mock_ainvoke(messages): + text = str(messages) + if "Step 1" in text: + return AIMessage(content="--- [1] DECISION FRAME: SCREEN AT ACTION TIME ---") + if "Step 2" in text: + return AIMessage(content="x" * 2000) + return AIMessage(content="ok.") + + summarizer._llm = Mock() + summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke) + + for step in (1, 2, 3): + summarizer.dispatch( + step_number=step, + action_name="click", + action_args={"target": [1, 2]}, + pre_img_bytes=b"pre", + post_img_bytes=None, + exec_outcome="executed", + ) + await summarizer.flush() + + for step in (1, 2, 3): + assert summarizer.has_failed(step), f"step {step} should have failed" + assert not summarizer.has_summary(step) + + +@pytest.mark.asyncio +async def test_raw_lens_call_meters_llm_usage_without_touching_context_base(mock_context): + """The raw-model lens bypass records an llm_usage trace, but its tiny + prompt never overwrites the session's last_prompt_tokens (the compaction + thresholds' live context base).""" + from artemis.services.token_meter import get_meter + + engine = Mock() + engine.current_session_id = "lens-meter-session" + engine.current_step_id = None + mock_context.data_engine = engine + meter = get_meter("lens-meter-session") + meter.last_prompt_tokens = 777 + + summarizer = VisualStepSummarizer(mock_context) + summarizer._llm = Mock() + summarizer._llm.ainvoke = AsyncMock( + return_value=AIMessage( + content=( + "In Step 1, I tapped the 'Battery' row marked in red; the list" + " showed 'Display' and 'Battery'." + ), + usage_metadata={"input_tokens": 120, "output_tokens": 30, "total_tokens": 150}, + ) + ) + + summarizer.dispatch( + step_number=1, + action_name="click", + action_args={"target": [1, 2]}, + pre_img_bytes=b"pre", + post_img_bytes=b"post", + exec_outcome="executed", + ) + await summarizer.flush() + + usage_calls = [ + c + for c in engine.record_trace.call_args_list + if c.kwargs.get("name") == "llm_usage" + ] + assert usage_calls, "the raw lens bypass must record an llm_usage trace" + payload = usage_calls[0].kwargs["payload"] + assert payload["source"].startswith("lens:visual_transition:") + assert payload["prompt_tokens"] == 120 + assert meter.last_prompt_tokens == 777 + + def test_flash_config_and_builder(): """Verify Flash profile configuration model and SDK builder fluent API.""" cfg = Builders.AgentConfig.with_flash_config( @@ -395,7 +602,13 @@ async def test_data_engine_integration_with_step_summarizer(mock_context): ) await summarizer.flush() - mock_de.update_step_summary.assert_called_once_with(1, "[Step 1 State: Input text confirmed.]") + mock_de.update_step_summary.assert_called_once_with( + 1, + "[Step 1 State: Input text confirmed.]", + status="ready", + source="visual_transition", + model=ANY, + ) def test_draw_action_overlay_rendering(): diff --git a/tests/unit/agents/test_historical_state_hint.py b/tests/unit/agents/test_historical_state_hint.py new file mode 100644 index 0000000..a766e25 --- /dev/null +++ b/tests/unit/agents/test_historical_state_hint.py @@ -0,0 +1,201 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""HistoricalStateHintPromptComponent (M4): perceptual-hash hint + silence rules. + +Division of labor under test: the hint fires only for matches *older* than the +recent window (returned-to-an-early-state regime); any match within the last 3 +steps silences it entirely — that regime belongs to the pixel-level +ScreenshotSimilarityPromptComponent. +""" + +import base64 +import io +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +from PIL import Image, ImageDraw + +from artemis.agents.operator.prompts import HistoricalStateHintPromptComponent, PromptBuilder +from artemis.utils.image_hash import dhash_hex + + +def _jpeg(draw_fn) -> bytes: + img = Image.new("RGB", (270, 600), "white") + draw_fn(ImageDraw.Draw(img)) + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=85) + return buf.getvalue() + + +def _screen_a(d): + d.rectangle([20, 40, 250, 90], fill="#3355ff") + d.rectangle([60, 450, 210, 510], fill="#22aa66") + + +def _screen_b(d): + for y in range(60, 560, 80): + d.rectangle([10, y, 260, y + 50], fill="#dddddd") + + +SCREEN_A = _jpeg(_screen_a) +SCREEN_B = _jpeg(_screen_b) +HASH_A = dhash_hex(SCREEN_A) +HASH_B = dhash_hex(SCREEN_B) +B64_A = base64.b64encode(SCREEN_A).decode("utf-8") + + +def _step(number: int, post_hash: str | None) -> dict: + meta = {"post_image_dhash": post_hash} if post_hash else {} + return {"step_number": number, "extra_metadata": meta} + + +def _cfg(similarity_hint=True, max_distance=8): + return SimpleNamespace( + memory=SimpleNamespace( + transcript=SimpleNamespace( + similarity_hint=similarity_hint, + similarity_max_distance=max_distance, + ) + ) + ) + + +async def _run(steps, b64=B64_A, cfg=None): + builder = PromptBuilder() + component = HistoricalStateHintPromptComponent() + with patch("artemis.config.load_agent_config", return_value=cfg or _cfg()): + await component( + builder, MagicMock(), MagicMock(), latest_screenshot_b64=b64, steps=steps + ) + return [p for p in builder.human_parts if isinstance(p, str)] + + +@pytest.mark.asyncio +async def test_hint_fires_on_old_matching_step(): + steps = [ + _step(1, HASH_B), + _step(2, HASH_A), # old match + _step(3, HASH_B), + _step(4, HASH_B), + _step(5, HASH_B), + _step(6, HASH_B), + ] + parts = await _run(steps) + assert len(parts) == 1 + assert "Historical state hint" in parts[0] + assert "Step 2" in parts[0] + assert "recall_history" in parts[0] + + +@pytest.mark.asyncio +async def test_silent_when_recent_step_also_matches(): + steps = [ + _step(1, HASH_A), # old match exists... + _step(2, HASH_B), + _step(3, HASH_B), + _step(4, HASH_B), + _step(5, HASH_A), # ...but a recent (last-3) match silences the hint + _step(6, HASH_B), + ] + assert await _run(steps) == [] + + +@pytest.mark.asyncio +async def test_silent_without_any_match(): + steps = [_step(n, HASH_B) for n in range(1, 7)] + assert await _run(steps) == [] + + +@pytest.mark.asyncio +async def test_silent_when_config_disabled(): + steps = [ + _step(1, HASH_A), + _step(2, HASH_B), + _step(3, HASH_B), + _step(4, HASH_B), + _step(5, HASH_B), + _step(6, HASH_B), + ] + assert await _run(steps, cfg=_cfg(similarity_hint=False)) == [] + + +@pytest.mark.asyncio +async def test_silent_with_too_few_steps(): + # Everything within the recent window belongs to the pixel-level note. + assert await _run([_step(1, HASH_A), _step(2, HASH_A)]) == [] + + +@pytest.mark.asyncio +async def test_tie_prefers_most_recent_old_step(): + steps = [ + _step(1, HASH_A), + _step(2, HASH_A), # same distance, more recent + _step(3, HASH_B), + _step(4, HASH_B), + _step(5, HASH_B), + _step(6, HASH_B), + ] + parts = await _run(steps) + assert len(parts) == 1 and "Step 2" in parts[0] + + +@pytest.mark.asyncio +async def test_steps_without_hashes_are_skipped(): + steps = [ + _step(1, None), + _step(2, None), + _step(3, None), + _step(4, None), + _step(5, None), + _step(6, None), + ] + assert await _run(steps) == [] + + +def test_record_step_stamps_perceptual_hashes(tmp_path): + from artemis.context import ArtemisContext + from artemis.data_engine.engine import DataEngine + + mock_ctx = MagicMock(spec=ArtemisContext) + setup = MagicMock() + setup.traces_path = str(tmp_path) + mock_ctx.execution_setup = setup + mock_ctx.device = None + engine = DataEngine(mock_ctx) + engine.start_session("dhash stamping test") + + engine.record_step( + pre_screenshot_bytes=SCREEN_A, + post_screenshot_bytes=SCREEN_B, + summary="two distinct screens", + ) + # Same pre/post bytes: post_image_name is nulled but the post-action + # screen IS the pre screen, so the post hash mirrors the pre hash. + engine.record_step( + pre_screenshot_bytes=SCREEN_A, + post_screenshot_bytes=SCREEN_A, + summary="screen unchanged", + ) + for t in list(engine._pending_threads): + t.join() + + steps = engine.storage.get_steps(engine.current_session_id) + meta1 = steps[0].extra_metadata + assert meta1["pre_image_dhash"] == HASH_A + assert meta1["post_image_dhash"] == HASH_B + meta2 = steps[1].extra_metadata + assert meta2["pre_image_dhash"] == HASH_A + assert meta2["post_image_dhash"] == HASH_A diff --git a/tests/unit/agents/test_history_analyzer.py b/tests/unit/agents/test_history_analyzer.py index af0064e..df33a56 100644 --- a/tests/unit/agents/test_history_analyzer.py +++ b/tests/unit/agents/test_history_analyzer.py @@ -344,7 +344,7 @@ async def test_history_analyzer_integration_with_task_tree(): mock_path.exists.return_value = True mock_path.read_text.return_value = fake_plan - # Mock build_plan_and_history + # Mock the policy-table compiled-history renderer (M4) with ( patch( "artemis.agents.history_analyzer.history_analyzer.get_llm", @@ -355,23 +355,23 @@ async def test_history_analyzer_integration_with_task_tree(): return_value=mock_path, ), patch( - "artemis.agents.history_analyzer.history_analyzer.build_plan_and_history" + "artemis.agents.history_analyzer.history_analyzer.build_history_for" ) as mock_build, ): mock_build.return_value = "Mocked operation history" analyzer = HistoryAnalyzer(mock_ctx) await analyzer.run("query") - # Verify build_plan_and_history call args + # Verify build_history_for call args mock_build.assert_called_once() args, kwargs = mock_build.call_args - assert args[0] == fake_plan - assert args[1] == steps + assert args[0] == "history_analyzer" + assert args[1] == fake_plan + assert args[2] == steps import hashlib expected_hash = hashlib.md5(b"Active Subgoal").hexdigest() - assert args[2] == expected_hash + assert args[3] == expected_hash - assert kwargs.get("last_n_detailed") == 1 - assert kwargs.get("min_summaries") == len(steps) + assert kwargs.get("engine") is mock_ctx.data_engine diff --git a/tests/unit/agents/test_operator.py b/tests/unit/agents/test_operator.py index cc7924a..b71bbb1 100644 --- a/tests/unit/agents/test_operator.py +++ b/tests/unit/agents/test_operator.py @@ -16,9 +16,15 @@ import json from unittest.mock import AsyncMock, MagicMock, Mock, patch from artemis.agents.operator.operator import OperatorNode +from artemis.config.agent import MemoryTranscriptConfig from artemis.context import ArtemisContext import pytest +# These tests exercise the legacy 2-message prompt path. Since M5 the +# transcript path is the default, so flag-off semantics must be requested +# explicitly (never via the config default). +LEGACY_TRANSCRIPT = MemoryTranscriptConfig(enabled=False) + @pytest.mark.asyncio async def test_operator_node_fast_path(): @@ -58,7 +64,7 @@ async def test_operator_node_fast_path(): # Patch dependencies with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): - node = OperatorNode(mock_ctx) + node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) # Verify state was updated with action @@ -77,7 +83,7 @@ async def test_perform_action_validation(): from artemis.graph.state import State mock_ctx = MagicMock() - node = OperatorNode(mock_ctx) + node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) mock_state = MagicMock(spec=State) mock_state.indexed_points = [[200, 300], [400, 500]] @@ -114,6 +120,7 @@ async def test_perform_action_validation(): "target_bounds": "[100,100][300,500]", "target_resource_id": "btn_1", "target_class": "android.widget.Button", + "target_label_source": "index", } ] @@ -143,6 +150,7 @@ async def test_perform_action_validation(): "target_bounds": "[300,400][500,600]", "target_resource_id": "input_1", "target_class": "android.widget.EditText", + "target_label_source": "index", } # Test swipe direction @@ -170,6 +178,7 @@ async def test_perform_action_validation(): "target_bounds": "[300,400][500,600]", "target_resource_id": "input_1", "target_class": "android.widget.EditText", + "target_label_source": "index", } ] @@ -222,6 +231,59 @@ async def test_perform_action_validation(): assert "Unsupported app management action" in err +@pytest.mark.asyncio +async def test_bare_coordinate_click_enriched_by_hit_test(): + """A coordinate click hit tests the pre-action frame for element semantics.""" + from artemis.agents.operator.operator import OperatorNode + from unittest.mock import MagicMock + from artemis.graph.state import State + + mock_ctx = MagicMock() + node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) + + mock_state = MagicMock(spec=State) + mock_state.indexed_elements = [ + { + "center": [550, 1440], + "text": "Confirm", + "bounds": [500, 1400, 600, 1480], + "class": "android.widget.Button", + "resource_id": "btn_confirm", + "is_ocr": False, + } + ] + + # Normalized (500, 600) on the default 1080x2400 device -> pixel (540, 1440), + # which falls inside the "Confirm" button bounds. + actions, err = node._translate_and_validate_tool( + {"name": "click", "args": {"target": [500, 600]}}, mock_state + ) + assert err is None + assert actions == [ + { + "action": "tap", + "coordinates": [540, 1440], + "normalized_coordinates": [500, 600], + "times": 1, + "delay_ms": 100, + "target_text": "Confirm", + "target_bounds": [500, 1400, 600, 1480], + "target_resource_id": "btn_confirm", + "target_class": "android.widget.Button", + "target_label_source": "hit_test", + } + ] + + # Graceful degradation: no perception data at all -> bare coordinates, "none". + mock_state.indexed_elements = [] + actions, err = node._translate_and_validate_tool( + {"name": "click", "args": {"target": [500, 600]}}, mock_state + ) + assert err is None + assert actions[0]["target_text"] is None + assert actions[0]["target_label_source"] == "none" + + @pytest.mark.asyncio async def test_operator_node_multiple_actions(): from artemis.agents.operator.operator import OperatorNode @@ -258,7 +320,7 @@ async def test_operator_node_multiple_actions(): mock_llm.bind_tools.return_value = mock_llm with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): - node = OperatorNode(mock_ctx) + node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) update_dict = node_update @@ -302,7 +364,7 @@ async def test_operator_node_no_record_step(): mock_llm.bind_tools.return_value = mock_llm with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): - node = OperatorNode(mock_ctx) + node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) mock_ctx.data_engine.record_step.assert_not_called() @@ -352,7 +414,7 @@ async def test_operator_dynamic_prompt(): mock_llm.bind_tools.return_value = mock_llm with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): - node = OperatorNode(mock_ctx, prompt_components=[MockComponent()]) + node = OperatorNode(mock_ctx, prompt_components=[MockComponent()], transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) @@ -488,7 +550,7 @@ async def test_operator_history_compression(): patch("pathlib.Path.exists", return_value=True), patch("pathlib.Path.read_text", mock_read_text), ): - node = OperatorNode(mock_ctx, last_n_detailed=5) + node = OperatorNode(mock_ctx, last_n_detailed=5, transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) @@ -506,7 +568,7 @@ async def test_operator_loop_detection(): mock_state.subagent_calls = ["ask_explorer", "ask_explorer", "ask_explorer"] mock_state.initial_goal = "Test goal" - node = OperatorNode(mock_ctx) + node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) with pytest.raises(RuntimeError) as exc_info: node_update = await node(mock_state) @@ -568,7 +630,7 @@ async def test_operator_tracks_subagent_calls(): side_effect=lambda t, ctx: t, ), ): - node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool]) + node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) update_dict = node_update @@ -660,7 +722,7 @@ async def test_operator_defer_action_for_gathering(): side_effect=lambda t, ctx: t, ), ): - node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool]) + node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool], transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) # Subagent tool should have been executed @@ -683,7 +745,7 @@ async def test_wait_actions_translation(): from artemis.graph.state import State mock_ctx = MagicMock() - node = OperatorNode(mock_ctx) + node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) mock_state = MagicMock(spec=State) # 1. Test wait_for_delay translation @@ -708,7 +770,7 @@ async def test_long_press_action_translation(): from artemis.graph.state import State mock_ctx = MagicMock() - node = OperatorNode(mock_ctx) + node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) mock_state = MagicMock(spec=State) mock_state.indexed_points = [[200, 300], [400, 500]] mock_state.indexed_elements = [ @@ -744,6 +806,7 @@ async def test_long_press_action_translation(): "target_bounds": "[100,100][300,500]", "target_resource_id": "btn_1", "target_class": "android.widget.Button", + "target_label_source": "index", } ] @@ -765,6 +828,7 @@ async def test_long_press_action_translation(): "target_bounds": None, "target_resource_id": None, "target_class": None, + "target_label_source": "none", } ] @@ -844,7 +908,7 @@ async def test_operator_no_defer_for_note_updating(): side_effect=lambda t, ctx: t, ), ): - node = OperatorNode(mock_ctx, tools=[mock_update_note_tool]) + node = OperatorNode(mock_ctx, tools=[mock_update_note_tool], transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) # The update_note tool should have been executed immediately @@ -934,7 +998,7 @@ async def test_operator_defer_for_note_reading(): side_effect=lambda t, ctx: t, ), ): - node = OperatorNode(mock_ctx, tools=[mock_read_note_tool]) + node = OperatorNode(mock_ctx, tools=[mock_read_note_tool], transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) # The read_note tool should have been executed @@ -1022,7 +1086,7 @@ async def test_operator_no_defer_for_note_appending(): side_effect=lambda t, ctx: t, ), ): - node = OperatorNode(mock_ctx, tools=[mock_append_note_tool]) + node = OperatorNode(mock_ctx, tools=[mock_append_note_tool], transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) # The append_note tool should have been executed immediately @@ -1111,7 +1175,7 @@ async def test_operator_background_tasks_prompt_injection(): side_effect=mock_invoke_llm_loop, ), ): - node = OperatorNode(mock_ctx) + node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) # Assert that the captured messages (prompts) contain the background tasks section @@ -1202,7 +1266,7 @@ async def test_operator_task_plan_warning_prompt_injection(): side_effect=mock_invoke_llm_loop_1, ), ): - node = OperatorNode(mock_ctx) + node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) full_content_1 = "" @@ -1255,7 +1319,7 @@ async def test_operator_task_plan_warning_prompt_injection(): side_effect=mock_invoke_llm_loop_2, ), ): - node = OperatorNode(mock_ctx) + node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) full_content_2 = "" @@ -1322,7 +1386,7 @@ async def test_operator_tool_limit_exceeded_warning(): side_effect=mock_invoke_llm_loop, ), ): - node = OperatorNode(mock_ctx) + node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) full_content = "" @@ -1380,7 +1444,7 @@ async def test_operator_fallback_function_call_parsing(): mock_llm.bind_tools.return_value = mock_llm with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): - node = OperatorNode(mock_ctx) + node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) node_update = await node(mock_state) # Verify state was updated with parsed click action @@ -1402,7 +1466,7 @@ async def test_operator_swipe_translation(): mock_ctx = MagicMock(spec=ArtemisContext) mock_state = MagicMock() mock_state.operator_raw_data = {"width": 1080, "height": 2400} - node = OperatorNode(mock_ctx) + node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT) # 1. Directional swipe actions, err = node._translate_and_validate_tool( diff --git a/tests/unit/agents/test_operator_prompts.py b/tests/unit/agents/test_operator_prompts.py index eb379b6..73749aa 100644 --- a/tests/unit/agents/test_operator_prompts.py +++ b/tests/unit/agents/test_operator_prompts.py @@ -24,7 +24,7 @@ def test_operator_prompt_contract_matches_note_runtime_semantics(): assert "write-through tools `update_note` and `append_note`" in prompt assert "may accompany at most one Turn-Ending Action" in prompt - assert "`read_note`, `list_notes`, and `save_note`" in prompt + assert "`read_note`, `list_notes`, `save_note`, and `recall_history`" in prompt assert "Do NOT submit a Turn-Ending Action at the same time" not in prompt assert "memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`" not in prompt diff --git a/tests/unit/agents/test_operator_transcript.py b/tests/unit/agents/test_operator_transcript.py new file mode 100644 index 0000000..907d72d --- /dev/null +++ b/tests/unit/agents/test_operator_transcript.py @@ -0,0 +1,311 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Operator dual prompt path (M2): legacy golden parity and transcript mode. + +Flag off (``agent.memory.transcript.enabled=false``): the legacy 2-message +build must stay byte-for-byte identical — pinned by SHA-256 goldens recorded +against the pre-split implementation. + +Flag on: ``_build_prompt`` renders ``S + F + A + tail`` from the session +transcript ledger; the turn's messages are staged after the tool loop and +committed (with step key and validator result) at the next build. +""" + +import base64 +import hashlib +import re +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from langchain_core.messages import HumanMessage, SystemMessage + +from artemis.agents.operator.operator import OperatorNode +from artemis.agents.operator.prompt_builder import load_operator_prompts +from artemis.agents.operator.prompts import ( + PLAN_HISTORY_STATIC_POINTER, + PLAN_HISTORY_TEMPLATE_SECTION, + PromptBuilder, + TemplatePromptComponent, + render_transcript_static_system, +) +from artemis.config.agent import MemoryTranscriptConfig +from artemis.context import ArtemisContext +from artemis.memory.step_memory import StepMemoryService +from artemis.memory.transcript import ( + EXECUTION_RESULT_MARKER, + PLAN_RECITATION_MARKER, + RESTORED_HISTORY_HEADER, + TranscriptLedger, +) + +# SHA-256 of the legacy system message rendered by TemplatePromptComponent +# with the fixed inputs below. Originally recorded BEFORE the M2 template +# split; re-recorded once in M5 (2026-09-01) after the only intended template +# change since — removing the Short-Term Memory instruction block from +# operator.json (the relay was removed end to end per +# pro-context-memory-redesign §3.3). Any other drift here means the flag-off +# path is no longer byte-identical. +GOLDEN_EMPTY_PLAN = "c45aec7d67069dad3a195534dd3c97dadb48454ca186a40eb471d5f9136589fa" +GOLDEN_SENTINEL_PLAN = "670580efd48f237c3257d996a814145fbef4ec0bb467aa4dfa6eb2db4a613826" + +SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8") + + +async def _render_legacy_system(plan_and_history: str) -> str: + builder = PromptBuilder() + component = TemplatePromptComponent() + ctx = SimpleNamespace(execution_setup=None, actuator=None, data_engine=None) + state = SimpleNamespace(initial_goal="GOLDEN_FIXED_GOAL") + await component( + builder, + state, + ctx, + prompts=load_operator_prompts(), + template_name="main_template", + plan_and_history=plan_and_history, + ) + return "".join(builder.system_parts) + + +@pytest.mark.asyncio +async def test_legacy_prompt_bytes_match_pre_split_golden(): + empty = await _render_legacy_system("No plan or history yet.") + sentinel = await _render_legacy_system("PLAN_AND_HISTORY_SENTINEL\nline2") + assert hashlib.sha256(empty.encode("utf-8")).hexdigest() == GOLDEN_EMPTY_PLAN + assert hashlib.sha256(sentinel.encode("utf-8")).hexdigest() == GOLDEN_SENTINEL_PLAN + + +def test_main_template_contains_exactly_one_plan_history_section(): + template = load_operator_prompts()["main_template"] + assert template.count(PLAN_HISTORY_TEMPLATE_SECTION) == 1 + + +def test_transcript_static_system_is_stable_and_carries_no_history(): + ctx = SimpleNamespace(execution_setup=None, actuator=None, data_engine=None) + state = SimpleNamespace(initial_goal="STATIC_GOAL") + prompts = load_operator_prompts() + + first = render_transcript_static_system(prompts, ctx, state) + second = render_transcript_static_system(prompts, ctx, state) + assert first == second # byte-stable S region + + assert "STATIC_GOAL" in first + # The volatile section is replaced by the static pointer... + assert PLAN_HISTORY_STATIC_POINTER.splitlines()[1] in first + # ...and no unrendered slot or trailing observation header survives. + assert "{{" not in first + assert not first.endswith("# CURRENT OBSERVATION\n") + + +def _transcript_ctx(): + ctx = MagicMock(spec=ArtemisContext) + ctx.execution_setup = None + ctx.actuator = None + ctx.data_engine = None + ctx.step_memory = StepMemoryService(ctx=None) + ctx.transcript_ledger = None + return ctx + + +def _transcript_state(**overrides): + state = MagicMock() + state.subagent_calls = [] + state.initial_goal = "Transcript goal" + state.operator_feedback = None + state.injected_instruction = None + state.operator_tool_limit_exceeded = False + state.structured_decisions = None + state.last_execution_result = None + state.current_step_id = None + state.latest_ui_hierarchy = [] + state.operator_raw_data = { + "screenshot_b64": SCREENSHOT_B64, + "xml_hierarchy": [], + "ocr_results": None, + "width": 1080, + "height": 2400, + } + for key, value in overrides.items(): + setattr(state, key, value) + return state + + +def _no_action_llm(captured: list): + mock_llm = MagicMock() + mock_response = MagicMock() + mock_response.tool_calls = [] + mock_response.content = "no action this turn" + + async def mock_ainvoke(*args, **kwargs): + captured.append(list(args[0])) + return mock_response + + mock_llm.ainvoke = AsyncMock(side_effect=mock_ainvoke) + mock_llm.bind_tools.return_value = mock_llm + return mock_llm + + +@pytest.mark.asyncio +async def test_transcript_mode_two_turns_build_four_regions(tmp_path): + ctx = _transcript_ctx() + # Regression data shape (M2 cold-start bug, on-device A/B 2026-09-01): + # turn 1 runs before any step record exists; by turn 2 the first step HAS + # been recorded, so `steps` is non-empty while the previous turn is still + # staged (turn_count == 0). The cold-start check must not read that + # live-session state as a process restart — seeding would be rejected by + # the empty-ledger invariant and silently demote every turn to the legacy + # 2-message path. + ctx.data_engine = MagicMock() + ctx.data_engine.base_dir = str(tmp_path) + ctx.data_engine.global_base_dir = str(tmp_path) + ctx.data_engine.get_agent_friendly_steps.return_value = [] + captured: list = [] + mock_llm = _no_action_llm(captured) + + with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): + node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True)) + + # Turn 1 + await node(_transcript_state()) + ledger = ctx.transcript_ledger + assert isinstance(ledger, TranscriptLedger) + assert ledger.has_static_prefix + assert ledger.has_staged_turn + assert ledger.turn_count == 0 + + turn1_messages = captured[0] + assert isinstance(turn1_messages[0], SystemMessage) + assert isinstance(turn1_messages[-1], HumanMessage) + tail1_text = str(turn1_messages[-1].content) + assert "# CURRENT OBSERVATION [T+" in tail1_text + assert PLAN_RECITATION_MARKER in tail1_text + + # Turn 2: step 1 is now recorded, and the previous turn is committed + # with its step id and result. + ctx.data_engine.get_agent_friendly_steps.return_value = [ + { + "step_id": "step-1", + "step_number": 1, + "summary": "Step 1 summary", + "relative_time": "1.0s", + "extra_metadata": {"subgoal_hash": "default"}, + } + ] + await node( + _transcript_state( + current_step_id="step-1", + structured_decisions='[{"action": "tap"}]', + last_execution_result={"status": "success"}, + ) + ) + + # The staged-but-uncommitted turn 2 state never triggers cold-start + # seeding (the transcript path stayed live instead of falling back). + assert not ledger.has_restored_history + assert ledger.turn_count == 1 + turn2_messages = captured[1] + # S region is the same object with the same bytes. + assert turn2_messages[0] is turn1_messages[0] + + active = list(ledger.active_messages) + # Committed turn-1 observation followed by its validator result. + assert any(EXECUTION_RESULT_MARKER in str(m.content) for m in active) + committed_obs = active[0] + # Depth-1 scrub removed the old plan recitation copy. + assert PLAN_RECITATION_MARKER not in str(committed_obs.content) + # The live tail still recites the plan. + assert PLAN_RECITATION_MARKER in str(turn2_messages[-1].content) + # Message layout is S + A + tail. + assert turn2_messages[-1] is not turn1_messages[-1] + assert re.search(r"T\+\d{2,}:\d{2}", str(turn2_messages[-1].content)) + + +@pytest.mark.asyncio +async def test_transcript_actionless_turn_commits_without_validator_message(): + ctx = _transcript_ctx() + captured: list = [] + mock_llm = _no_action_llm(captured) + + with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): + node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True)) + await node(_transcript_state()) + # Planner-rejected turns clear structured_decisions: no validator ran. + await node( + _transcript_state( + current_step_id="step-1", + structured_decisions="", + last_execution_result={"status": "success"}, + ) + ) + + ledger = ctx.transcript_ledger + assert ledger.turn_count == 1 + assert not any( + EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages + ) + + +@pytest.mark.asyncio +async def test_transcript_cold_start_builds_restored_history_block(tmp_path): + ctx = _transcript_ctx() + ctx.data_engine = MagicMock() + ctx.data_engine.base_dir = str(tmp_path) + ctx.data_engine.global_base_dir = str(tmp_path) + ctx.data_engine.get_agent_friendly_steps.return_value = [ + { + "step_id": f"step_{i}", + "step_number": i, + "summary": f"Step {i} summary", + "relative_time": f"{i}.0s", + "extra_metadata": {"subgoal_hash": "default"}, + } + for i in range(1, 4) + ] + + captured: list = [] + mock_llm = _no_action_llm(captured) + + with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): + node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True)) + await node(_transcript_state()) + + ledger = ctx.transcript_ledger + assert ledger.has_restored_history + + messages = captured[0] + restored_text = str(messages[1].content) + assert RESTORED_HISTORY_HEADER in restored_text + assert "--- Execution History ---" in restored_text + assert "Step 1 summary" in restored_text + # The live tail still follows normally and A stays appendable. + assert "# CURRENT OBSERVATION [T+" in str(messages[-1].content) + assert ledger.has_staged_turn + + +@pytest.mark.asyncio +async def test_flag_off_keeps_legacy_two_message_build_and_no_ledger(): + ctx = _transcript_ctx() + captured: list = [] + mock_llm = _no_action_llm(captured) + + with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm): + node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=False)) + await node(_transcript_state()) + + assert len(captured[0]) == 2 + assert isinstance(captured[0][0], SystemMessage) + assert isinstance(captured[0][1], HumanMessage) + assert ctx.transcript_ledger is None diff --git a/tests/unit/agents/test_outputter.py b/tests/unit/agents/test_outputter.py index 148b841..fd91c97 100644 --- a/tests/unit/agents/test_outputter.py +++ b/tests/unit/agents/test_outputter.py @@ -286,6 +286,9 @@ async def test_outputter_builds_concise_history(mock_get_llm): } ] mock_data_engine.get_agent_friendly_steps.return_value = steps + # M5: the transcript flag defaults on, so the outputter's compiled view + # consults the chunk store; no persisted chunks keeps the classic path. + mock_data_engine.get_history_chunks.return_value = [] mock_data_engine.base_dir = "/tmp/mock_session" ctx.data_engine = mock_data_engine diff --git a/tests/unit/agents/test_prompt_assembly.py b/tests/unit/agents/test_prompt_assembly.py index 70aee94..5c5d8dc 100644 --- a/tests/unit/agents/test_prompt_assembly.py +++ b/tests/unit/agents/test_prompt_assembly.py @@ -92,7 +92,7 @@ def test_full_toolset_is_the_default(): ) -@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer",)) +@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer", "recall_history")) def test_removed_tool_leaves_no_reference(tool): """The executable definition of 'an absent tool costs the model nothing'.""" for name, template in load_operator_prompts().items(): @@ -154,8 +154,8 @@ def test_full_set_enum_slots_render_verbatim(): ) assert ( "(`ask_explorer`, `ask_diagnoser`, `video_analyzer`, `run_adb_command`," - " `manage_task`, `analyze_task_output`, `read_note`, `list_notes`, and" - " `save_note`)" in out + " `manage_task`, `analyze_task_output`, `read_note`, `list_notes`," + " `save_note`, and `recall_history`)" in out ) diff --git a/tests/unit/agents/test_summarizer.py b/tests/unit/agents/test_summarizer.py index d82bb47..8e8a5f4 100644 --- a/tests/unit/agents/test_summarizer.py +++ b/tests/unit/agents/test_summarizer.py @@ -12,11 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json -from unittest.mock import MagicMock, Mock, patch +"""SummarizerNode tests (M2: visual-transition lens dispatch). + +Per the history redesign §5 the Pro SummarizerNode no longer runs its own LLM +call to produce a text capsule: it dispatches the step to the shared +step-memory service (``ctx.step_memory``) with the pre/post screenshot bytes +pulled from the DataEngine, and degrades gracefully when images are missing. +""" + +from types import SimpleNamespace +from unittest.mock import MagicMock, Mock + +import pytest + from artemis.agents.summarizer.summarizer import SummarizerNode from artemis.context import ArtemisContext -import pytest class DummyState: @@ -36,181 +46,107 @@ class DummyState: @pytest.fixture -def mock_context(): +def mock_context(tmp_path): ctx = Mock(spec=ArtemisContext) ctx.device = None - ctx.data_engine = Mock() ctx.background_tasks = [] + ctx.step_memory = MagicMock() + + pre_path = tmp_path / "pre.jpg" + post_path = tmp_path / "post.jpg" + pre_path.write_bytes(b"PRE_IMAGE") + post_path.write_bytes(b"POST_IMAGE") + + engine = Mock() + engine.get_step_number.return_value = 7 + engine.get_step_record.return_value = SimpleNamespace( + action_taken=[{"action": "tap", "coordinates": [500, 600], "target_text": "Search"}] + ) + engine.get_step_image_path.side_effect = lambda step_number, which="pre": ( + pre_path if which == "pre" else post_path + ) + ctx.data_engine = engine return ctx @pytest.mark.asyncio -async def test_summarizer_success(mock_context): - """Test that SummarizerNode successfully generates summary from raw thinking and execution result.""" - decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}]) - thinking = "The operator clicked the search button." - result = { - "status": "success", - "executed_actions": [{"action": "tap", "coordinates": [500, 600]}], - } - +async def test_summarizer_dispatches_visual_lens(mock_context): + """The node dispatches to the shared service instead of calling an LLM.""" state = DummyState( - structured_decisions=decisions, - operator_raw_thinking=thinking, - last_execution_result=result, + last_execution_result={"status": "success"}, current_step_id="12345678-1234-5678-1234-567812345678", ) - # Mock LLM response - mock_response = Mock() - mock_response.content = "Step completed successfully." - mock_llm = MagicMock() + node = SummarizerNode(mock_context) + result = await node(state) - async def mock_astream(*args, **kwargs): - yield mock_response - - mock_llm.astream.side_effect = mock_astream - - with ( - patch( - "artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm - ) as mock_get_llm, - patch("asyncio.create_task") as mock_create_task, - ): - # Run the node call - node = SummarizerNode(mock_context) - await node(state) - - # Verify task was registered in context - assert len(mock_context.background_tasks) == 1 - assert mock_context.background_tasks[0] == mock_create_task.return_value - - # Verify background task was spawned - assert mock_create_task.called - background_coroutine = mock_create_task.call_args[0][0] - - # Await the background task synchronously for test verification - await background_coroutine - - # Verify LLM was invoked with proper structured messages - assert mock_llm.astream.called - invoked_messages = mock_llm.astream.call_args[0][0] - - system_msg = invoked_messages[0].content - human_msg = invoked_messages[1].content - - assert "Summarizer Agent" in system_msg - assert "Decisions made" in human_msg - assert "The operator clicked the search button." in human_msg - - # Verify Data Engine updated the step summary - from uuid import UUID - - mock_context.data_engine.update_step_summary.assert_called_once_with( - UUID("12345678-1234-5678-1234-567812345678"), - "Step completed successfully.", + assert result == {} + mock_context.step_memory.dispatch.assert_called_once_with( + step_number=7, + action_name="tap", + action_args={"coordinates": [500, 600], "target_text": "Search"}, + pre_img_bytes=b"PRE_IMAGE", + post_img_bytes=b"POST_IMAGE", + exec_outcome="success", + data_engine_step_id="12345678-1234-5678-1234-567812345678", ) + # No direct capsule write from the node: the lens owns the versioned + # summary write when its background job completes. + mock_context.data_engine.update_step_summary.assert_not_called() @pytest.mark.asyncio -async def test_summarizer_with_native_thinking(mock_context): - """Test that SummarizerNode successfully generates summary from both explicit and native thoughts.""" - decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}]) - raw_thinking = "Explicit monologue." - native_thinking = "Silent core reasoning." - result = {"status": "success"} +async def test_summarizer_degrades_gracefully_without_images(mock_context): + """Missing DataEngine screenshots dispatch with None bytes, no exception.""" + mock_context.data_engine.get_step_image_path.side_effect = None + mock_context.data_engine.get_step_image_path.return_value = None state = DummyState( - structured_decisions=decisions, - operator_raw_thinking=raw_thinking, - operator_native_thinking=native_thinking, - last_execution_result=result, + last_execution_result={"status": "failed", "error": "tap missed"}, current_step_id="12345678-1234-5678-1234-567812345678", ) - mock_response = Mock() - mock_response.content = "Summary result." - mock_llm = MagicMock() + node = SummarizerNode(mock_context) + await node(state) - async def mock_astream(*args, **kwargs): - yield mock_response - - mock_llm.astream.side_effect = mock_astream - - with ( - patch( - "artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm - ) as mock_get_llm, - patch("asyncio.create_task") as mock_create_task, - ): - node = SummarizerNode(mock_context) - await node(state) - - # Verify task was registered in context - assert len(mock_context.background_tasks) == 1 - assert mock_context.background_tasks[0] == mock_create_task.return_value - - background_coroutine = mock_create_task.call_args[0][0] - await background_coroutine - - assert mock_llm.astream.called - invoked_messages = mock_llm.astream.call_args[0][0] - human_msg = invoked_messages[1].content - - assert "Operator explicit thoughts:" in human_msg - assert "Explicit monologue." in human_msg - assert "Operator native thoughts:" in human_msg - assert "Silent core reasoning." in human_msg + kwargs = mock_context.step_memory.dispatch.call_args.kwargs + assert kwargs["pre_img_bytes"] is None + assert kwargs["post_img_bytes"] is None + assert kwargs["exec_outcome"] == "Error: tap missed" @pytest.mark.asyncio -async def test_summarizer_with_chronological_trace(mock_context): - """Test that SummarizerNode generates a prompt containing the chronological step trace when available.""" - step_id = "12345678-1234-5678-1234-567812345678" +async def test_summarizer_falls_back_to_structured_decisions(mock_context): + """Without a step record the action comes from structured_decisions, and + extra actions ride along as additional_actions.""" + mock_context.data_engine.get_step_record.return_value = None + state = DummyState( - current_step_id=step_id, + structured_decisions=( + '[{"action": "swipe", "coordinates": [1, 2, 3, 4]},' + ' {"action": "tap", "coordinates": [9, 9]}]' + ), + last_execution_result={"status": "success"}, + current_step_id="12345678-1234-5678-1234-567812345678", ) - mock_step_data = { - "step_id": step_id, - "step_number": 1, - "relative_time": "5.0s", - "action_taken": [{"action": "click", "target_text": "Search"}], - "last_execution_result": {"status": "success"}, - "interleaved_events": [ - {"type": "thought", "content": "Let's click search."}, - ], - } + node = SummarizerNode(mock_context) + await node(state) - mock_context.data_engine.get_agent_friendly_steps.return_value = [mock_step_data] + kwargs = mock_context.step_memory.dispatch.call_args.kwargs + assert kwargs["action_name"] == "swipe" + assert kwargs["action_args"]["coordinates"] == [1, 2, 3, 4] + assert kwargs["action_args"]["additional_actions"] == [ + {"action": "tap", "coordinates": [9, 9]} + ] - # Mock LLM response - mock_response = Mock() - mock_response.content = "Summary using chronological trace." - mock_llm = MagicMock() - async def mock_astream(*args, **kwargs): - yield mock_response +@pytest.mark.asyncio +async def test_summarizer_noop_without_step_id(mock_context): + state = DummyState(current_step_id=None) - mock_llm.astream.side_effect = mock_astream + node = SummarizerNode(mock_context) + result = await node(state) - with ( - patch("artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm), - patch("asyncio.create_task") as mock_create_task, - ): - node = SummarizerNode(mock_context) - await node(state) - - # Verify task was registered in context - assert len(mock_context.background_tasks) == 1 - assert mock_context.background_tasks[0] == mock_create_task.return_value - - background_coroutine = mock_create_task.call_args[0][0] - await background_coroutine - - assert mock_llm.astream.called - invoked_messages = mock_llm.astream.call_args[0][0] - human_msg = invoked_messages[1].content - - assert "--- Execution History ---" in human_msg - assert "Let's click search." in human_msg + assert result == {} + mock_context.step_memory.dispatch.assert_not_called() diff --git a/tests/unit/data_engine/test_engine.py b/tests/unit/data_engine/test_engine.py index d92d4c6..197b976 100644 --- a/tests/unit/data_engine/test_engine.py +++ b/tests/unit/data_engine/test_engine.py @@ -380,3 +380,62 @@ def test_update_step_summary_includes_step_number_in_sse(tmp_path): assert last_update["step_number"] == 1 + +def _foreground_engine(tmp_path): + mock_ctx = MagicMock(spec=ArtemisContext) + mock_execution_setup = MagicMock() + mock_execution_setup.traces_path = str(tmp_path) + mock_ctx.execution_setup = mock_execution_setup + mock_ctx.device = None + engine = DataEngine(mock_ctx) + engine.start_session("foreground_app persistence") + return engine + + +def test_record_step_persists_explicit_foreground_app(tmp_path): + """M5: the historically dropped foreground_app parameter is persisted.""" + engine = _foreground_engine(tmp_path) + engine.record_step( + foreground_app="com.android.settings", + ui_tree=[{"package": "com.other.app", "bounds": "[0,0][10,10]"}], + summary="Explicit parameter wins", + ) + for t in list(engine._pending_threads): + t.join() + + step = engine.storage.get_steps(engine.current_session_id)[0] + assert step.extra_metadata["foreground_app"] == "com.android.settings" + + +def test_record_step_derives_foreground_app_from_ui_tree(tmp_path): + """Without the parameter, the dominant non-overlay package is stamped.""" + engine = _foreground_engine(tmp_path) + engine.record_step( + ui_tree=[ + {"package": "com.android.systemui", "bounds": "[0,0][10,10]"}, + {"package": "com.google.android.deskclock"}, + { + "packageName": "com.google.android.deskclock", + "children": [{"package": "com.google.android.deskclock"}], + }, + ], + summary="Derived from UI tree", + ) + for t in list(engine._pending_threads): + t.join() + + step = engine.storage.get_steps(engine.current_session_id)[0] + assert step.extra_metadata["foreground_app"] == "com.google.android.deskclock" + + +def test_record_step_without_app_data_stamps_nothing(tmp_path): + engine = _foreground_engine(tmp_path) + engine.record_step( + ui_tree=[{"package": "com.android.systemui"}], + summary="Only overlay packages", + ) + for t in list(engine._pending_threads): + t.join() + + step = engine.storage.get_steps(engine.current_session_id)[0] + assert "foreground_app" not in (step.extra_metadata or {}) diff --git a/tests/unit/data_engine/test_history_chunk_storage.py b/tests/unit/data_engine/test_history_chunk_storage.py new file mode 100644 index 0000000..9963954 --- /dev/null +++ b/tests/unit/data_engine/test_history_chunk_storage.py @@ -0,0 +1,105 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""History-chunk storage (M3 §6.1): append-only versioned rows keyed by step range.""" + +from uuid import uuid4 + +import pytest + +from artemis.data_engine.models import HistoryChunkRecord +from artemis.data_engine.storage import StorageManager + +SESSION = "11111111-1111-1111-1111-111111111111" + + +@pytest.fixture +def storage(tmp_path): + return StorageManager(tmp_path / "test.db", tmp_path / "traces") + + +def _chunk(start: int, end: int, version: int = 1, **overrides) -> HistoryChunkRecord: + fields = { + "chunk_id": uuid4(), + "session_id": SESSION, + "start_step_id": f"sid-{start}", + "end_step_id": f"sid-{end}", + "start_step_number": start, + "end_step_number": end, + "source_step_ids": [f"sid-{n}" for n in range(start, end + 1)], + "subgoal_hash": "hash-a", + "version": version, + "status": "pending", + "band1": {}, + "band2": None, + "band3": f"- Step {start} (T+00:10): Tapped 'x' at [1, 2] -> executed", + "rendered_text": "[Chunk 1 | ...]", + } + fields.update(overrides) + return HistoryChunkRecord(**fields) + + +def test_history_chunk_roundtrip(storage): + storage.create_history_chunk(_chunk(1, 4)) + chunks = storage.get_history_chunks(SESSION) + assert len(chunks) == 1 + rec = chunks[0] + assert (rec.start_step_number, rec.end_step_number) == (1, 4) + assert rec.source_step_ids == [f"sid-{n}" for n in range(1, 5)] + assert rec.subgoal_hash == "hash-a" + assert rec.status == "pending" + assert "T+00:10" in rec.band3 + + +def test_history_chunk_versions_are_append_only_latest_wins(storage): + storage.create_history_chunk(_chunk(1, 4, version=1, status="pending")) + storage.create_history_chunk( + _chunk( + 1, + 4, + version=2, + status="ready", + band1={"doing": "d", "verified_facts": ["f1"]}, + band2=" - Steps 1–4: did things", + ) + ) + + latest = storage.get_history_chunks(SESSION) + assert len(latest) == 1 + assert latest[0].version == 2 + assert latest[0].status == "ready" + assert latest[0].band1["verified_facts"] == ["f1"] + + trail = storage.get_history_chunks(SESSION, all_versions=True) + assert [r.version for r in trail] == [1, 2] + + +def test_history_chunks_ordered_by_step_range(storage): + storage.create_history_chunk(_chunk(9, 12)) + storage.create_history_chunk(_chunk(1, 4)) + storage.create_history_chunk(_chunk(5, 8)) + latest = storage.get_history_chunks(SESSION) + assert [(r.start_step_number, r.end_step_number) for r in latest] == [ + (1, 4), + (5, 8), + (9, 12), + ] + + +def test_history_chunks_are_session_scoped(storage): + storage.create_history_chunk(_chunk(1, 4)) + other = "22222222-2222-2222-2222-222222222222" + storage.create_history_chunk(_chunk(1, 4, session_id=other)) + assert len(storage.get_history_chunks(SESSION)) == 1 + assert len(storage.get_history_chunks(other)) == 1 diff --git a/tests/unit/data_engine/test_step_summary_versioning.py b/tests/unit/data_engine/test_step_summary_versioning.py new file mode 100644 index 0000000..a20c60b --- /dev/null +++ b/tests/unit/data_engine/test_step_summary_versioning.py @@ -0,0 +1,171 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the versioned, status-carrying step summary write (M0 §6.1).""" + +from unittest.mock import MagicMock + +from artemis.context import ArtemisContext +from artemis.data_engine.engine import DataEngine + + +def _make_engine(tmp_path): + mock_ctx = MagicMock(spec=ArtemisContext) + mock_execution_setup = MagicMock() + mock_execution_setup.traces_path = str(tmp_path) + mock_ctx.execution_setup = mock_execution_setup + mock_ctx.device = None + engine = DataEngine(mock_ctx) + engine.start_session("summary versioning test") + return engine + + +def _flush(engine): + for t in list(engine._pending_threads): + t.join() + + +def _get_step(engine, step_id): + steps = engine.storage.get_steps(engine.current_session_id) + for step in steps: + if str(step.step_id) == str(step_id): + return step + return None + + +def test_ready_write_persists_summary_and_metadata(tmp_path): + engine = _make_engine(tmp_path) + step_id = engine.record_step(summary="initial") + _flush(engine) + + applied = engine.storage.update_step_summary( + step_id, + "Tapped the login button.", + source="visual_transition", + model="gemini-2.5-flash-lite", + status="ready", + ) + assert applied is True + + step = _get_step(engine, step_id) + assert step.summary == "Tapped the login button." + assert step.extra_metadata["summary_status"] == "ready" + assert step.extra_metadata["summary_source"] == "visual_transition" + assert step.extra_metadata["summary_version"] == 1 + assert step.extra_metadata["summary_model"] == "gemini-2.5-flash-lite" + + +def test_auto_version_increments_per_write(tmp_path): + engine = _make_engine(tmp_path) + step_id = engine.record_step(summary="initial") + _flush(engine) + + engine.storage.update_step_summary(step_id, "v1", status="ready") + engine.storage.update_step_summary(step_id, "v2", status="ready") + + step = _get_step(engine, step_id) + assert step.summary == "v2" + assert step.extra_metadata["summary_version"] == 2 + + +def test_stale_explicit_version_is_dropped(tmp_path): + engine = _make_engine(tmp_path) + step_id = engine.record_step(summary="initial") + _flush(engine) + + assert engine.storage.update_step_summary(step_id, "newer", version=5, status="ready") + applied = engine.storage.update_step_summary(step_id, "older", version=3, status="ready") + assert applied is False + + step = _get_step(engine, step_id) + assert step.summary == "newer" + assert step.extra_metadata["summary_version"] == 5 + + +def test_pending_does_not_downgrade_ready(tmp_path): + engine = _make_engine(tmp_path) + step_id = engine.record_step(summary="initial") + _flush(engine) + + engine.storage.update_step_summary(step_id, "done", status="ready") + applied = engine.storage.update_step_summary(step_id, None, status="pending") + assert applied is False + + step = _get_step(engine, step_id) + assert step.summary == "done" + assert step.extra_metadata["summary_status"] == "ready" + + +def test_failed_marks_status_without_touching_summary(tmp_path): + engine = _make_engine(tmp_path) + step_id = engine.record_step(summary="original text") + _flush(engine) + + engine.storage.update_step_summary(step_id, None, status="pending", source="visual_transition") + applied = engine.storage.update_step_summary( + step_id, None, status="failed", source="visual_transition" + ) + assert applied is True + + step = _get_step(engine, step_id) + assert step.summary == "original text" + assert step.extra_metadata["summary_status"] == "failed" + + # A failed write must never clobber an already-ready summary. + engine.storage.update_step_summary(step_id, "recovered", status="ready") + applied = engine.storage.update_step_summary(step_id, None, status="failed") + assert applied is False + step = _get_step(engine, step_id) + assert step.extra_metadata["summary_status"] == "ready" + + +def test_missing_step_returns_false(tmp_path): + from uuid import uuid4 + + engine = _make_engine(tmp_path) + assert engine.storage.update_step_summary(uuid4(), "text", status="ready") is False + + +def test_engine_level_write_carries_metadata_and_publishes_ready_only(tmp_path): + engine = _make_engine(tmp_path) + published = [] + engine.subscribe(lambda event_type, data: published.append((event_type, data))) + + step_id = engine.record_step(summary="initial") + _flush(engine) + published.clear() + + # Non-ready statuses persist metadata but publish no SSE. + engine.update_step_summary(step_id, None, status="pending", source="visual_transition") + _flush(engine) + assert not [e for e in published if e[0] == "step_updated"] + step = _get_step(engine, step_id) + assert step.extra_metadata["summary_status"] == "pending" + + engine.update_step_summary( + step_id, + "Ready text.", + status="ready", + source="visual_transition", + model="test-model", + ) + _flush(engine) + ready_events = [e for e in published if e[0] == "step_updated"] + assert len(ready_events) == 1 + assert ready_events[0][1]["summary"] == "Ready text." + + step = _get_step(engine, step_id) + assert step.summary == "Ready text." + assert step.extra_metadata["summary_status"] == "ready" + assert step.extra_metadata["summary_model"] == "test-model" diff --git a/tests/unit/mcp/test_action_executor_semantics.py b/tests/unit/mcp/test_action_executor_semantics.py new file mode 100644 index 0000000..e52db43 --- /dev/null +++ b/tests/unit/mcp/test_action_executor_semantics.py @@ -0,0 +1,158 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for record-time target semantics enrichment in McpActionExecutor.""" + +from unittest.mock import Mock + +from artemis.mcp.action_executor import McpActionExecutor + + +def _make_executor(width=1080, height=2400): + ctx = Mock() + ctx.device.device_width = width + ctx.device.device_height = height + actuator = Mock() + actuator.controller = Mock() + return McpActionExecutor(ctx, actuator=actuator) + + +def _state_with_elements(elements): + state = Mock() + state.indexed_elements = elements + return state + + +def test_click_target_semantics_from_pre_action_frame(): + executor = _make_executor() + # Normalized (500, 600) on a 1080x2400 device -> pixel (540, 1440). + state = _state_with_elements( + [ + { + "text": "Confirm", + "bounds": [500, 1400, 600, 1480], + "class": "android.widget.Button", + "resource_id": "btn_confirm", + "is_ocr": False, + } + ] + ) + semantics = executor._target_semantics("click", {"target": [500, 600]}, state) + assert semantics == { + "target_label_source": "hit_test", + "target_text": "Confirm", + "target_class": "android.widget.Button", + "target_resource_id": "btn_confirm", + } + + +def test_semantics_degrade_to_none_without_element_data(): + executor = _make_executor() + + # Empty perception data + semantics = executor._target_semantics( + "click", {"target": [500, 600]}, _state_with_elements([]) + ) + assert semantics == {"target_label_source": "none"} + + # No state at all + semantics = executor._target_semantics("click", {"target": [500, 600]}, None) + assert semantics == {"target_label_source": "none"} + + +def test_non_targeted_actions_and_missing_targets_skip_enrichment(): + executor = _make_executor() + state = _state_with_elements([{"text": "X", "bounds": [0, 0, 10, 10]}]) + + assert executor._target_semantics("press_key", {"key": "BACK"}, state) is None + # input_text without a coordinate target (focused field typing) + assert executor._target_semantics("input_text", {"text": "hi", "target": None}, state) is None + # Multi-point actions without resolvable points skip enrichment too. + assert executor._target_semantics("click_sequence", {"sequence": []}, state) is None + assert executor._target_semantics("swipe", {"start": None, "end": None}, state) is None + + +def test_click_sequence_hit_tests_every_point_first_point_is_main_label(): + """M5: multi-point actions get per-point best-effort semantics; the first + point's fields are hoisted as the action's main label.""" + executor = _make_executor() + state = _state_with_elements( + [ + { + "text": "Digit 1", + "bounds": [500, 1400, 600, 1480], + "class": "android.widget.Button", + "is_ocr": False, + }, + { + "text": "Digit 2", + "bounds": [700, 1400, 800, 1480], + "class": "android.widget.Button", + "is_ocr": False, + }, + ] + ) + # Normalized points -> pixels: (500,600)->(540,1440) hits Digit 1; + # (690,600)->(745,1440) hits Digit 2; (10,10)->(10,24) hits nothing. + semantics = executor._target_semantics( + "click_sequence", + {"sequence": [[500, 600], [690, 600], [10, 10]], "delay_ms": 50}, + state, + ) + assert semantics["target_text"] == "Digit 1" + assert semantics["target_label_source"] == "hit_test" + per_point = semantics["points_semantics"] + assert len(per_point) == 3 + assert per_point[0]["target_text"] == "Digit 1" + assert per_point[1]["target_text"] == "Digit 2" + assert per_point[2] == {"target_label_source": "none"} + + +def test_swipe_hit_tests_start_and_end_start_is_main_label(): + executor = _make_executor() + state = _state_with_elements( + [ + { + "text": "Brightness slider", + "bounds": [100, 1400, 900, 1480], + "class": "android.widget.SeekBar", + "is_ocr": False, + } + ] + ) + semantics = executor._target_semantics( + "swipe", {"start": [200, 600], "end": [800, 600], "duration_ms": 400}, state + ) + assert semantics["target_text"] == "Brightness slider" + assert semantics["start_semantics"]["target_text"] == "Brightness slider" + assert semantics["end_semantics"]["target_text"] == "Brightness slider" + assert semantics["target_label_source"] == "hit_test" + + +def test_long_press_uses_ocr_fallback(): + executor = _make_executor() + state = _state_with_elements( + [ + { + "text": "OCR Word", + "bounds": [520, 1420, 560, 1460], + "class": None, + "resource_id": None, + "is_ocr": True, + } + ] + ) + semantics = executor._target_semantics("long_press", {"target": [500, 600]}, state) + assert semantics["target_label_source"] == "ocr" + assert semantics["target_text"] == "OCR Word" diff --git a/tests/unit/memory/test_context_policy.py b/tests/unit/memory/test_context_policy.py new file mode 100644 index 0000000..0b204be --- /dev/null +++ b/tests/unit/memory/test_context_policy.py @@ -0,0 +1,304 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ContextPolicy table (M4): byte-exact parity with the eight historical call +sites, config overrides, and the chunk hand-off to the compiled view.""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from artemis.memory.context_policy import ( + CONTEXT_POLICIES, + DIGEST_LEDGER_MARKER_TEMPLATE, + build_history_for, + load_chunk_blocks, + resolve_policy, +) +from artemis.utils.task_tree import build_plan_and_history + +HASH_A = "hash-a" +HASH_B = "hash-b" + +TASK_PLAN = f"- [x] First milestone \n- [/] Second milestone " + + +def _step(number: int, subgoal_hash: str, summary: str | None = None) -> dict: + return { + "step_id": f"id-{number}", + "step_number": number, + "relative_time": f"{number * 10}.0s", + "summary": summary or f"Step {number} summary text.", + "action_taken": {"action": "click", "target_text": f"button {number}"}, + "operator_raw_thinking": f"thinking {number}", + "last_execution_result": {"status": "success"}, + "extra_metadata": {"subgoal_hash": subgoal_hash}, + } + + +STEPS = [ + _step(1, HASH_A), + _step(2, HASH_A), + _step(3, HASH_A), + _step(4, HASH_B), + _step(5, HASH_B), + _step(6, HASH_B), + _step(7, HASH_B), +] + + +# --------------------------------------------------------------------------- +# Golden parity: the policy table must reproduce every historical hard-coded +# call byte for byte (kwargs recorded verbatim from the 2026-09-01 call sites). +# --------------------------------------------------------------------------- + + +GOLDEN_CALLS = { + "operator": lambda: build_plan_and_history( + TASK_PLAN, + STEPS, + HASH_B, + last_n_detailed=1, + strict_milestone_pruning=True, + recent_window_size=3, + ), + "operator_cold_start": lambda: build_plan_and_history( + TASK_PLAN, + STEPS, + HASH_B, + min_summaries=len(STEPS), + last_n_detailed=1, + ), + "failure_analyzer": lambda: build_plan_and_history( + TASK_PLAN, + STEPS, + HASH_B, + last_n_detailed=1, + strict_milestone_pruning=True, + recent_window_size=3, + for_failure_analyzer=True, + ), + "planner": lambda: build_plan_and_history( + "", + STEPS, + "default", + last_n_detailed=2, + strict_milestone_pruning=True, + recent_window_size=5, + ), + "outputter": lambda: build_plan_and_history( + TASK_PLAN, + STEPS, + HASH_B, + last_n_detailed=0, + min_summaries=len(STEPS), + strict_milestone_pruning=False, + ), + "history_analyzer": lambda: build_plan_and_history( + TASK_PLAN, + STEPS, + HASH_B, + last_n_detailed=1, + min_summaries=len(STEPS), + ), + "diagnoser": lambda: build_plan_and_history( + TASK_PLAN, + STEPS, + HASH_B, + keep_subgoal_hashes={HASH_B}, + ), + "committee": lambda: build_plan_and_history( + TASK_PLAN, + STEPS, + HASH_B, + keep_subgoal_hashes={HASH_B}, + last_n_detailed=3, + ), +} + +POLICY_CALLS = { + "operator": lambda: build_history_for( + "operator", TASK_PLAN, STEPS, HASH_B, last_n_detailed=1 + ), + "operator_cold_start": lambda: build_history_for( + "operator_cold_start", TASK_PLAN, STEPS, HASH_B + ), + "failure_analyzer": lambda: build_history_for( + "failure_analyzer", TASK_PLAN, STEPS, HASH_B + ), + "planner": lambda: build_history_for("planner", "", STEPS, "default"), + "outputter": lambda: build_history_for("outputter", TASK_PLAN, STEPS, HASH_B), + "history_analyzer": lambda: build_history_for( + "history_analyzer", TASK_PLAN, STEPS, HASH_B + ), + "diagnoser": lambda: build_history_for( + "diagnoser", TASK_PLAN, STEPS, HASH_B, keep_subgoal_hashes={HASH_B} + ), + "committee": lambda: build_history_for( + "committee", TASK_PLAN, STEPS, HASH_B, keep_subgoal_hashes={HASH_B} + ), +} + + +@pytest.mark.parametrize("agent", sorted(GOLDEN_CALLS)) +def test_policy_table_reproduces_call_site_bytes(agent): + assert POLICY_CALLS[agent]() == GOLDEN_CALLS[agent]() + + +def test_every_declared_policy_is_golden_tested(): + assert set(GOLDEN_CALLS) == set(CONTEXT_POLICIES) + + +# --------------------------------------------------------------------------- +# Config overrides +# --------------------------------------------------------------------------- + + +def _config_with_policies(policies): + return SimpleNamespace(memory=SimpleNamespace(policies=policies)) + + +def test_resolve_policy_applies_config_override(): + cfg = _config_with_policies({"planner": {"last_n_detailed": 3, "bogus_field": 1}}) + with patch("artemis.config.load_agent_config", return_value=cfg): + policy = resolve_policy("planner") + assert policy.last_n_detailed == 3 + assert policy.strict_milestone_pruning is True # untouched fields survive + + +def test_resolve_policy_unknown_agent_raises(): + with pytest.raises(KeyError): + resolve_policy("no_such_agent") + + +# --------------------------------------------------------------------------- +# Chunk hand-off (§10 decision 4) +# --------------------------------------------------------------------------- + + +RENDERED_CHUNK = ( + "[Chunk 1 | Steps 1–3 | T+00:10 → T+00:30]\n" + "\n" + "① Synopsis & effects\n" + " What this segment was doing: warming up.\n" + "\n" + "② Compressed step summary\n" + " - Steps 1–3: opened the app and logged in\n" + "\n" + "③ Step action ledger\n" + "- Step 1 (T+00:10): click['button 1'] -> executed\n" + "- Step 2 (T+00:20): click['button 2'] -> executed\n" + "- Step 3 (T+00:30): click['button 3'] -> executed" +) + + +class _FakeEngine: + def __init__(self, rows): + self._rows = rows + + def get_history_chunks(self, **kwargs): + return self._rows + + +def _chunk_row(): + return SimpleNamespace( + start_step_number=1, + end_step_number=3, + rendered_text=RENDERED_CHUNK, + status="ready", + ) + + +def _transcript_cfg(enabled: bool): + return SimpleNamespace( + memory=SimpleNamespace( + transcript=SimpleNamespace(enabled=enabled), policies={} + ) + ) + + +def test_flag_off_output_is_byte_identical_even_with_engine(): + engine = _FakeEngine([_chunk_row()]) + with patch("artemis.config.load_agent_config", return_value=_transcript_cfg(False)): + with_engine = build_history_for( + "outputter", TASK_PLAN, STEPS, HASH_B, engine=engine + ) + without_engine = build_history_for("outputter", TASK_PLAN, STEPS, HASH_B) + assert with_engine == without_engine + assert "[Chunk 1" not in with_engine + + +def test_flag_on_full_view_renders_chunk_block_and_drops_covered_steps(): + engine = _FakeEngine([_chunk_row()]) + with patch("artemis.config.load_agent_config", return_value=_transcript_cfg(True)): + out = build_history_for("outputter", TASK_PLAN, STEPS, HASH_B, engine=engine) + # Full view: the whole three-band block including the ③ ledger. + assert "[Chunk 1 | Steps 1–3" in out + assert "③ Step action ledger" in out + assert "- Step 2 (T+00:20)" in out + # Covered per-step summary lines are replaced by the chunk block. + assert "*Step 2 (Start: 20.0s)" not in out + # Steps outside the chunk range still render. + assert "Step 5" in out + + +def test_flag_on_digest_view_replaces_ledger_with_recall_marker(): + engine = _FakeEngine([_chunk_row()]) + cfg = _transcript_cfg(True) + with patch("artemis.config.load_agent_config", return_value=cfg): + out = build_history_for( + "diagnoser", + TASK_PLAN, + STEPS, + HASH_B, + keep_subgoal_hashes={HASH_A, HASH_B}, + engine=engine, + ) + assert "[Chunk 1 | Steps 1–3" in out + assert "② Compressed step summary" in out + marker = DIGEST_LEDGER_MARKER_TEMPLATE.format(start=1, end=3) + assert marker in out + assert "- Step 2 (T+00:20)" not in out # digest carries no ③ rows + + +def test_chunk_view_none_never_loads_chunks(): + engine = _FakeEngine([_chunk_row()]) + with patch("artemis.config.load_agent_config", return_value=_transcript_cfg(True)): + assert load_chunk_blocks(engine, None) is None + # The cold-start policy declares chunk_view=None: even with an engine + # and the flag on, its output carries no chunk blocks (the restored + # F region must stay a pure step rendering). + out = build_history_for( + "operator_cold_start", TASK_PLAN, STEPS, HASH_B, engine=engine + ) + assert CONTEXT_POLICIES["operator_cold_start"].chunk_view is None + assert "[Chunk 1" not in out + + +def test_most_recent_step_is_never_suppressed_by_a_chunk(): + engine = _FakeEngine( + [ + SimpleNamespace( + start_step_number=1, + end_step_number=7, # covers every step incl. the newest + rendered_text=RENDERED_CHUNK, + status="ready", + ) + ] + ) + with patch("artemis.config.load_agent_config", return_value=_transcript_cfg(True)): + out = build_history_for("outputter", TASK_PLAN, STEPS, HASH_B, engine=engine) + # The most-recent step must keep its own (detailed) rendering. + assert "Step 7" in out diff --git a/tests/unit/memory/test_history_chunking.py b/tests/unit/memory/test_history_chunking.py new file mode 100644 index 0000000..dbab313 --- /dev/null +++ b/tests/unit/memory/test_history_chunking.py @@ -0,0 +1,1145 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""L2/L3 history chunk compression (history redesign §3.3, M3). + +Covers: the mechanical band-③ action ledger (per-step lines, FA recovery +sub-lines, verbatim never-evict user injections), the band-② interval coverage +machine check, the three-band chunk block structure, all triggers (real +milestone switch via stamped-hash change, pseudo-switch rollback protection, +segment size, fake-meter soft/hard thresholds), the sliding-window floor, the +tool-pair invariant at the F/A boundary, frozen-block byte stability between +compression events, checkpoint post-hoc annotation version bumps, era merging, +the extreme-layer recall-only period paragraph (step range + start/end +offsets + mechanically assembled synopsis, §10 decision 5 final review), and +step addressability (step number + ``T+mm:ss``) after every compression level +— degrading to period addressability at the extreme layer only. +""" + +import json +import re +from types import SimpleNamespace + +import pytest +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage + +from artemis.memory.chunking import ( + CHUNK_PENDING_NOTE, + ChunkState, + EraState, + HistoryChunkManager, + StepCapsuleLens, + build_action_ledger, + render_era_block, + render_era_period_paragraph, + validate_interval_coverage, +) +from artemis.memory.step_memory import StepMemoryService +from artemis.memory.transcript import TranscriptLedger + +SESSION_START = 1000.0 + + +# --------------------------------------------------------------------------- +# Fixtures / helpers +# --------------------------------------------------------------------------- + + +def _step(i: int, subgoal: str = "hash-a", **overrides) -> dict: + step = { + "step_id": f"s{i}", + "step_number": i, + "timestamp": SESSION_START + 10.0 * i, + "summary": f"Visual transition of step {i}.", + "action_taken": {"action": "tap", "target_text": f"btn{i}", "coordinates": [i, i]}, + "last_execution_result": {"status": "success"}, + "operator_raw_thinking": f"thinking {i}", + "interleaved_events": [], + "extra_metadata": {"subgoal_hash": subgoal}, + } + step.update(overrides) + return step + + +class FakeEngine: + def __init__(self, steps: list[dict], base_dir=None): + self.steps = steps + self.session_start_time = SESSION_START + self.current_session_id = "session-1" + self.chunk_writes: list[dict] = [] + if base_dir is not None: + self.base_dir = base_dir + + def get_agent_friendly_steps(self): + return self.steps + + def record_history_chunk(self, **kwargs): + self.chunk_writes.append(kwargs) + return f"chunk-row-{len(self.chunk_writes)}" + + +class StubCapsuleService(StepMemoryService): + """Records capsule submissions without spawning asyncio tasks (sync tests). + + ``auto_resolve=True`` makes every submitted capsule ready immediately, so + the ready-gated swap fires at the same render that closed the segment — + the shape trigger/era/L3 tests need. The default (manual) form keeps + capsules pending until ``resolve`` is called, which is the gating tests' + lever. + """ + + def __init__(self, auto_resolve: bool = False): + super().__init__(ctx=None) + self.submitted: list[str] = [] + self.auto_resolve = auto_resolve + + def submit(self, key, payload, aliases=()): + self.submitted.append(str(key)) + self._step_inputs[key] = payload + self._failed.discard(key) + if self.auto_resolve: + self.resolve(key, _capsule(payload["start_step"], payload["end_step"])) + + def resolve(self, key, capsule: dict): + self._summaries[key] = json.dumps(capsule, ensure_ascii=False) + + +def _capsule(start: int, end: int) -> dict: + return { + "doing": "Working toward the login milestone.", + "did": "Walked the SMS-code login path", + "effect": "Logged-in state; note notes/login_flow.md records the code entry path.", + "entry_state": "Home screen, logged out", + "exit_state": "Settings page open", + "verified_facts": [f"fact-{start}"], + "unresolved": ["u1"], + "failed_paths": [], + "important_entities": ["account A"], + "intervals": [ + {"start_step": start, "end_step": end, "text": "carried out the segment's taps"} + ], + } + + +def _observation(i: int) -> HumanMessage: + return HumanMessage( + content=[ + {"type": "text", "text": f"# CURRENT OBSERVATION [T+00:{i % 60:02d}]"}, + {"type": "text", "text": "--- Current Screenshot ---"}, + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG_{i}"}}, + ] + ) + + +def _turn(i: int) -> list: + tool_call = {"name": "click", "args": {"target": i}, "id": f"tc{i}", "type": "tool_call"} + return [ + _observation(i), + AIMessage(content=f"thinking {i}", tool_calls=[tool_call]), + ToolMessage(tool_call_id=f"tc{i}", content="Action Recorded"), + ] + + +def _make( + steps: list[dict], + *, + min_active: int = 2, + max_steps: int = 12, + target_tokens: int = 10**9, + max_chunks: int = 8, + max_eras: int | None = None, + meter=None, + auto_capsule: bool = False, +): + ledger = TranscriptLedger(step_memory=StepMemoryService(ctx=None)) + engine = FakeEngine(steps) + capsule = StubCapsuleService(auto_resolve=auto_capsule) + chunker = HistoryChunkManager( + engine=engine, + chunking_config=SimpleNamespace( + max_steps=max_steps, + target_source_tokens=target_tokens, + model="test-model", + max_chunks=max_chunks, + max_eras=max_eras, + ), + transcript_config=SimpleNamespace( + context_budget_tokens=100_000, + soft_ratio=0.7, + hard_ratio=0.9, + min_active_steps=min_active, + ), + capsule_service=capsule, + meter_getter=meter or (lambda: None), + goal="TEST GOAL", + ) + ledger.attach_chunker(chunker) + return ledger, chunker, engine, capsule + + +def _run_turns(ledger, chunker, start: int, upto: int, hash_for=None): + """Mimic the real per-turn ordering: commit previous → render (chunker + hook) → stage this turn → stamp this turn's step (execution_check).""" + hash_for = hash_for or (lambda i: "hash-a") + rendered = None + for i in range(start, upto + 1): + ledger.commit_staged( + step_key=f"s{i - 1}" if i > 1 else None, + validator_result={"status": "success"} if i > 1 else None, + ) + rendered = ledger.render([_observation(i)]) + ledger.stage_turn(_turn(i)) + chunker.on_step_stamped(f"s{i}", hash_for(i)) + return rendered + + +def _rendered_text(messages) -> str: + return "\n".join(str(getattr(m, "content", "")) for m in messages) + + +def _frozen_text(ledger) -> str: + return "\n".join(str(m.content) for m in ledger.frozen_blocks) + + +# --------------------------------------------------------------------------- +# Band ③ — mechanical action ledger +# --------------------------------------------------------------------------- + + +def test_action_ledger_lines_carry_step_number_offset_action_and_result(): + steps = [_step(1), _step(2, last_execution_result={"status": "failed", "error": "boom"})] + ledger_text = build_action_ledger(steps, SESSION_START) + lines = ledger_text.splitlines() + assert lines[0] == "- Step 1 (T+00:10): Tapped 'btn1' at [1, 1] -> executed" + assert lines[1].startswith("- Step 2 (T+00:20): Tapped 'btn2' at [2, 2] -> Error: boom") + + +def test_action_ledger_renders_fa_recovery_sublines_indented(): + steps = [ + _step( + 3, + interleaved_events=[ + { + "type": "tool_call", + "name": "_exec_click", + "args": {"target": [5, 6], "target_text": "Retry"}, + "result": "ok", + }, + {"type": "tool_call", "name": "report_failure_analysis", "args": {}, "result": ""}, + ], + ) + ] + text = build_action_ledger(steps, SESSION_START) + lines = text.splitlines() + assert lines[0].startswith("- Step 3 ") + assert lines[1].startswith(" FA: Tapped 'Retry'") + assert "report_failure_analysis" not in text + + +def test_action_ledger_preserves_injected_instruction_verbatim(): + steps = [ + _step(4, extra_metadata={"subgoal_hash": "h", "injected_instruction": "跳过弹窗，直接登录"}) + ] + full = build_action_ledger(steps, SESSION_START) + minimal = build_action_ledger(steps, SESSION_START, minimal=True) + expected = ' User @ Step 4: "跳过弹窗，直接登录"' + assert expected in full + assert expected in minimal # never-evict at every width + + +def test_action_ledger_minimal_width_keeps_step_and_time_addressability(): + steps = [_step(i) for i in range(1, 4)] + minimal = build_action_ledger(steps, SESSION_START, minimal=True) + for i in range(1, 4): + assert f"- Step {i} (T+00:{10 * i:02d}): Tapped 'btn{i}'" in minimal + assert "-> executed" not in minimal + + +# --------------------------------------------------------------------------- +# Band ② — machine-checked coverage + capsule parsing +# --------------------------------------------------------------------------- + + +def test_interval_coverage_requires_seamless_union(): + ok = [{"start_step": 3, "end_step": 5, "text": "a"}, {"start_step": 6, "end_step": 6, "text": "b"}] + assert validate_interval_coverage(ok, 3, 6) + gap = [{"start_step": 3, "end_step": 4, "text": "a"}, {"start_step": 6, "end_step": 6, "text": "b"}] + assert not validate_interval_coverage(gap, 3, 6) + overlap = [{"start_step": 3, "end_step": 5, "text": "a"}, {"start_step": 5, "end_step": 6, "text": "b"}] + assert not validate_interval_coverage(overlap, 3, 6) + short = [{"start_step": 3, "end_step": 5, "text": "a"}] + assert not validate_interval_coverage(short, 3, 6) + assert not validate_interval_coverage([], 3, 6) + + +def test_capsule_parse_rejects_coverage_gap_and_accepts_full_cover(): + lens = StepCapsuleLens(model_name="test", llm=object()) + payload = {"start_step": 3, "end_step": 6} + good = json.dumps(_capsule(3, 6)) + parsed = lens.parse_capsule(good, payload) + assert parsed is not None + assert parsed["verified_facts"] == ["fact-3"] + + bad = _capsule(3, 6) + bad["intervals"] = [{"start_step": 3, "end_step": 4, "text": "partial"}] + assert lens.parse_capsule(json.dumps(bad), payload) is None + assert lens.parse_capsule("not json", payload) is None + + +@pytest.mark.asyncio +async def test_capsule_lens_render_returns_none_on_gap_so_service_retries(): + class GapLLM: + async def ainvoke(self, messages): + capsule = _capsule(1, 2) + capsule["intervals"] = [{"start_step": 1, "end_step": 1, "text": "only one"}] + return SimpleNamespace(content=json.dumps(capsule)) + + lens = StepCapsuleLens(model_name="test", llm=GapLLM()) + payload = {"start_step": 1, "end_step": 2, "steps": []} + assert await lens.render("chunk:1-2", payload) is None + + class FullLLM: + async def ainvoke(self, messages): + return SimpleNamespace(content=json.dumps(_capsule(1, 2))) + + lens_ok = StepCapsuleLens(model_name="test", llm=FullLLM()) + result = await lens_ok.render("chunk:1-2", payload) + assert result is not None and json.loads(result)["doing"] + + +# --------------------------------------------------------------------------- +# Triggers +# --------------------------------------------------------------------------- + + +def _hashes(boundary_after: int): + return lambda i: "hash-a" if i <= boundary_after else "hash-b" + + +def test_milestone_switch_closes_previous_segment_whole_and_gates_swap(): + """Ready-gated swap (user decision 2026-09-01): a milestone switch closes + the previous segment whole and dispatches its capsule, but the original + turns stay in the transcript until the capsule header is ready; the swap + happens at the first render after readiness.""" + steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 9)] + ledger, chunker, engine, capsule = _make(steps, min_active=2) + rendered = _run_turns(ledger, chunker, 1, 8, _hashes(4)) + + # Closed whole as ONE awaiting chunk; NOT swapped (capsule pending). + assert len(chunker.chunks) == 0 + assert len(chunker.awaiting_chunks) == 1 + chunk = chunker.awaiting_chunks[0] + assert (chunk.start_step_number, chunk.end_step_number) == (1, 4) + assert chunk.subgoal_hash == "hash-a" + assert chunk.status == "pending" + + # v1 pending row persisted with band ③ (recall/auxiliary views see it). + assert engine.chunk_writes[0]["version"] == 1 + assert engine.chunk_writes[0]["status"] == "pending" + assert "- Step 1 (T+00:10)" in engine.chunk_writes[0]["band3"] + + # Capsule dispatched in the background. + assert capsule.submitted == ["chunk:1-4"] + + # Original history retained verbatim: no chunk block, no frozen region, + # turn 1's observation still live in the rendered transcript. + text = _rendered_text(rendered) + assert "[Chunk 1" not in text + assert ledger.frozen_blocks == () + assert "# CURRENT OBSERVATION [T+00:01]" in text + + # Capsule ready → the next render swaps: frozen ①②③ block, originals out. + capsule.resolve("chunk:1-4", _capsule(1, 4)) + ledger.commit_staged(step_key="s8", validator_result={"status": "success"}) + rendered_after = ledger.render([_observation(9)]) + assert len(chunker.chunks) == 1 + assert chunker.awaiting_chunks == () + frozen = _frozen_text(ledger) + assert "[Chunk 1 | Steps 1–4" in frozen + assert "① Synopsis & effects" in frozen + assert "③ Step action ledger" in frozen + assert CHUNK_PENDING_NOTE not in frozen + assert "# CURRENT OBSERVATION [T+00:01]" not in _rendered_text(rendered_after) + # The ready write bumped the DB version (append-only). + ready_rows = [w for w in engine.chunk_writes if w["status"] == "ready"] + assert ready_rows and ready_rows[0]["version"] == 2 + + +def test_plan_write_hint_without_stamp_change_is_discarded(): + """Pseudo-switch protection: a vetoed+rolled-back plan write queues a hint + that the next stamped step (same hash) discards — no chunk is created.""" + steps = [_step(i) for i in range(1, 9)] + ledger, chunker, engine, capsule = _make(steps, min_active=2) + + _run_turns(ledger, chunker, 1, 3) + chunker.queue_boundary_hint() # plan write claimed a completion... + assert chunker.boundary_hint_pending + _run_turns(ledger, chunker, 4, 8) # ...but stamps never change + + assert not chunker.boundary_hint_pending + assert len(chunker.chunks) == 0 + assert engine.chunk_writes == [] + assert ledger.frozen_blocks == () + + +def test_segment_size_threshold_triggers_without_milestone(): + steps = [_step(i) for i in range(1, 10)] + ledger, chunker, engine, _ = _make(steps, min_active=2, max_steps=4, auto_capsule=True) + _run_turns(ledger, chunker, 1, 9) + + # Single milestone, but the open segment exceeded max_steps. + assert len(chunker.chunks) >= 1 + assert chunker.chunks[0].start_step_number == 1 + total = sum( + c.end_step_number - c.start_step_number + 1 for c in chunker.chunks + ) + assert total >= 4 + assert len(ledger.unchunked_turns()) >= 2 # floor respected + + +def test_source_token_threshold_triggers_chunk(): + steps = [_step(i) for i in range(1, 8)] + ledger, chunker, _, _ = _make( + steps, min_active=2, max_steps=100, target_tokens=10, auto_capsule=True + ) + _run_turns(ledger, chunker, 1, 7) + assert len(chunker.chunks) >= 1 + + +def test_soft_threshold_compresses_oldest_open_segment(): + steps = [_step(i) for i in range(1, 9)] + meter = {"value": None} + ledger, chunker, _, _ = _make( + steps, min_active=2, max_steps=100, meter=lambda: meter["value"], auto_capsule=True + ) + _run_turns(ledger, chunker, 1, 6) + assert len(chunker.chunks) == 0 # nothing due yet + + meter["value"] = 75_000 # ≥ 0.7 * 100k, < 0.9 * 100k + _run_turns(ledger, chunker, 7, 7) + assert len(chunker.chunks) == 1 + assert chunker.chunks[0].start_step_number == 1 + + +def test_min_active_floor_blocks_all_triggers(): + steps = [_step(i, "hash-a" if i <= 2 else "hash-b") for i in range(1, 6)] + ledger, chunker, engine, _ = _make( + steps, min_active=5, max_steps=2, meter=lambda: 95_000 + ) + _run_turns(ledger, chunker, 1, 6, _hashes(2)) + # Only 5 committed turns; the floor keeps everything raw. + assert len(chunker.chunks) == 0 + assert chunker.awaiting_chunks == () + assert engine.chunk_writes == [] + + +def test_compression_boundary_never_splits_tool_call_pairs(): + steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 9)] + ledger, chunker, _, _ = _make(steps, min_active=2, auto_capsule=True) + _run_turns(ledger, chunker, 1, 8, _hashes(4)) + assert len(chunker.chunks) == 1 + + rendered = ledger.render([_observation(99)]) + # Walk the rendered list: every AIMessage with tool_calls must be + # immediately followed by its ToolMessages. + for idx, msg in enumerate(rendered): + if isinstance(msg, AIMessage) and msg.tool_calls: + following = rendered[idx + 1 : idx + 1 + len(msg.tool_calls)] + assert [getattr(m, "tool_call_id", None) for m in following] == [ + tc["id"] for tc in msg.tool_calls + ] + # And no orphan ToolMessage right after the frozen blocks. + first_active = next( + (m for m in rendered if isinstance(m, (AIMessage, ToolMessage, HumanMessage)) + and "# CURRENT OBSERVATION" in str(getattr(m, "content", ""))), + None, + ) + assert first_active is not None + + +# --------------------------------------------------------------------------- +# Async capsule discipline: ready-gated swap, byte-stable frozen region +# --------------------------------------------------------------------------- + + +def test_frozen_region_stays_byte_stable_between_swap_events(): + """A pending closed segment never mutates the frozen region (its original + turns stay live); the region re-renders only when a swap event fires.""" + steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 13)] + ledger, chunker, engine, capsule = _make(steps, min_active=2) + _run_turns(ledger, chunker, 1, 8, _hashes(4)) + + # Swap chunk 1 (Steps 1–4) once its capsule is ready. + capsule.resolve("chunk:1-4", _capsule(1, 4)) + ledger.commit_staged(step_key="s8", validator_result={"status": "success"}) + ledger.render([_observation(9)]) + assert len(chunker.chunks) == 1 + frozen_before = _frozen_text(ledger) + assert "① Synopsis & effects" in frozen_before + assert "② Compressed step summary" in frozen_before + assert "- Steps 1–4: carried out the segment's taps" in frozen_before + + # A hash-c stretch closes the hash-b segment — capsule pending, so the + # frozen text stays byte-identical and the pending chunk does not swap. + ledger.stage_turn(_turn(9)) + chunker.on_step_stamped("s9", "hash-c") + _run_turns(ledger, chunker, 10, 12, lambda i: "hash-c") + assert len(chunker.awaiting_chunks) == 1 + pending = chunker.awaiting_chunks[0] + assert pending.status == "pending" + assert _frozen_text(ledger) == frozen_before + + # Its capsule readies: the next render swaps and re-renders the region. + capsule.resolve(pending.capsule_key, _capsule(5, 8)) + ledger.commit_staged(step_key="s12", validator_result={"status": "success"}) + ledger.render([_observation(13)]) + assert len(chunker.chunks) == 2 + frozen_after = _frozen_text(ledger) + assert frozen_after != frozen_before + assert "[Chunk 2" in frozen_after + + # Ready writes bumped the DB versions (append-only). + ready_rows = [w for w in engine.chunk_writes if w["status"] == "ready"] + assert ready_rows and all(w["version"] == 2 for w in ready_rows) + + +def test_chunk_block_renders_three_bands_in_order(): + steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 13)] + ledger, chunker, _, capsule = _make(steps, min_active=2) + _run_turns(ledger, chunker, 1, 8, _hashes(4)) + capsule.resolve("chunk:1-4", _capsule(1, 4)) + _run_turns(ledger, chunker, 9, 12, lambda i: "hash-c") # second event + + block = next( + str(m.content) for m in ledger.frozen_blocks if "[Chunk 1" in str(m.content) + ) + # §3.3 structure: header, then ① → ② → ③ strictly in order. + i1 = block.index("① Synopsis & effects") + i2 = block.index("② Compressed step summary") + i3 = block.index("③ Step action ledger") + assert 0 < i1 < i2 < i3 + assert "What this segment was doing:" in block + assert "What was actually done:" in block + assert "Effects / left behind:" in block + # The effect line carries the notes left behind (from the capsule). + assert "notes/login_flow.md" in block + assert "Entry: Home screen, logged out" in block + assert "Exit: Settings page open" in block + assert "Verified: fact-1" in block + # ② lines carry step references; ③ is the untouched mechanical ledger. + assert "- Steps 1–4:" in block + assert "- Step 1 (T+00:10): Tapped 'btn1' at [1, 1] -> executed" in block + assert " ago" not in block + + +# --------------------------------------------------------------------------- +# Checkpoint post-hoc annotation +# --------------------------------------------------------------------------- + + +def test_checkpoint_annotation_bumps_version_and_rerenders_at_next_event(): + steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 13)] + ledger, chunker, engine, capsule = _make(steps, min_active=2) + _run_turns(ledger, chunker, 1, 8, _hashes(4)) + # Ready gating: the closed segment is still awaiting its capsule — + # post-hoc annotations are independent of the header and land anyway. + chunk = chunker.awaiting_chunks[0] + version_before = chunk.version + frozen_before = _frozen_text(ledger) + + annotated = chunker.annotate_from_checkpoint( + "hash-a", + [{"kind": "verify", "item_text": "logged in", "status": "failed", "evidence": "no session"}], + ) + assert annotated + assert chunk.version == version_before + 1 + # DB write is immediate... + assert engine.chunk_writes[-1]["version"] == chunk.version + assert engine.chunk_writes[-1]["band1"]["annotations"][0]["item_text"] == "logged in" + # ...but the frozen text only re-renders at the next compression event. + assert _frozen_text(ledger) == frozen_before + + capsule.resolve("chunk:1-4", _capsule(1, 4)) + _run_turns(ledger, chunker, 9, 12, lambda i: "hash-c") + frozen_after = _frozen_text(ledger) + assert "Post-hoc check results:" in frozen_after + assert "'logged in' → failed (no session)" in frozen_after + + +def test_checkpoint_annotation_survives_milestone_rename_via_alias_chain(): + """Chunk identity is the step range; the subgoal hash is a label resolved + through the alias chain — a renamed milestone still reaches its chunk.""" + import pathlib + import tempfile + + steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 9)] + with tempfile.TemporaryDirectory() as tmp: + base_dir = pathlib.Path(tmp) + (base_dir / "notes").mkdir() + (base_dir / "notes" / "subgoal_hash_chain.json").write_text( + json.dumps({"hash-a": "hash-a-renamed"}), encoding="utf-8" + ) + ledger, chunker, engine, _ = _make(steps, min_active=2) + engine.base_dir = base_dir + _run_turns(ledger, chunker, 1, 8, _hashes(4)) + assert chunker.awaiting_chunks[0].subgoal_hash == "hash-a" + + assert chunker.annotate_from_checkpoint( + "hash-a-renamed", + [{"kind": "verify", "item_text": "x", "status": "passed", "evidence": "ok"}], + ) + assert chunker.awaiting_chunks[0].annotations + + +def test_checkpoint_annotation_ignores_unmatched_subgoal(): + steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 9)] + ledger, chunker, engine, _ = _make(steps, min_active=2) + _run_turns(ledger, chunker, 1, 8, _hashes(4)) + writes_before = len(engine.chunk_writes) + assert not chunker.annotate_from_checkpoint( + "hash-unknown", [{"kind": "verify", "item_text": "x", "status": "failed", "evidence": "e"}] + ) + assert len(engine.chunk_writes) == writes_before + + +# --------------------------------------------------------------------------- +# Era merging + recall-only overflow + L3 snapshot +# --------------------------------------------------------------------------- + + +def _many_chunks( + chunk_count: int, + *, + max_chunks: int, + max_eras: int | None = None, + injected_step: int | None = None, +): + """Drive enough milestone switches to accumulate ``chunk_count`` chunks + of 2 steps each (min_active=2).""" + total_steps = chunk_count * 2 + 6 + hash_for = lambda i: f"hash-{(i - 1) // 2}" # noqa: E731 — switch every 2 steps + steps = [] + for i in range(1, total_steps + 1): + extra = {"subgoal_hash": hash_for(i)} + if injected_step == i: + extra["injected_instruction"] = "keep account A logged in" + steps.append(_step(i, hash_for(i), extra_metadata=extra)) + ledger, chunker, engine, capsule = _make( + steps, min_active=2, max_chunks=max_chunks, max_eras=max_eras, auto_capsule=True + ) + _run_turns(ledger, chunker, 1, total_steps, hash_for) + return ledger, chunker, engine, capsule + + +def test_era_merge_keeps_ledgers_and_merges_headers(): + ledger, chunker, _, capsule = _many_chunks(6, max_chunks=3) + assert len(chunker.chunks) <= 3 + assert len(chunker.eras) >= 1 + + merged_eras = [e for e in chunker.eras if not e.recall_only] + assert merged_eras + era_text = "\n".join( + str(m.content) + for m in ledger.frozen_blocks + if "① Merged synopsis" in str(m.content) + ) + assert "① Merged synopsis (structured fields, set-merged)" in era_text + assert "② Segment titles" in era_text + assert "③ Step action ledger" in era_text + # ③ is retained 1:1 for every era-folded chunk (recall-only overflow + # aside, covered separately): every step keeps its addressable line + # (step number + T+ time). + for era in merged_eras: + for chunk in era.chunks: + for n in range(chunk.start_step_number, chunk.end_step_number + 1): + assert f"- Step {n} (T+" in era_text + + +def test_era_overflow_compresses_oldest_era_to_period_paragraph(): + """§10 decision 5 (final review 2026-09-01): era overflow renders the + oldest era as a period paragraph — step range AND start/end session + offsets in the header, a non-empty mechanically assembled synopsis, the + recall guidance line, and verbatim user-injection lines.""" + ledger, chunker, _, _ = _many_chunks(14, max_chunks=2, injected_step=1) + assert len(chunker.eras) > 2 + recall_eras = [e for e in chunker.eras if e.recall_only] + assert recall_eras, "oldest eras should have overflowed to recall-only" + + frozen = _frozen_text(ledger) + era = recall_eras[0] + header = ( + f"[Era {era.ordinal} | Steps {era.start_step_number}–{era.end_step_number}" + f" | {era.chunks[0].start_offset} → {era.chunks[-1].end_offset}]" + ) + assert header in frozen + # The paragraph follows on the same line and is never empty. + match = re.search(re.escape(header) + r" (.+)", frozen) + assert match and match.group(1).strip() + # Mechanical assembly from band ①: doing survives verbatim, note + # references from effect are merged in, verified facts are quoted. + assert "Working toward the login milestone." in match.group(1) + assert "notes/login_flow.md" in match.group(1) + assert "Verified: " in match.group(1) + # Recall guidance line is retained. + assert ( + f" (Step-level ledger via recall_history for steps" + f" {era.start_step_number}–{era.end_step_number})" + ) in frozen + # Never-evict: the user injection survives even the period paragraph. + assert 'User @ Step 1: "keep account A logged in"' in frozen + + +def _chunk_state( + ordinal: int, + start: int, + end: int, + *, + band1: dict | None = None, + milestone: str | None = None, + user_lines: tuple = (), +) -> ChunkState: + return ChunkState( + ordinal=ordinal, + start_step_number=start, + end_step_number=end, + start_step_id=f"s{start}", + end_step_id=f"s{end}", + source_step_ids=[f"s{i}" for i in range(start, end + 1)], + subgoal_hash=None, + milestone_label=milestone, + start_offset=f"T+{start:02d}:00", + end_offset=f"T+{end:02d}:59", + band3="", + minimal_index="", + user_lines=list(user_lines), + status="ready" if band1 else "pending", + band1=band1 or {}, + ) + + +def test_period_paragraph_falls_back_per_chunk_without_band1(): + """A pending (band-①-less) chunk contributes its milestone label — or its + step range when unlabeled — to the 'Did:' chain instead of vanishing.""" + era = EraState( + ordinal=7, + chunks=[ + _chunk_state(1, 1, 4, band1=_capsule(1, 4)), + _chunk_state(2, 5, 8, milestone="Checkout flow"), # pending, labeled + _chunk_state(3, 9, 12), # pending, unlabeled + ], + recall_only=True, + ) + paragraph = render_era_period_paragraph(era) + assert "Working toward the login milestone." in paragraph + assert "Checkout flow" in paragraph + assert "steps 9–12" in paragraph + assert "notes/login_flow.md" in paragraph + assert "fact-1" in paragraph + + +def test_period_paragraph_degrades_to_milestone_list_when_all_band1_missing(): + """With no band ① anywhere the paragraph degrades to the milestone-label + list + step ranges — never empty — and the block still carries the range/ + offset header, the recall guidance line, and verbatim user lines.""" + era = EraState( + ordinal=3, + chunks=[ + _chunk_state(1, 1, 4, milestone="Login"), + _chunk_state(2, 5, 8, user_lines=[' User @ Step 6: "stay logged in"']), + ], + recall_only=True, + ) + paragraph = render_era_period_paragraph(era) + assert paragraph.strip() + assert paragraph == "Milestones: Login (Steps 1–4); segment (Steps 5–8)." + + block = render_era_block(era) + assert block.startswith(f"[Era 3 | Steps 1–8 | T+01:00 → T+08:59] {paragraph}") + assert "(Step-level ledger via recall_history for steps 1–8)" in block + assert 'User @ Step 6: "stay logged in"' in block + + +def test_independent_max_eras_cap_decouples_from_max_chunks(): + """M5: ``chunking.max_eras`` caps eras independently; None follows + ``max_chunks`` (the pre-M5 equal-value behavior, asserted by the + overflow test above).""" + # A generous era cap keeps every folded era's ledger despite max_chunks=2. + _, roomy, _, _ = _many_chunks(14, max_chunks=2, max_eras=50) + assert len(roomy.eras) > 2 + assert not [e for e in roomy.eras if e.recall_only] + + # A tight era cap overflows old eras even though max_chunks is large. + _, tight, _, _ = _many_chunks(14, max_chunks=2, max_eras=1) + assert [e for e in tight.eras if e.recall_only] + + +def test_hard_threshold_renders_l3_snapshot_with_minimal_step_index(): + steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 13)] + meter = {"value": None} + ledger, chunker, _, capsule = _make( + steps, min_active=2, meter=lambda: meter["value"] + ) + _run_turns(ledger, chunker, 1, 8, _hashes(4)) + capsule.resolve("chunk:1-4", _capsule(1, 4)) + + meter["value"] = 95_000 # ≥ 0.9 * 100k → L3 + _run_turns(ledger, chunker, 9, 12, lambda i: "hash-c") + + frozen = _frozen_text(ledger) + assert "[Session snapshot" in frozen + assert "Overall goal: TEST GOAL" in frozen + assert "Verified facts: fact-1" in frozen + assert "Device/app state: Settings page open" in frozen + # The minimal per-step index survives: step + time + action phrase for + # every chunked step (steps 1–4 and the second event's chunk). + assert "--- Step index (minimal width, per chunk) ---" in frozen + for chunk in chunker.chunks: + for n in range(chunk.start_step_number, chunk.end_step_number + 1): + assert f"- Step {n} (T+" in frozen + assert "-> executed" not in frozen.split("--- Step index")[1] + + +def test_step_addressability_survives_every_compression_level(): + """Hard invariant (§8, revised by §10 decision 5 final review): after L2 + chunks, era merges, and L3, every compressed step's number and + session-offset time are directly readable; the extreme period-paragraph + layer alone degrades to *period* addressability — step range and start/end + offsets must be present in the header.""" + ledger, chunker, _, _ = _many_chunks(6, max_chunks=3) + frozen = _frozen_text(ledger) + covered_steps = [ + n + for era in chunker.eras + if not era.recall_only + for c in era.chunks + for n in range(c.start_step_number, c.end_step_number + 1) + ] + [ + n + for c in chunker.chunks + for n in range(c.start_step_number, c.end_step_number + 1) + ] + assert covered_steps + for n in covered_steps: + assert f"- Step {n} (T+" in frozen, f"step {n} lost its addressable line" + + # Extreme layer: recall-only eras stay *period*-addressable — the header + # regex pins step range and T+..→T+.. start/end offsets in place. + overflow_ledger, overflow_chunker, _, _ = _many_chunks(14, max_chunks=2) + overflow_frozen = _frozen_text(overflow_ledger) + recall_eras = [e for e in overflow_chunker.eras if e.recall_only] + assert recall_eras + for era in recall_eras: + pattern = ( + rf"\[Era {era.ordinal} \| Steps {era.start_step_number}" + rf"–{era.end_step_number} \| T\+\d{{2,}}:\d{{2}} → T\+\d{{2,}}:\d{{2}}\]" + ) + assert re.search(pattern, overflow_frozen), ( + f"era {era.ordinal} lost its period-addressable header" + ) + + +@pytest.mark.asyncio +async def test_capsule_retry_exhaustion_degrades_to_pending_chunk(): + """Bounded retries: a lens that never covers the range exhausts its retry + budget, the job enters the explicit failed state, and the chunk simply + stays pending — band ③ remains independently usable.""" + from artemis.memory.chunking import ChunkCapsuleService + + class NeverCoversLens(StepCapsuleLens): + def __init__(self): + super().__init__(model_name="test", llm=object()) + self.attempts = 0 + + async def render(self, key, payload): + self.attempts += 1 + return None + + lens = NeverCoversLens() + service = ChunkCapsuleService(ctx=None, lens=lens, retry_limit=1) + service.submit("chunk:1-4", {"start_step": 1, "end_step": 4, "steps": []}) + await service.flush() + + assert lens.attempts == 2 # 1 + retry_limit, never unbounded + assert service.has_failed("chunk:1-4") + assert service.get_summary("chunk:1-4") is None + + # A chunker harvesting this key leaves the chunk pending — and under + # ready gating the segment never swaps: the original turns stay live. + steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 9)] + ledger, chunker, _, _ = _make(steps, min_active=2) + _run_turns(ledger, chunker, 1, 8, _hashes(4)) + chunker._capsule_service = service # type: ignore[attr-defined] + chunker._harvest_capsules() + assert chunker.awaiting_chunks[0].status == "pending" + assert chunker.chunks == () + assert ledger.frozen_blocks == () + + +# --------------------------------------------------------------------------- +# Capsule availability hardening (2026-09-01 A/B root cause: the dedicated +# `chunking.model` endpoint was down all day, so every capsule attempt died +# and all 21 chunk headers stayed pending at version 1 forever) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_capsule_lens_falls_back_to_secondary_model_on_provider_outage(): + """A provider outage on the primary capsule model switches the attempt to + the configured fallback model instead of failing the capsule outright.""" + + class DeadPrimary: + def __init__(self): + self.calls = 0 + + async def ainvoke(self, messages): + self.calls += 1 + raise Exception( + "503 Service Unavailable. This model is currently experiencing" + " high demand." + ) + + class HealthyFallback: + def __init__(self): + self.calls = 0 + + async def ainvoke(self, messages): + self.calls += 1 + return SimpleNamespace(content=json.dumps(_capsule(1, 2))) + + primary, fallback = DeadPrimary(), HealthyFallback() + lens = StepCapsuleLens( + model_name="dead-model", + llm=primary, + fallback_model_name="healthy-model", + fallback_llm=fallback, + ) + result = await lens.render("chunk:1-2", {"start_step": 1, "end_step": 2, "steps": []}) + assert result is not None and json.loads(result)["doing"] + assert primary.calls == 1 + assert fallback.calls == 1 + + +@pytest.mark.asyncio +async def test_capsule_outage_without_fallback_exhausts_to_failed(): + """The A/B breakage shape: one dead endpoint and no fallback fails every + attempt; the job exhausts to the explicit failed state (and the chunk + stays pending, per test_capsule_retry_exhaustion_degrades_to_pending_chunk).""" + from artemis.memory.chunking import ChunkCapsuleService + + class DeadLLM: + async def ainvoke(self, messages): + raise Exception("503 Service Unavailable") + + lens = StepCapsuleLens(model_name="dead", llm=DeadLLM()) + assert not lens.has_fallback + service = ChunkCapsuleService(ctx=None, lens=lens, retry_limit=1) + service._retry_delays = (0.0,) + service.submit("chunk:1-2", {"start_step": 1, "end_step": 2, "steps": []}) + await service.flush() + assert service.has_failed("chunk:1-2") + assert service.get_summary("chunk:1-2") is None + + +def test_capsule_fallback_model_resolution_google_only_and_not_primary(): + mgr = HistoryChunkManager( + capsule_service=StubCapsuleService(), + chunking_config=SimpleNamespace( + max_steps=12, + target_source_tokens=2000, + model="gemini-3.7-flash", + max_chunks=8, + ), + ) + + def ctx_with(provider, model): + return SimpleNamespace( + llm_config=SimpleNamespace( + summarizer=SimpleNamespace( + fallback=SimpleNamespace(provider=provider, model=model) + ) + ) + ) + + assert ( + mgr._resolve_capsule_fallback_model(ctx_with("google", "gemini-3.6-flash")) + == "gemini-3.6-flash" + ) + # Non-google fallbacks cannot ride the raw google model path. + assert mgr._resolve_capsule_fallback_model(ctx_with("openai", "gpt-4o-mini")) is None + # A fallback identical to the primary adds nothing. + assert ( + mgr._resolve_capsule_fallback_model(ctx_with("google", "gemini-3.7-flash")) is None + ) + + +@pytest.mark.asyncio +async def test_flush_harvests_late_capsule_and_persists_ready_version(): + """Short-session backstop: a capsule that becomes ready after the last + render is harvested and persisted at flush; flush itself never swaps (no + freeze at shutdown), and the next render performs the swap normally.""" + steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 9)] + ledger, chunker, engine, capsule = _make(steps, min_active=2) + _run_turns(ledger, chunker, 1, 8, _hashes(4)) + assert chunker.awaiting_chunks[0].status == "pending" + v1_writes = len(engine.chunk_writes) + + capsule.resolve("chunk:1-4", _capsule(1, 4)) + await chunker.flush(0.1) + + chunk = chunker.awaiting_chunks[0] + assert chunk.status == "ready" + assert chunk.version == 2 + ready_rows = [w for w in engine.chunk_writes[v1_writes:] if w["status"] == "ready"] + assert ready_rows and ready_rows[-1]["version"] == 2 + # Flush drains and persists but never freezes the transcript. + assert ledger.frozen_blocks == () + + # A later render (e.g. after a process resume) swaps normally. + ledger.commit_staged(step_key="s8", validator_result={"status": "success"}) + ledger.render([_observation(9)]) + assert chunker.chunks and chunker.chunks[0].status == "ready" + + +# --------------------------------------------------------------------------- +# Ready-gated swap (user decision 2026-09-01): degradation ladder +# --------------------------------------------------------------------------- + + +def test_soft_pressure_alone_never_swaps_unready_segment(): + """Soft-threshold pressure closes segments but never swaps an unready one: + correctness over tokens — the original text stays until the header lands.""" + steps = [_step(i) for i in range(1, 9)] + meter = {"value": 75_000} # ≥ 0.7 * 100k, < 0.9 * 100k + ledger, chunker, _, _ = _make( + steps, min_active=2, max_steps=100, meter=lambda: meter["value"] + ) + _run_turns(ledger, chunker, 1, 8) + + assert len(chunker.awaiting_chunks) >= 1 # soft pressure closed a portion + assert chunker.chunks == () # ...but nothing swapped + assert ledger.frozen_blocks == () # original text retained + + +def test_failed_capsule_redispatches_and_retains_original(): + """Failure rung of the ladder: an exhausted capsule job is re-dispatched at + the next trigger/pressure render while the original turns stay live.""" + steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 9)] + meter = {"value": None} + ledger, chunker, _, capsule = _make(steps, min_active=2, meter=lambda: meter["value"]) + _run_turns(ledger, chunker, 1, 8, _hashes(4)) + key = chunker.awaiting_chunks[0].capsule_key + assert capsule.submitted.count(key) == 1 + + # Bounded retries exhausted → explicit failed state, chunk stays pending. + capsule._failed.add(key) + meter["value"] = 75_000 # pressure render = a re-dispatch occasion + ledger.commit_staged(step_key="s8", validator_result={"status": "success"}) + ledger.render([_observation(9)]) + + assert capsule.submitted.count(key) == 2 # re-dispatched + assert chunker.chunks == () # still not swapped + assert ledger.frozen_blocks == () # original text retained + + +def test_hard_threshold_force_swaps_pending_chunks_into_l3(): + """Emergency path: only the hard threshold may swap pending chunks — the + frozen region becomes the L3 snapshot and ③'s minimal index survives.""" + steps = [_step(i, "hash-a" if i <= 4 else "hash-b") for i in range(1, 9)] + meter = {"value": None} + ledger, chunker, _, _ = _make(steps, min_active=2, meter=lambda: meter["value"]) + _run_turns(ledger, chunker, 1, 8, _hashes(4)) + assert chunker.awaiting_chunks[0].status == "pending" + assert ledger.frozen_blocks == () + + meter["value"] = 95_000 # ≥ 0.9 * 100k → hard + ledger.commit_staged(step_key="s8", validator_result={"status": "success"}) + ledger.render([_observation(9)]) + + assert chunker.awaiting_chunks == () + assert len(chunker.chunks) >= 1 + assert all(c.status == "pending" for c in chunker.chunks) + frozen = _frozen_text(ledger) + assert "[Session snapshot" in frozen + for chunk in chunker.chunks: + for n in range(chunk.start_step_number, chunk.end_step_number + 1): + assert f"- Step {n} (T+" in frozen + assert len(ledger.unchunked_turns()) < 8 # turns actually consumed + + +# --------------------------------------------------------------------------- +# Band ① note-linkage machine check (user decision 2026-09-01) +# --------------------------------------------------------------------------- + + +def test_capsule_note_coverage_is_machine_checked(): + """Every note key written during the segment must surface in the band-① + text; a missing key fails the attempt (regenerate), same rank as the ② + interval-coverage check. Segments without notes are unaffected.""" + lens = StepCapsuleLens(model_name="test", llm=object()) + payload = { + "start_step": 3, + "end_step": 6, + "steps": [ + { + "step_number": 3, + "note_writes": [ + {"tool": "save_note", "key": "login_flow", "gist": "code entry"} + ], + }, + { + "step_number": 5, + "note_writes": [ + {"tool": "append_note", "key": "prices", "gist": "totals"} + ], + }, + ], + } + capsule = _capsule(3, 6) # effect mentions notes/login_flow.md only + + # 'prices' is missing from every band-① field → judged failed. + assert lens.parse_capsule(json.dumps(capsule), payload) is None + + covered = dict( + capsule, + effect=capsule["effect"] + " Also appended notes/prices.md with the totals.", + ) + parsed = lens.parse_capsule(json.dumps(covered), payload) + assert parsed is not None + assert "prices" in parsed["effect"] + + # A no-notes segment keeps the original acceptance behavior. + no_notes = {"start_step": 3, "end_step": 6, "steps": []} + assert lens.parse_capsule(json.dumps(capsule), no_notes) is not None + + +# --------------------------------------------------------------------------- +# Formal lens interface (M1 difference-b closure) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_lens_backed_service_uses_lens_render(): + from artemis.memory.step_memory import StepLens + + class EchoLens(StepLens): + name = "echo" + + async def render(self, key, payload): + return f"summary-for-{key}" + + service = StepMemoryService(ctx=None, lens=EchoLens()) + service.submit("k1", {"step_number": 1}) + await service.flush() + assert service.get_summary("k1") == "summary-for-k1" + assert not service.has_failed("k1") diff --git a/tests/unit/memory/test_step_memory_service.py b/tests/unit/memory/test_step_memory_service.py new file mode 100644 index 0000000..8f7a0d8 --- /dev/null +++ b/tests/unit/memory/test_step_memory_service.py @@ -0,0 +1,187 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the shared StepMemoryService scheduling runtime (M1).""" + +import asyncio +from unittest.mock import Mock + +import pytest + +from artemis.memory.step_memory import StepMemoryService + + +class _ScriptedService(StepMemoryService): + """StepMemoryService with a scriptable attempt function and status log.""" + + def __init__(self, attempt_fn, **kwargs): + super().__init__(Mock(), **kwargs) + self._attempt_fn = attempt_fn + self.status_events: list[tuple] = [] + + async def _attempt(self, key): + return await self._attempt_fn(self, key) + + def _on_status(self, key, status): + self.status_events.append((key, status)) + + +@pytest.mark.asyncio +async def test_zero_blocking_submit_and_flush(): + """submit() returns immediately; flush() drains the background job.""" + release = asyncio.Event() + + async def attempt(svc, key): + await release.wait() + svc._summaries[key] = f"summary-for-{key}" + return True + + service = _ScriptedService(attempt) + service.submit("step-1", {"step_number": 1}) + + # Zero-blocking: the job cannot have completed yet (its gate is closed). + assert service.is_pending("step-1") + assert not service.has_summary("step-1") + + release.set() + await service.flush() + + assert service.get_summary("step-1") == "summary-for-step-1" + assert not service.is_pending("step-1") + + +@pytest.mark.asyncio +async def test_bounded_retry_enters_failed_state(): + """A permanently failing job stops after 1 + retry_limit attempts.""" + attempts = 0 + + async def attempt(svc, key): + nonlocal attempts + attempts += 1 + return False + + service = _ScriptedService(attempt, retry_limit=2) + service._retry_delays = (0.0,) + service.submit("step-x", {"step_number": 7}) + await service.flush() + + assert attempts == 3 + assert service.has_failed("step-x") + assert not service.has_summary("step-x") + # The failed job still reads as pending (lossless semantics downstream). + assert service.is_pending("step-x") + assert service.status_events == [("step-x", "pending"), ("step-x", "failed")] + + +@pytest.mark.asyncio +async def test_step_id_keying_with_tool_call_alias(): + """Jobs keyed by step id remain queryable through tool_call_id aliases.""" + + async def attempt(svc, key): + svc._summaries[key] = "canonical summary" + return True + + service = _ScriptedService(attempt) + service.submit("uuid-123", {"step_number": 4}, aliases=("tc-abc",)) + await service.flush() + + assert service.resolve_key("tc-abc") == "uuid-123" + assert service.has_job("tc-abc") + assert service.has_summary("tc-abc") + assert service.get_summary("tc-abc") == "canonical summary" + assert service.get_summary("uuid-123") == "canonical summary" + assert service.get_job_payload("tc-abc") == {"step_number": 4} + assert not service.has_failed("tc-abc") + + +@pytest.mark.asyncio +async def test_max_concurrency_bounds_parallel_attempts(): + """With max_concurrency=1, two jobs never run their attempts concurrently.""" + active = 0 + max_active = 0 + + async def attempt(svc, key): + nonlocal active, max_active + active += 1 + max_active = max(max_active, active) + await asyncio.sleep(0.01) + active -= 1 + svc._summaries[key] = "done" + return True + + service = _ScriptedService(attempt, max_concurrency=1) + service.submit("a", {"step_number": 1}) + service.submit("b", {"step_number": 2}) + await service.flush() + + assert service.has_summary("a") and service.has_summary("b") + assert max_active == 1 + + +@pytest.mark.asyncio +async def test_flush_timeout_cancels_stragglers(): + """flush() cancels jobs that outlive the timeout without failing them.""" + + async def attempt(svc, key): + await asyncio.sleep(30) + return True + + service = _ScriptedService(attempt) + service.submit("slow", {"step_number": 1}) + await service.flush(timeout_seconds=0.05) + + assert service.is_pending("slow") + assert not service.has_failed("slow") + assert service._pending_tasks["slow"].done() + + +@pytest.mark.asyncio +async def test_flush_uses_configured_default_timeout(): + """flush() with no argument honors the constructor's flush_timeout_s.""" + + async def attempt(svc, key): + await asyncio.sleep(30) + return True + + service = _ScriptedService(attempt, flush_timeout_s=0.05) + service.submit("slow", {"step_number": 1}) + await asyncio.wait_for(service.flush(), timeout=5) + + assert service._pending_tasks["slow"].done() + + +@pytest.mark.asyncio +async def test_resubmit_clears_failed_state(): + """Re-submitting a failed job gives it a fresh retry budget.""" + behaviors = {"fail": True} + + async def attempt(svc, key): + if behaviors["fail"]: + return False + svc._summaries[key] = "second time lucky" + return True + + service = _ScriptedService(attempt, retry_limit=0) + service._retry_delays = (0.0,) + service.submit("job", {"step_number": 1}) + await service.flush() + assert service.has_failed("job") + + behaviors["fail"] = False + service._retry_counts.clear() + service.submit("job", {"step_number": 1}) + await service.flush() + + assert not service.has_failed("job") + assert service.get_summary("job") == "second time lucky" diff --git a/tests/unit/memory/test_transcript_ledger.py b/tests/unit/memory/test_transcript_ledger.py new file mode 100644 index 0000000..7ed9739 --- /dev/null +++ b/tests/unit/memory/test_transcript_ledger.py @@ -0,0 +1,298 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the Pro session transcript ledger (history redesign §3.2, M2). + +Covers the four-region discipline: S-region byte stability across turns, the +append-only active region, depth-1 stripping of old UI lists and plan +recitations, the depth-K screenshot scrub in the Pro message shape (grace / +placeholder / freeze — the M1 race regressions re-run against HumanMessage +observations keyed by step id), ``T+mm:ss`` session-offset timestamps, the +tool-call/response pairing invariant, and the cold-start restored-history +block. +""" + +import json +import re + +import pytest +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage + +from artemis.memory.step_memory import StepMemoryService +from artemis.memory.transcript import ( + EXECUTION_RESULT_MARKER, + PLAN_RECITATION_MARKER, + PRO_UI_LIST_MARKER, + RESTORED_HISTORY_HEADER, + TranscriptLedger, + format_session_offset, +) + +OFFSET_RE = re.compile(r"T\+\d{2,}:\d{2}") + + +def _service() -> StepMemoryService: + return StepMemoryService(ctx=None) + + +def _observation(i: int) -> HumanMessage: + return HumanMessage( + content=[ + {"type": "text", "text": f"# CURRENT OBSERVATION [T+00:0{i % 10}]"}, + {"type": "text", "text": f"{PLAN_RECITATION_MARKER}\n- [/] milestone {i}"}, + {"type": "text", "text": "--- Current Screenshot ---"}, + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,IMG_{i}"}}, + {"type": "text", "text": f"{PRO_UI_LIST_MARKER}\n[1] button {i}"}, + ] + ) + + +def _turn(i: int, tool_call_count: int = 1) -> list: + messages: list = [_observation(i)] + if tool_call_count: + tool_calls = [ + {"name": "click", "args": {"target": n + 1}, "id": f"tc{i}-{n}", "type": "tool_call"} + for n in range(tool_call_count) + ] + messages.append(AIMessage(content=f"thinking {i}", tool_calls=tool_calls)) + for tc in tool_calls: + messages.append(ToolMessage(tool_call_id=tc["id"], content="Action Recorded")) + return messages + + +def _play_turn(ledger: TranscriptLedger, i: int, *, prev_key=None, prev_result=None, **turn_kwargs): + """Mimic the operator's real ordering: commit previous, render, stage.""" + ledger.commit_staged(step_key=prev_key, validator_result=prev_result) + turn = _turn(i, **turn_kwargs) + rendered = ledger.render([turn[0]]) + ledger.stage_turn(turn) + return rendered + + +def _fingerprint(msg) -> str: + return json.dumps(msg.content, sort_keys=True, default=str) + + +def _has_image(msg) -> bool: + return any( + isinstance(b, dict) and b.get("type") in ("image_url", "image") for b in msg.content + ) + + +def test_format_session_offset(): + assert format_session_offset(0) == "T+00:00" + assert format_session_offset(83) == "T+01:23" + # Minutes never wrap into hours: byte-stable monotonic labels. + assert format_session_offset(3725) == "T+62:05" + assert format_session_offset(-5) == "T+00:00" + + +def test_static_prefix_set_once_and_byte_stable(): + ledger = TranscriptLedger(step_memory=_service()) + system = SystemMessage(content="STATIC SYSTEM PROMPT") + ledger.set_static_prefix([system]) + + with pytest.raises(RuntimeError): + ledger.set_static_prefix([SystemMessage(content="other")]) + + fingerprints = set() + for i in range(1, 6): + rendered = _play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None) + assert rendered[0] is system + fingerprints.add(rendered[0].content) + assert fingerprints == {"STATIC SYSTEM PROMPT"} + + +def test_active_region_is_append_only(): + ledger = TranscriptLedger(step_memory=_service()) + seen_ids: list[int] = [] + for i in range(1, 5): + _play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None) + active = ledger.active_messages + # The previously observed prefix is unchanged in identity and order. + assert [id(m) for m in active[: len(seen_ids)]] == seen_ids + seen_ids = [id(m) for m in active] + assert ledger.turn_count == 3 # 3 committed, 1 still staged + + +def test_validator_result_message_carries_session_offset(): + ledger = TranscriptLedger(step_memory=_service()) + _play_turn(ledger, 1) + _play_turn( + ledger, + 2, + prev_key="step-1", + prev_result={"status": "success", "execution": [{"action": "tap", "attempts": ["ok"]}]}, + ) + + result_messages = [ + m + for m in ledger.active_messages + if isinstance(m, HumanMessage) + and str(m.content).find(EXECUTION_RESULT_MARKER) >= 0 + ] + assert len(result_messages) == 1 + text = result_messages[0].content[0]["text"] + assert OFFSET_RE.search(text), text + assert "Status: success" in text + assert "ago" not in text + + +def test_validator_result_skipped_for_actionless_turn(): + ledger = TranscriptLedger(step_memory=_service()) + _play_turn(ledger, 1) + _play_turn(ledger, 2, prev_key="step-1", prev_result=None) + assert not any( + EXECUTION_RESULT_MARKER in str(m.content) for m in ledger.active_messages + ) + + +def test_old_turn_recitation_and_ui_list_stripped_at_depth_1(): + ledger = TranscriptLedger(step_memory=_service()) + _play_turn(ledger, 1) + rendered = _play_turn(ledger, 2, prev_key="step-1") + + committed_obs = ledger.active_messages[0] + committed_text = str(committed_obs.content) + # Depth-1 scrub: the previous turn's plan recitation and UI list are gone. + assert PLAN_RECITATION_MARKER not in committed_text + assert PRO_UI_LIST_MARKER not in committed_text + # Its screenshot is still inside the K-depth active window. + assert _has_image(committed_obs) + # The live tail keeps both. + tail_text = str(rendered[-1].content) + assert PLAN_RECITATION_MARKER in tail_text + assert PRO_UI_LIST_MARKER in tail_text + + +def test_depth_k_image_resolved_to_ready_visual_summary(): + service = _service() + service._step_inputs["step-1"] = {"step_number": 1} + service._summaries["step-1"] = "Objective visual transition 1." + + ledger = TranscriptLedger(step_memory=service, image_scrub_depth=3) + for i in range(1, 5): + _play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None) + + first_obs = ledger.active_messages[0] + assert not _has_image(first_obs) + assert { + "type": "text", + "text": "--- Historical Visual Transition ---\nObjective visual transition 1.", + } in first_obs.content + + +def test_pending_grace_then_placeholder_never_backfilled(): + service = _service() + service._step_inputs["step-1"] = {"step_number": 1} # pending job, no summary + + ledger = TranscriptLedger( + step_memory=service, image_scrub_depth=2, pending_grace_steps=1 + ) + # Enough turns to push turn 1 past K + grace. + for i in range(1, 7): + _play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None) + + first_obs = ledger.active_messages[0] + assert not _has_image(first_obs) + assert "[visual summary pending; evidence at DataEngine step 1]" in str( + first_obs.content + ) + frozen = _fingerprint(first_obs) + + # A late summary must never mutate the frozen message. + service._summaries["step-1"] = "Too late." + _play_turn(ledger, 7, prev_key="step-6") + assert _fingerprint(ledger.active_messages[0]) == frozen + + +def test_failed_summary_becomes_unavailable_placeholder(): + service = _service() + service._step_inputs["step-1"] = {"step_number": 1} + service._failed.add("step-1") + + ledger = TranscriptLedger( + step_memory=service, image_scrub_depth=2, pending_grace_steps=5 + ) + for i in range(1, 4): + _play_turn(ledger, i, prev_key=f"step-{i - 1}" if i > 1 else None) + + first_obs = ledger.active_messages[0] + assert not _has_image(first_obs) + assert "[visual summary unavailable; evidence at DataEngine step 1]" in str( + first_obs.content + ) + + +def test_tool_call_response_pairs_are_never_split(): + ledger = TranscriptLedger(step_memory=_service(), image_scrub_depth=2) + for i in range(1, 7): + _play_turn( + ledger, + i, + prev_key=f"step-{i - 1}" if i > 1 else None, + prev_result={"status": "success"} if i > 1 else None, + tool_call_count=2, + ) + + active = list(ledger.active_messages) + for idx, msg in enumerate(active): + if isinstance(msg, AIMessage) and msg.tool_calls: + following = active[idx + 1 : idx + 1 + len(msg.tool_calls)] + assert [getattr(m, "tool_call_id", None) for m in following] == [ + tc["id"] for tc in msg.tool_calls + ], f"tool-call pairing split at active index {idx}" + + +def test_restored_history_only_seeds_an_empty_ledger(): + ledger = TranscriptLedger(step_memory=_service()) + ledger.set_restored_history(f"{RESTORED_HISTORY_HEADER} steps 1-9 ...") + assert ledger.has_restored_history + + with pytest.raises(RuntimeError): + ledger.set_restored_history("again") + + ledger.set_static_prefix([SystemMessage(content="S")]) + rendered = _play_turn(ledger, 1) + # Order: S region, then the frozen restored block, then the live tail. + assert rendered[0].content == "S" + assert RESTORED_HISTORY_HEADER in str(rendered[1].content) + assert "# CURRENT OBSERVATION" in str(rendered[-1].content) + + ledger2 = TranscriptLedger(step_memory=_service()) + _play_turn(ledger2, 1) + ledger2.commit_staged(step_key="step-1") + with pytest.raises(RuntimeError): + ledger2.set_restored_history("late") + + +def test_stage_twice_commits_the_forgotten_turn(): + ledger = TranscriptLedger(step_memory=_service()) + ledger.stage_turn(_turn(1)) + ledger.stage_turn(_turn(2)) + assert ledger.turn_count == 1 + assert ledger.has_staged_turn + + +def test_no_ago_wording_in_ledger_output(): + ledger = TranscriptLedger(step_memory=_service()) + for i in range(1, 4): + _play_turn( + ledger, + i, + prev_key=f"step-{i - 1}" if i > 1 else None, + prev_result={"status": "success"} if i > 1 else None, + ) + blob = " ".join(str(m.content) for m in ledger.active_messages) + assert " ago" not in blob diff --git a/tests/unit/test_memory_config.py b/tests/unit/test_memory_config.py new file mode 100644 index 0000000..f03b89f --- /dev/null +++ b/tests/unit/test_memory_config.py @@ -0,0 +1,134 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for the agent.memory configuration block (M1/M2).""" + +from artemis.config import AgentGlobalConfig + + +def test_memory_defaults(): + cfg = AgentGlobalConfig.model_validate({}) + # 2 per the 2026-09-01 on-device baseline (serial queue at 1 pushed + # summary-ready P90 past the scrub-edge grace window). + assert cfg.memory.runtime.max_concurrency == 2 + assert cfg.memory.runtime.retry_limit == 3 + assert cfg.memory.runtime.flush_timeout_s == 30.0 + # M5 (2026-09-01): the transcript path ships ON by default; explicit + # false is the byte-for-byte rollback switch (see the flip test below). + assert cfg.memory.transcript.enabled is True + assert cfg.memory.transcript.image_scrub_depth == 3 + assert cfg.memory.transcript.pending_grace_steps == 3 + assert cfg.memory.transcript.xml_scrub_depth == 1 + + +def test_memory_transcript_flag_explicit_disable_is_rollback_switch(): + cfg = AgentGlobalConfig.model_validate( + {"memory": {"transcript": {"enabled": False}}} + ) + assert cfg.memory.transcript.enabled is False + # Sibling scrub defaults are unaffected by flipping the flag. + assert cfg.memory.transcript.image_scrub_depth == 3 + + +def test_memory_explicit_values(): + cfg = AgentGlobalConfig.model_validate( + { + "memory": { + "runtime": {"max_concurrency": 4, "retry_limit": 5, "flush_timeout_s": 10}, + "transcript": { + "image_scrub_depth": 5, + "pending_grace_steps": 1, + "xml_scrub_depth": 2, + }, + } + } + ) + assert cfg.memory.runtime.max_concurrency == 4 + assert cfg.memory.runtime.retry_limit == 5 + assert cfg.memory.runtime.flush_timeout_s == 10.0 + assert cfg.memory.transcript.image_scrub_depth == 5 + assert cfg.memory.transcript.pending_grace_steps == 1 + assert cfg.memory.transcript.xml_scrub_depth == 2 + + +def test_legacy_step_summarizer_retry_limit_seeds_memory_runtime(): + """An explicit legacy flash.step_summarizer.retry_limit keeps working.""" + cfg = AgentGlobalConfig.model_validate( + {"flash": {"step_summarizer": {"retry_limit": 7}}} + ) + assert cfg.flash.step_summarizer.retry_limit == 7 + assert cfg.memory.runtime.retry_limit == 7 + + +def test_explicit_memory_runtime_wins_over_legacy_alias(): + cfg = AgentGlobalConfig.model_validate( + { + "flash": {"step_summarizer": {"retry_limit": 7}}, + "memory": {"runtime": {"retry_limit": 2}}, + } + ) + assert cfg.memory.runtime.retry_limit == 2 + # The legacy key itself is untouched for back-compat readers. + assert cfg.flash.step_summarizer.retry_limit == 7 + + +def test_legacy_default_does_not_override_memory_default(): + """A step_summarizer block without retry_limit must not seed anything.""" + cfg = AgentGlobalConfig.model_validate( + {"flash": {"step_summarizer": {"enabled": True}}} + ) + assert cfg.memory.runtime.retry_limit == 3 + + +def test_recall_and_similarity_defaults(): + """M4 defaults: recall block on, bounded; similarity hint on. The distance + threshold was calibrated to 5 in M5 (2026-09-01 on-device dHash data: + same screen <=4, different screens >=7, valley at 5-6).""" + cfg = AgentGlobalConfig() + assert cfg.memory.recall.enabled is True + assert cfg.memory.recall.max_results == 5 + assert cfg.memory.recall.max_text_tokens == 2000 + assert cfg.memory.recall.max_image_steps == 1 + assert cfg.memory.transcript.similarity_hint is True + assert cfg.memory.transcript.similarity_max_distance == 5 + assert cfg.memory.policies == {} + + +def test_chunking_era_cap_follows_max_chunks_by_default(): + """M5: max_eras is an independent key; None (default) follows max_chunks.""" + cfg = AgentGlobalConfig() + assert cfg.memory.chunking.max_chunks == 8 + assert cfg.memory.chunking.max_eras is None + + overridden = AgentGlobalConfig.model_validate( + {"memory": {"chunking": {"max_chunks": 12, "max_eras": 4}}} + ) + assert overridden.memory.chunking.max_chunks == 12 + assert overridden.memory.chunking.max_eras == 4 + + +def test_recall_and_policies_overridable(): + cfg = AgentGlobalConfig.model_validate( + { + "memory": { + "recall": {"enabled": False, "max_results": 3}, + "transcript": {"similarity_hint": False}, + "policies": {"planner": {"last_n_detailed": 3}}, + } + } + ) + assert cfg.memory.recall.enabled is False + assert cfg.memory.recall.max_results == 3 + assert cfg.memory.transcript.similarity_hint is False + assert cfg.memory.policies == {"planner": {"last_n_detailed": 3}} diff --git a/tests/unit/test_token_meter.py b/tests/unit/test_token_meter.py new file mode 100644 index 0000000..7ef5c76 --- /dev/null +++ b/tests/unit/test_token_meter.py @@ -0,0 +1,173 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the M0 session token metering pipeline (pure recording).""" + +from unittest.mock import Mock +from uuid import uuid4 + +from langchain_core.messages import AIMessage + +from artemis.services.token_meter import ( + SessionTokenMeter, + extract_usage, + get_meter, + record_llm_usage, +) + + +def _msg_with_usage(prompt=1000, completion=50, cached=0): + usage = { + "input_tokens": prompt, + "output_tokens": completion, + "total_tokens": prompt + completion, + } + if cached: + usage["input_token_details"] = {"cache_read": cached} + return AIMessage(content="ok", usage_metadata=usage) + + +def test_extract_usage_from_usage_metadata(): + usage = extract_usage(_msg_with_usage(prompt=1200, completion=80, cached=900)) + assert usage == { + "prompt_tokens": 1200, + "completion_tokens": 80, + "total_tokens": 1280, + "cached_tokens": 900, + } + + +def test_extract_usage_from_response_metadata_fallback(): + msg = AIMessage(content="ok") + msg.response_metadata = { + "usage_metadata": { + "prompt_token_count": 500, + "candidates_token_count": 25, + "total_token_count": 525, + "cached_content_token_count": 400, + } + } + usage = extract_usage(msg) + assert usage == { + "prompt_tokens": 500, + "completion_tokens": 25, + "total_tokens": 525, + "cached_tokens": 400, + } + + +def test_extract_usage_absent_returns_none(): + assert extract_usage(AIMessage(content="no usage")) is None + assert extract_usage(object()) is None + + +def test_meter_accumulates_session_totals(): + meter = SessionTokenMeter("s1") + meter.record({"prompt_tokens": 100, "completion_tokens": 10, "cached_tokens": 0}) + snapshot = meter.record( + {"prompt_tokens": 300, "completion_tokens": 20, "cached_tokens": 250} + ) + assert snapshot == { + "session_llm_calls": 2, + "session_prompt_tokens": 400, + "session_completion_tokens": 30, + "session_cached_tokens": 250, + "session_cache_hit_calls": 1, + } + assert meter.last_prompt_tokens == 300 + + +def test_record_llm_usage_records_trace_with_context_base(): + engine = Mock() + engine.current_session_id = uuid4() + engine.current_step_id = uuid4() + + payload = record_llm_usage( + engine, _msg_with_usage(prompt=2000, completion=100, cached=1500), source="google:test" + ) + + assert payload is not None + assert payload["context_base_tokens"] == 2000 + assert payload["cached_tokens"] == 1500 + assert payload["source"] == "google:test" + + engine.record_trace.assert_called_once() + kwargs = engine.record_trace.call_args.kwargs + assert kwargs["type"] == "llm_call" + assert kwargs["name"] == "llm_usage" + assert kwargs["step_id"] == engine.current_step_id + assert kwargs["payload"]["prompt_tokens"] == 2000 + + +def test_record_llm_usage_accumulates_across_calls_same_session(): + engine = Mock() + engine.current_session_id = uuid4() + engine.current_step_id = None + + record_llm_usage(engine, _msg_with_usage(prompt=100, completion=5)) + payload = record_llm_usage(engine, _msg_with_usage(prompt=250, completion=10, cached=90)) + + assert payload["session_llm_calls"] == 2 + assert payload["session_prompt_tokens"] == 350 + assert payload["session_cached_tokens"] == 90 + assert payload["session_cache_hit_calls"] == 1 + + meter = get_meter(engine.current_session_id) + assert meter.last_prompt_tokens == 250 + + +def test_record_llm_usage_never_raises_and_skips_gracefully(): + # No engine + assert record_llm_usage(None, _msg_with_usage()) is None + + # Engine without a session + engine = Mock() + engine.current_session_id = None + assert record_llm_usage(engine, _msg_with_usage()) is None + engine.record_trace.assert_not_called() + + # Response without usage + engine2 = Mock() + engine2.current_session_id = uuid4() + assert record_llm_usage(engine2, AIMessage(content="nope")) is None + engine2.record_trace.assert_not_called() + + # record_trace blowing up must not propagate + engine3 = Mock() + engine3.current_session_id = uuid4() + engine3.record_trace.side_effect = RuntimeError("boom") + assert record_llm_usage(engine3, _msg_with_usage()) is None + + +def test_record_llm_usage_lens_calls_do_not_touch_last_prompt_tokens(): + """Background lens calls accumulate totals but never overwrite the live + context base (last_prompt_tokens) consumed by the compaction thresholds.""" + engine = Mock() + engine.current_session_id = uuid4() + engine.current_step_id = None + + record_llm_usage(engine, _msg_with_usage(prompt=15000)) + meter = get_meter(engine.current_session_id) + assert meter.last_prompt_tokens == 15000 + + payload = record_llm_usage( + engine, + _msg_with_usage(prompt=300, completion=40), + source="lens:visual_transition:test-model", + update_last_prompt=False, + ) + assert payload["source"] == "lens:visual_transition:test-model" + assert meter.last_prompt_tokens == 15000 # untouched by the lens call + assert meter.llm_calls == 2 + assert meter.prompt_tokens == 15300 diff --git a/tests/unit/tools/test_history_recall.py b/tests/unit/tools/test_history_recall.py new file mode 100644 index 0000000..1588519 --- /dev/null +++ b/tests/unit/tools/test_history_recall.py @@ -0,0 +1,265 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""recall_history boundaries and search correctness (M4, design §10). + +Hard boundaries under test: bounded result count, bounded response tokens, +image recall limited to the real stored screenshots of a single step, and +every result carrying a step number. Plus: keyword/step-range/notes filter +correctness and the era-marker re-entry (a step_range returns the full-width +``build_action_ledger`` rows for that range). +""" + +import io +import re +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from PIL import Image + +from artemis.context import ArtemisContext +from artemis.data_engine.engine import DataEngine +from artemis.tools.history_recall import ( + RecallHistoryTool, + recall_history, + search_history, +) +from artemis.utils.notes import save_note_content + + +def _jpeg(color: str) -> bytes: + buf = io.BytesIO() + Image.new("RGB", (64, 128), color).save(buf, format="JPEG") + return buf.getvalue() + + +def _make_engine(tmp_path): + mock_ctx = MagicMock(spec=ArtemisContext) + mock_execution_setup = MagicMock() + mock_execution_setup.traces_path = str(tmp_path) + mock_ctx.execution_setup = mock_execution_setup + mock_ctx.device = None + engine = DataEngine(mock_ctx) + engine.start_session("recall test session") + return engine + + +def _flush(engine): + for t in list(engine._pending_threads): + t.join() + + +def _cfg(**overrides): + values = {"enabled": True, "max_results": 5, "max_text_tokens": 2000, "max_image_steps": 1} + values.update(overrides) + return SimpleNamespace(**values) + + +@pytest.fixture +def engine(tmp_path): + engine = _make_engine(tmp_path) + engine.record_step( + pre_screenshot_bytes=_jpeg("red"), + summary="Opened the login page.", + action_taken={"action": "click", "target_text": "Login entry"}, + last_execution_result={"status": "success"}, + ) + engine.record_step( + pre_screenshot_bytes=_jpeg("green"), + post_screenshot_bytes=_jpeg("blue"), + summary="Typed the verification code 4711 into the input.", + action_taken={"action": "input_text", "text": "4711"}, + last_execution_result={"status": "success"}, + operator_raw_thinking="The code arrived via SMS; entering 4711 now.", + ) + engine.record_step( + summary="A promo popup appeared and was dismissed.", + action_taken={"action": "click", "target_text": "Close popup"}, + last_execution_result={"status": "failed", "error": "element vanished"}, + ) + _flush(engine) + return engine + + +def _result_blocks(text: str) -> list[str]: + return [b for b in text.split("\n") if b.startswith("[")] + + +def test_keyword_search_finds_step_and_carries_step_number(engine): + out = search_history(engine, query="verification code", recall_config=_cfg()) + assert isinstance(out, str) + assert "Step 2" in out + assert "4711" in out + # Every result header names a step / step range / step anchor. + for header in _result_blocks(out): + assert re.search(r"Step[s]? \d", header), header + + +def test_step_range_filters_matches(engine): + out = search_history( + engine, query="popup", step_range=[1, 2], recall_config=_cfg() + ) + # Step 3 (the popup step) is outside the range: only the range ledger and + # a no-match line may appear. + assert "A promo popup appeared" not in out + assert "No matches" in out + + +def test_step_range_returns_full_width_ledger_rows(engine): + """The recall-only era period paragraph's '(Step-level ledger via + recall_history for steps a–b)' re-entry point.""" + out = search_history(engine, query="", step_range=[1, 3], recall_config=_cfg()) + assert "Action ledger for Steps 1–3" in out + # Full-width build_action_ledger rows: step number + T+mm:ss offset + + # semantic action + result phrase. + assert re.search(r"- Step 1 \(T\+\d{2}:\d{2}\): .*Login entry", out) + assert re.search(r"- Step 2 \(T\+\d{2}:\d{2}\): ", out) + assert re.search(r"- Step 3 \(T\+\d{2}:\d{2}\): ", out) + + +def test_result_count_is_clamped_by_config(engine): + for i in range(8): + engine.record_step( + summary=f"Scrolled the anchorterm feed page {i}.", + action_taken={"action": "swipe"}, + last_execution_result={"status": "success"}, + ) + _flush(engine) + out = search_history( + engine, query="anchorterm", max_results=50, recall_config=_cfg(max_results=3) + ) + assert out.count("[Step ") == 3 + assert "more not shown" in out + + +def test_response_is_truncated_at_token_budget(engine): + engine.record_step( + summary="needleword " + "filler content " * 400, + action_taken={"action": "click"}, + last_execution_result={"status": "success"}, + ) + _flush(engine) + out = search_history( + engine, + query="needleword", + include_details=True, + recall_config=_cfg(max_text_tokens=150), + ) + assert len(out) <= 150 * 4 + 200 # budget + truncation notice + assert "truncated at the recall token budget" in out + + +def test_notes_are_searchable_and_anchored_to_a_step(engine): + save_note_content(engine.base_dir, "login_flow", "The OTP entry lives behind the SMS tab.") + out = search_history(engine, query="OTP entry", recall_config=_cfg()) + assert "[Note 'login_flow'" in out + assert re.search(r"as of Step \d|last written at Step \d", out) + + +def test_chunk_rows_are_searchable(engine): + engine.record_history_chunk( + start_step_number=1, + end_step_number=2, + version=1, + status="ready", + band2=" - Steps 1–2: walked the chunkneedle flow", + band3="- Step 1 (T+00:01): click -> executed", + rendered_text=( + "[Chunk 1 | Steps 1–2]\n② Compressed step summary\n" + " - Steps 1–2: walked the chunkneedle flow\n" + "③ Step action ledger\n- Step 1 (T+00:01): click -> executed" + ), + ) + _flush(engine) + out = search_history(engine, query="chunkneedle", recall_config=_cfg()) + assert "[History chunk | Steps 1–2" in out + + +def test_images_only_for_a_single_step_and_only_real_files(engine): + out = search_history( + engine, + query="verification code", + include_images=True, + recall_config=_cfg(), + ) + assert isinstance(out, list) + images = [b for b in out if b.get("type") == "image_url"] + # Step 2 has a real pre and post screenshot on disk — but never more than + # one step's worth of images. + assert 1 <= len(images) <= 2 + labels = [b["text"] for b in out if b.get("type") == "text" and "screenshot of Step" in b.get("text", "")] + assert all("Step 2" in label for label in labels) + for block in images: + assert block["image_url"]["url"].startswith("data:image/jpeg;base64,") + + +def test_images_disabled_by_config_cap(engine): + out = search_history( + engine, + query="verification code", + include_images=True, + recall_config=_cfg(max_image_steps=0), + ) + assert isinstance(out, str) + + +def test_missing_image_files_yield_note_not_blocks(engine): + # Step 3 recorded no screenshots at all. + out = search_history( + engine, + query="promo popup", + include_images=True, + recall_config=_cfg(), + ) + assert isinstance(out, list) + assert not [b for b in out if b.get("type") == "image_url"] + assert any( + "no stored screenshots exist" in b.get("text", "") for b in out if b.get("type") == "text" + ) + + +@pytest.mark.asyncio +async def test_execute_requires_query_or_range(engine): + tool = RecallHistoryTool() + ctx = SimpleNamespace(data_engine=engine) + out = await tool.execute(ctx=ctx, query="", step_range=None) + assert "needs a query and/or a step_range" in out + + +@pytest.mark.asyncio +async def test_execute_without_engine_degrades(): + tool = RecallHistoryTool() + out = await tool.execute(ctx=SimpleNamespace(data_engine=None), query="x") + assert "no active DataEngine session" in out + + +def test_availability_requires_engine_and_config(): + assert recall_history.is_available(None) is False + assert recall_history.is_available(SimpleNamespace(data_engine=None)) is False + assert recall_history.is_available(SimpleNamespace(data_engine=object())) is True + +def test_foreground_app_stamp_joins_search_surface(tmp_path): + """M5: a package-name query hits the record_step foreground_app stamp.""" + engine = _make_engine(tmp_path) + engine.record_step( + summary="Opened the alarms list.", + action_taken={"action": "click", "target_text": "Alarm"}, + ui_tree=[{"package": "com.google.android.deskclock", "bounds": "[0,0][9,9]"}], + ) + _flush(engine) + + out = search_history(engine, query="deskclock", recall_config=_cfg()) + assert "Step 1" in out + assert "No matches" not in out diff --git a/tests/unit/utils/test_element_hit_test.py b/tests/unit/utils/test_element_hit_test.py new file mode 100644 index 0000000..d5f3ca3 --- /dev/null +++ b/tests/unit/utils/test_element_hit_test.py @@ -0,0 +1,116 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for record-time action semantics enrichment (element hit testing).""" + +from artemis.utils.element_hit_test import find_element_at_point, hit_test_semantics + + +def _xml_el(text, bounds, resource_id=None, cls="android.widget.TextView"): + return { + "text": text, + "bounds": bounds, + "class": cls, + "resource_id": resource_id, + "is_ocr": False, + } + + +def test_smallest_covering_element_wins(): + elements = [ + _xml_el("Whole screen", [0, 0, 1080, 2400]), + _xml_el("Card", [100, 100, 500, 500]), + _xml_el("Button", [200, 200, 300, 260], resource_id="btn_ok"), + ] + el, source = find_element_at_point(elements, 250, 230) + assert source == "hit_test" + assert el["text"] == "Button" + + semantics = hit_test_semantics(elements, 250, 230) + assert semantics == { + "target_label_source": "hit_test", + "target_text": "Button", + "target_class": "android.widget.TextView", + "target_resource_id": "btn_ok", + } + + +def test_xml_element_preferred_over_smaller_ocr_element(): + elements = [ + _xml_el("Login", [100, 100, 500, 300]), + { + "text": "Login OCR", + "bounds": [200, 150, 260, 180], + "class": None, + "resource_id": None, + "is_ocr": True, + }, + ] + el, source = find_element_at_point(elements, 220, 160) + assert source == "hit_test" + assert el["text"] == "Login" + + +def test_ocr_fallback_when_no_xml_element_covers_point(): + elements = [ + _xml_el("Elsewhere", [800, 800, 900, 900]), + { + "text": "OCR Label", + "bounds": [100, 100, 300, 200], + "class": None, + "resource_id": None, + "is_ocr": True, + }, + ] + semantics = hit_test_semantics(elements, 150, 150) + assert semantics["target_label_source"] == "ocr" + assert semantics["target_text"] == "OCR Label" + assert "target_class" not in semantics + assert "target_resource_id" not in semantics + + +def test_string_bounds_are_parsed(): + elements = [_xml_el("Item", "[100,100][300,500]")] + el, source = find_element_at_point(elements, 200, 300) + assert source == "hit_test" + assert el["text"] == "Item" + + +def test_graceful_degradation_to_none(): + # No elements at all + assert hit_test_semantics(None, 100, 100) == {"target_label_source": "none"} + assert hit_test_semantics([], 100, 100) == {"target_label_source": "none"} + + # Point outside every element + elements = [_xml_el("Item", [0, 0, 50, 50])] + assert hit_test_semantics(elements, 500, 500) == {"target_label_source": "none"} + + # Malformed element data must not raise + malformed = [ + {"text": "no bounds"}, + {"text": "bad bounds", "bounds": "garbage"}, + {"text": "short bounds", "bounds": [1, 2]}, + {"text": "non-numeric", "bounds": ["a", "b", "c", "d"]}, + "not-a-dict", + None, + ] + assert hit_test_semantics(malformed, 10, 10) == {"target_label_source": "none"} + + +def test_element_without_optional_fields_only_reports_source(): + elements = [ + {"text": "", "bounds": [0, 0, 100, 100], "class": None, "resource_id": None} + ] + semantics = hit_test_semantics(elements, 50, 50) + assert semantics == {"target_label_source": "hit_test"} diff --git a/tests/unit/utils/test_image_hash.py b/tests/unit/utils/test_image_hash.py new file mode 100644 index 0000000..f32cadc --- /dev/null +++ b/tests/unit/utils/test_image_hash.py @@ -0,0 +1,81 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Perceptual-hash stability and distance behavior (M4 similarity hint).""" + +import io + +from PIL import Image, ImageDraw + +from artemis.utils.image_hash import HASH_SIZE, dhash_hex, hamming_distance_hex + + +def _screen(draw_fn, size=(270, 600), quality=85) -> bytes: + img = Image.new("RGB", size, "white") + draw_fn(ImageDraw.Draw(img)) + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=quality) + return buf.getvalue() + + +def _login_screen(d): + d.rectangle([20, 40, 250, 90], fill="#3355ff") + d.rectangle([30, 200, 240, 250], outline="#333333", width=4) + d.rectangle([30, 300, 240, 350], outline="#333333", width=4) + d.rectangle([60, 450, 210, 510], fill="#22aa66") + + +def _settings_screen(d): + for y in range(60, 560, 80): + d.rectangle([10, y, 260, y + 50], fill="#dddddd") + d.ellipse([200, 20, 250, 45], fill="#aa2222") + + +def test_dhash_is_stable_for_identical_bytes(): + img = _screen(_login_screen) + assert dhash_hex(img) == dhash_hex(img) + assert len(dhash_hex(img)) == HASH_SIZE * HASH_SIZE // 4 + + +def test_dhash_survives_jpeg_reencoding_with_small_distance(): + original = _screen(_login_screen, quality=95) + with Image.open(io.BytesIO(original)) as img: + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=60) + reencoded = buf.getvalue() + + distance = hamming_distance_hex(dhash_hex(original), dhash_hex(reencoded)) + assert distance is not None and distance <= 4 + + +def test_dhash_separates_different_screens(): + distance = hamming_distance_hex( + dhash_hex(_screen(_login_screen)), dhash_hex(_screen(_settings_screen)) + ) + assert distance is not None and distance > 8 + + +def test_dhash_best_effort_on_garbage(): + assert dhash_hex(None) is None + assert dhash_hex(b"") is None + assert dhash_hex(b"not an image at all") is None + + +def test_hamming_distance_edge_cases(): + assert hamming_distance_hex(None, "ff") is None + assert hamming_distance_hex("ff", None) is None + assert hamming_distance_hex("ff", "ffff") is None # length mismatch + assert hamming_distance_hex("zz", "aa") is None # not hex + assert hamming_distance_hex("00ff", "00ff") == 0 + assert hamming_distance_hex("0000", "000f") == 4 diff --git a/tests/unit/utils/test_task_tree.py b/tests/unit/utils/test_task_tree.py index 2d6c941..a3d5ddf 100644 --- a/tests/unit/utils/test_task_tree.py +++ b/tests/unit/utils/test_task_tree.py @@ -365,7 +365,7 @@ def test_build_plan_and_history_concatenates_thoughts(): assert " - Raw thought" in output -def test_build_plan_and_history_chronological_last_step(): +def test_build_plan_and_history_interleaved_decision_loop(): plan = "- [ ] Subgoal 1" steps = [ { @@ -390,9 +390,7 @@ def test_build_plan_and_history_chronological_last_step(): ], } ] - output = build_plan_and_history( - plan, steps, "default", last_n_detailed=1, chronological_last_step=True - ) + output = build_plan_and_history(plan, steps, "default", last_n_detailed=1) assert plan in output assert "--- Execution History ---" in output @@ -464,9 +462,7 @@ def test_build_plan_and_history_safety_net_and_failure_analyzer(): ], } ] - output = build_plan_and_history( - plan, steps, "default", last_n_detailed=1, chronological_last_step=True - ) + output = build_plan_and_history(plan, steps, "default", last_n_detailed=1) assert "- **Step 1 (Most Recent Step, Start: 5.0s)**" in output assert "* [Operator Decision Loop]:" in output