feat: harden video analysis and runtime resilience

artemis/agents/flash/context_compressor.py

@@ -14,9 +14,9 @@
 
 """Universal Context Compressor for Artemis Flash profile.
 
-Replaces pruned historical screenshots with high-density visual step summaries
-when ready, while seamlessly falling back to standard pruned action text if
-the background summary is still in flight.
+Replaces historical screenshots with high-density visual step summaries only
+after they are ready. Screenshots remain in context while summarization or
+retry is still in flight.
 """
 
 from typing import Any
@@ -28,6 +28,17 @@ from artemis.utils.logger import get_logger
 
 logger = get_logger(__name__)
 
+_HISTORY_SUMMARY_PREFIX = "--- Historical Visual Transition ---\n"
+_UI_LIST_MARKER = "--- UI Element List ---"
+
+
+def _without_historical_ui_list(text: str) -> str:
+    """Remove a historical UI list without discarding text before its marker."""
+    marker_index = text.find(_UI_LIST_MARKER)
+    if marker_index < 0:
+        return text
+    return text[:marker_index].rstrip()
+
 
 def compress_flash_messages(
     messages: list[BaseMessage],
@@ -40,8 +51,8 @@ def compress_flash_messages(
     1. Keeps the latest screenshot and live UI hierarchy completely intact.
     2. For earlier steps, if a VisualStepSummarizer has completed the step's summary,
        replaces the intermediate image block with the objective summary text.
-    3. If the background summary is not ready yet, gracefully falls back to the existing
-       behavior (omitting the image block and retaining the original action text).
+    3. If the background summary is not ready yet, retains the original image until
+       a successful summary can replace it.
     4. If prune_history_xml is True, removes heavy outdated UI Element lists from past turns.
     """
     if not messages:
@@ -61,7 +72,9 @@ def compress_flash_messages(
                 last_img_msg_idx = idx
                 break
 
-    # 2. Iterate through historical messages and apply compression/fallback
+    # 2. Iterate through historical messages and apply compression/fallback.
+    # ``tool_call_id`` is the stable identity of an action result. The legacy
+    # ordinal is retained only for summaries produced by older callers/tests.
     tool_step_counter = 0
     for idx in range(len(messages)):
         msg = messages[idx]
@@ -74,43 +87,77 @@ def compress_flash_messages(
             continue
 
         # Count tool steps strictly corresponding to FlashRunner action turns
-        is_tool_msg = getattr(msg, "tool_call_id", None) is not None
+        tool_call_id = getattr(msg, "tool_call_id", None)
+        is_tool_msg = tool_call_id is not None
         if is_tool_msg:
             tool_step_counter += 1
-            current_step_num = tool_step_counter
+            legacy_step_num = tool_step_counter
         else:
-            current_step_num = None
+            legacy_step_num = None
 
-        new_blocks: list[dict[str, Any]] = []
-        summary_injected = False
+        summary_text = None
+        summary_pending = False
+        if summarizer and is_tool_msg:
+            # New FlashRunner instances key summaries by tool_call_id, so generic
+            # tools, no-tool turns, and parallel tool calls cannot shift memories.
+            action_key = str(tool_call_id)
+            summary_text = summarizer.get_summary(action_key)
+            summary_pending = summarizer.is_pending(action_key)
+            if (
+                summary_text is None
+                and not summarizer.has_job(action_key)
+                and legacy_step_num is not None
+            ):
+                summary_text = summarizer.get_summary(legacy_step_num)
+                summary_pending = summarizer.is_pending(legacy_step_num)
+
+        new_blocks: list[Any] = []
 
         for b in msg_content:
             if not isinstance(b, dict):
+                # LangChain permits string parts in multimodal content lists.
+                # They are lightweight historical evidence and must not vanish.
+                new_blocks.append(b)
                 continue
             b_type = b.get("type", "")
             b_text = b.get("text", "")
 
-            # Filter heavy outdated XML hierarchy from past steps to save tokens
-            if prune_history_xml and b_type == "text" and "--- UI Element List ---" in b_text:
+            if b_type == "text" and b_text.startswith(_HISTORY_SUMMARY_PREFIX):
+                # Rebuild this generated block on every pass. This makes the
+                # compressor idempotent and lets a late summary replace itself.
+                continue
+
+            # Filter heavy outdated XML hierarchy from past steps to save tokens.
+            # Some adapters combine action text and the hierarchy in one block;
+            # retain the useful prefix instead of dropping the entire block.
+            if prune_history_xml and b_type == "text" and _UI_LIST_MARKER in b_text:
+                retained_text = _without_historical_ui_list(b_text)
+                if retained_text:
+                    retained_block = dict(b)
+                    retained_block["text"] = retained_text
+                    new_blocks.append(retained_block)
                 continue
 
             if b_type in ("image_url", "image"):
-                # Check if this is a tool execution step and background summarizer has a ready summary
-                if (
-                    current_step_num
-                    and summarizer
-                    and summarizer.has_summary(current_step_num)
-                    and not summary_injected
-                ):
-                    summary_text = summarizer.get_summary(current_step_num)
-                    if summary_text:
-                        new_blocks.append({"type": "text", "text": summary_text})
-                        summary_injected = True
-                # If summarizer is not ready or initial screen, gracefully omit the image block
+                # A submitted image remains visible until its summary is ready.
+                # This temporarily favors context size over losing visual evidence.
+                if summary_pending:
+                    new_blocks.append(b)
                 continue
 
             new_blocks.append(b)
 
+        # Do not tie injection to the presence of an image: the image is normally
+        # pruned before the background summary finishes. A stable tool-call key lets
+        # a later compression pass backfill the summary into the correct result.
+        if summary_text:
+            new_blocks.append(
+                {
+                    "type": "text",
+                    "text": f"{_HISTORY_SUMMARY_PREFIX}{summary_text}",
+                }
+            )
+
         # Defensive fallback: never leave ToolMessage or HumanMessage content empty
         if not new_blocks:
             new_blocks = [{"type": "text", "text": "Action completed." if is_tool_msg else ""}]

artemis/agents/flash/runner.py

@@ -155,6 +155,7 @@ class FlashRunner:
         ]
 
         turns = 0
+        action_sequence = 0
         final_report = None
         current_pre_screenshot_bytes = img_bytes
         current_xml_list = xml_list
@@ -186,15 +187,13 @@ class FlashRunner:
                 except Exception as e:
                     logger.warning(f"Failed to check injected instruction in FlashRunner: {e}")
 
-            # Compress previous history before LLM call to save context and inject objective summaries
-            if self.summarizer:
-                compress_flash_messages(
-                    messages,
-                    summarizer=self.summarizer,
-                    prune_history_xml=self.step_summarizer_cfg.prune_history_xml,
-                )
-            else:
-                prune_intermediate_screenshots(messages)
+            # Compress history even when visual summarization is disabled, so
+            # screenshot and historical XML pruning have identical semantics.
+            compress_flash_messages(
+                messages,
+                summarizer=self.summarizer,
+                prune_history_xml=self.step_summarizer_cfg.prune_history_xml,
+            )
 
             # Tool restriction on the final turn
             if turns == self.max_turns:
@@ -418,6 +417,7 @@ class FlashRunner:
                             )
 
                     # Record telemetry / step in DataEngine
+                    recorded_step_id = None
                     if self.ctx.data_engine and name in ACTION_TOOL_NAMES:
                         try:
                             if self.ctx.data_engine.current_step_id is None:
@@ -475,7 +475,7 @@ class FlashRunner:
                                 action_dict["normalized_start_coordinates"] = norm_start
                                 action_dict["normalized_end_coordinates"] = norm_end
 
-                            self.ctx.data_engine.record_step(
+                            recorded_step_id = self.ctx.data_engine.record_step(
                                 pre_screenshot_bytes=current_pre_screenshot_bytes,
                                 post_screenshot_bytes=post_img_bytes,
                                 ui_tree=(exec_result.ui_elements_text or current_xml_list),
@@ -489,13 +489,16 @@ class FlashRunner:
 
                     # ⚡ Non-blocking dispatch of objective visual transition summarizer
                     if self.summarizer and name in ACTION_TOOL_NAMES:
+                        action_sequence += 1
                         self.summarizer.dispatch(
-                            step_number=turns,
+                            step_number=action_sequence,
                             action_name=name,
                             action_args=args,
                             pre_img_bytes=current_pre_screenshot_bytes,
                             post_img_bytes=post_img_bytes,
                             exec_outcome=exec_result.text_summary,
+                            action_key=str(tc_id),
+                            data_engine_step_id=recorded_step_id,
                         )
 
                     if post_img_bytes:

artemis/agents/flash/summarizer.py

@@ -16,13 +16,14 @@
 
 Processes Before/After screenshot pairs and action metadata in the background
 to generate high-density, strictly objective visual state transition summaries.
-Ensures zero blocking latency for the main runner and includes out-of-order self-healing retries.
+Ensures zero blocking latency for the main runner and independently retries failed jobs.
 """
 
 import asyncio
 import base64
 from pathlib import Path
 from typing import Any
+from uuid import UUID
 
 from jinja2 import Template
 from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
@@ -40,18 +41,17 @@ class VisualStepSummarizer:
 
     Key design properties:
     1. Zero-blocking dispatch: The main Flash runner dispatches and proceeds immediately.
-    2. Graceful fallback: If a step summary is not ready, it cleanly falls back to the original action text.
-    3. Self-healing re-trigger: If a previous step's summary is stalled or lost while subsequent steps
-       complete, it automatically re-dispatches the stalled step.
+    2. Lossless pending state: The compressor retains the source image until its summary is ready.
+    3. Independent retry: Every action owns a retry loop and does not depend on later actions arriving.
     """
 
     def __init__(self, ctx: ArtemisContext, model_name: str | None = None):
         self.ctx = ctx
-        self._summaries: dict[int, str] = {}
-        self._pending_tasks: dict[int, asyncio.Task] = {}
-        self._step_inputs: dict[int, dict[str, Any]] = {}
-        self._retry_counts: dict[int, int] = {}
-        self._max_retries = 2
+        self._summaries: dict[int | str, str] = {}
+        self._pending_tasks: dict[int | str, asyncio.Task] = {}
+        self._step_inputs: dict[int | str, dict[str, Any]] = {}
+        self._retry_counts: dict[int | str, int] = {}
+        self._retry_delays = (0.0, 0.5, 1.0, 2.0, 3.0)
 
         # Initialize lightweight VLM: prioritize explicit model_name
         target_model = model_name or "gemini-2.5-flash-lite"
@@ -83,44 +83,50 @@ class VisualStepSummarizer:
         pre_img_bytes: bytes | None,
         post_img_bytes: bytes | None,
         exec_outcome: str,
+        *,
+        action_key: str | None = None,
+        data_engine_step_id: UUID | str | None = None,
     ) -> None:
         """Dispatches an asynchronous summarization task without blocking the caller."""
-        self._step_inputs[step_number] = {
+        key: int | str = action_key or step_number
+        self._step_inputs[key] = {
+            "step_number": step_number,
             "action_name": action_name,
             "action_args": action_args,
             "pre_img_bytes": pre_img_bytes,
             "post_img_bytes": post_img_bytes,
             "exec_outcome": exec_outcome,
+            "data_engine_step_id": data_engine_step_id,
         }
 
-        task = asyncio.create_task(self._run_summary(step_number))
-        self._pending_tasks[step_number] = task
+        task = asyncio.create_task(self._run_summary_until_ready(key))
+        self._pending_tasks[key] = task
 
-        # Check for and re-trigger any previous stalled steps
-        self._check_and_retrigger_stalled_steps(current_step=step_number)
+    async def _run_summary_until_ready(self, action_key: int | str) -> None:
+        """Retry one action independently until a non-empty summary is committed."""
+        while action_key not in self._summaries:
+            if await self._run_summary_once(action_key):
+                return
 
-    def _check_and_retrigger_stalled_steps(self, current_step: int) -> None:
-        """Re-dispatches any previous steps that remain unsummarized while later steps arrive."""
-        for past_step in range(1, current_step):
-            if past_step not in self._summaries and past_step in self._step_inputs:
-                task = self._pending_tasks.get(past_step)
-                retries = self._retry_counts.get(past_step, 0)
-                if (task is None or task.done()) and retries < self._max_retries:
-                    self._retry_counts[past_step] = retries + 1
-                    logger.info(
-                        f"VisualStepSummarizer: Retriggering stalled summary for Step {past_step} "
-                        f"(attempt {self._retry_counts[past_step] + 1})"
-                    )
-                    self._pending_tasks[past_step] = asyncio.create_task(
-                        self._run_summary(past_step)
-                    )
+            retry_count = self._retry_counts.get(action_key, 0) + 1
+            self._retry_counts[action_key] = retry_count
+            delay = self._retry_delays[min(retry_count - 1, len(self._retry_delays) - 1)]
+            input_data = self._step_inputs.get(action_key)
+            display_step = input_data.get("step_number") if input_data else action_key
+            logger.info(
+                f"VisualStepSummarizer: Retrying Step {display_step} "
+                f"(attempt {retry_count + 1}) after {delay:.1f}s"
+            )
+            if delay:
+                await asyncio.sleep(delay)
 
-    async def _run_summary(self, step_number: int) -> None:
-        """Executes the lightweight VLM call to summarize the step transition."""
-        input_data = self._step_inputs.get(step_number)
+    async def _run_summary_once(self, action_key: int | str) -> bool:
+        """Execute one lightweight VLM attempt for a step transition."""
+        input_data = self._step_inputs.get(action_key)
         if not input_data:
-            return
+            return False
 
+        step_number = input_data["step_number"]
         action_name = input_data["action_name"]
         action_args = input_data["action_args"]
         pre_bytes = input_data["pre_img_bytes"]
@@ -183,44 +189,60 @@ class VisualStepSummarizer:
 
             summary_text = summary_raw.strip()
             if summary_text:
-                self._summaries[step_number] = summary_text
+                self._summaries[action_key] = summary_text
                 logger.info(
                     f"VisualStepSummarizer: Generated summary for Step {step_number}: {summary_text[:80]}..."
                 )
 
                 # Free binary image buffers from memory once summary is secured
-                if step_number in self._step_inputs:
-                    self._step_inputs[step_number]["pre_img_bytes"] = None
-                    self._step_inputs[step_number]["post_img_bytes"] = None
+                if action_key in self._step_inputs:
+                    self._step_inputs[action_key]["pre_img_bytes"] = None
+                    self._step_inputs[action_key]["post_img_bytes"] = None
 
                 # Update DataEngine telemetry if active
                 if self.ctx.data_engine:
                     try:
-                        self.ctx.data_engine.update_step_summary(step_number, summary_text)
+                        target_step = input_data.get("data_engine_step_id") or step_number
+                        self.ctx.data_engine.update_step_summary(target_step, summary_text)
                     except Exception as de_err:
                         logger.debug(f"DataEngine step summary update skipped: {de_err}")
+                return True
 
         except Exception as e:
             logger.warning(
                 f"VisualStepSummarizer: Error generating summary for step {step_number}: {e}"
             )
-            if self._retry_counts.get(step_number, 0) >= self._max_retries:
-                # Max retries reached, free memory buffer
-                if step_number in self._step_inputs:
-                    self._step_inputs[step_number]["pre_img_bytes"] = None
-                    self._step_inputs[step_number]["post_img_bytes"] = None
+        return False
 
-    def get_summary(self, step_number: int, fallback_text: str | None = None) -> str | None:
-        """Retrieves summarized state for step_number, or fallback_text if not ready yet."""
-        return self._summaries.get(step_number, fallback_text)
+    def get_summary(
+        self, action_key: int | str, fallback_text: str | None = None
+    ) -> str | None:
+        """Retrieve the summary for a stable action key, or fallback text."""
+        return self._summaries.get(action_key, fallback_text)
 
-    def has_summary(self, step_number: int) -> bool:
-        """Checks if a step summary has completed."""
-        return step_number in self._summaries
+    def has_summary(self, action_key: int | str) -> bool:
+        """Check whether an action summary has completed."""
+        return action_key in self._summaries
 
-    async def flush(self) -> None:
-        """Flushes and awaits all pending background summary tasks upon completion."""
+    def has_job(self, action_key: int | str) -> bool:
+        """Check whether an action has been submitted for visual summarization."""
+        return action_key in self._step_inputs
+
+    def is_pending(self, action_key: int | str) -> bool:
+        """Check whether an action still needs a summary."""
+        return self.has_job(action_key) and not self.has_summary(action_key)
+
+    async def flush(self, timeout_seconds: float = 30.0) -> None:
+        """Wait for active jobs up to a bound, then cancel remaining retry loops."""
         tasks = [t for t in self._pending_tasks.values() if not t.done()]
         if tasks:
             logger.info(f"VisualStepSummarizer: Flushing {len(tasks)} pending summary tasks...")
-            await asyncio.gather(*tasks, return_exceptions=True)
+            _, pending = await asyncio.wait(tasks, timeout=timeout_seconds)
+            if pending:
+                logger.warning(
+                    f"VisualStepSummarizer: Cancelling {len(pending)} unfinished summary tasks "
+                    f"after {timeout_seconds:.1f}s flush timeout."
+                )
+                for task in pending:
+                    task.cancel()
+                await asyncio.gather(*pending, return_exceptions=True)

artemis/agents/operator/prompts.py

@@ -58,6 +58,17 @@ _LEGACY_TROUBLESHOOTER_CHALLENGE = """   - *Challenge Checker*: If you visually
 _TROUBLESHOOTER_CHALLENGE = """   - *Challenge Checker*: If current evidence clearly proves the subgoal is already achieved, invoke `reply_to_checker` directly. This branch takes precedence over diagnosis; do not call `ask_diagnoser` first.
    - *Clear Local Correction*: If the rejection cause is explicit and a safe correction is evident from the current screen, perform that correction directly. This branch also takes precedence over generic diagnosis triggers. Use `ask_diagnoser` only for ambiguity, conflicting evidence, or repeated failure."""
 
+_LEGACY_LARGE_LIST_STRATEGY = """- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:
+  1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor. Calibrate your scroll distance so that this anchor remains visible near the top of the subsequent screen, creating a seamless visual overlap that guarantees zero missed items. Terminate exploration when a definitive boundary is reached (e.g., list bottom reached or no new content appears after scrolling).
+  2. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.
+  3. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for "double-checking". Only perform incremental re-verification if the list has been refreshed or content has mutated, while maintaining strict single-pass handling of new events."""
+
+_LARGE_LIST_STRATEGY = """- **Large List & Long Text Exploration Strategy (Zero-Miss, Zero-Duplication Single-Pass Principle)**: When traversing large lists, browsing long text bodies, or scanning continuous content feeds, strictly adhere to the single-pass exploration standard:
+  1. **Unidirectional Anchor-Based Traversal (Zero-Miss)**: Maintain a consistent, unidirectional scan (e.g., uniform downward scrolling). Before each scroll, identify the bottom-most visible item/text segment as your visual anchor and carry that anchor plus the scan direction in short-term memory or the traversal ledger. After the scroll, evaluate this handoff before doing anything unrelated: if the anchor remains visible with new content beyond it, record the new content and continue in the same direction without scrolling back to verify; if the anchor is missing, continuity is uncertain, so perform at most one minimal reverse recovery to re-establish overlap, then resume the original direction. While the traversal's declared exit condition remains unresolved, keep this traversal active and do not advance to another milestone or mark it complete.
+  2. **Boundary-Proven Completion**: A single scroll that reveals no new content is not sufficient evidence that the list is complete. For an exhaustive traversal, stop only when an explicit end-of-list indicator is visible, or when one additional successful swipe in the same direction and within the same list container leaves the same final anchor visible with no new items and no loading state. This boundary probe must remain unidirectional; never reverse direction merely to prove completion. If the task declares an earlier exit condition (for example, find one verified match or process N items), that explicit condition may end the traversal without reaching the list boundary.
+  3. **Memory & Plan-Driven Deduplication (Zero-Duplication)**: Systematically log extracted item names, identifiers, or processing states into notes or memory as you progress. After every screen transition, cross-reference visible items against your recorded ledger and strictly interact only with newly surfaced, unrecorded items to eliminate redundant actions on overlapping elements. Proactively maintain subtasks in `task_plan` (e.g., segmenting by batch, category, or alphabetical ranges like `a-d`), tracking exact paths and progress to prevent backtracking.
+  4. **Single-Pass Trust & Anti-Oscillation (No Over-Verification)**: Treat recorded notes as verified ground truth. Once a segment is traversed with confirmed anchor continuity and logged, consider it permanently resolved. Do NOT scroll back-and-forth or perform redundant re-scans for "double-checking", even if older execution history has been pruned. Resume from the last recorded anchor and direction. Only the single bounded recovery above, or a refreshed/mutated list, permits limited re-verification; otherwise continue the single pass over new content."""
+
 
 def apply_operator_prompt_contract(prompt_template: str) -> str:
     """Align prompt-level tool and recovery rules with Operator runtime semantics."""
@@ -77,6 +88,10 @@ def apply_operator_prompt_contract(prompt_template: str) -> str:
         _LEGACY_TROUBLESHOOTER_CHALLENGE,
         _TROUBLESHOOTER_CHALLENGE,
     )
+    prompt_template = prompt_template.replace(
+        _LEGACY_LARGE_LIST_STRATEGY,
+        _LARGE_LIST_STRATEGY,
+    )
     return prompt_template
 
 

artemis/agents/video_analyzer/blackboard.py

@@ -0,0 +1,821 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Persistent, video-scoped memory for video analysis agents."""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from dataclasses import dataclass
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import sqlite3
+import threading
+import time
+from typing import Any, Iterator, Literal
+from uuid import uuid4
+
+from artemis.context import ArtemisContext
+from artemis.utils.logger import get_logger
+from artemis.utils.video import get_active_session
+
+logger = get_logger(__name__)
+
+
+ClaimState = Literal["claimed", "cached", "in_progress"]
+
+
+@dataclass(frozen=True)
+class SegmentClaim:
+    """Result of atomically attempting to acquire one analysis segment."""
+
+    state: ClaimState
+    lease_owner: str | None = None
+    summary: str | None = None
+    analysis: str | None = None
+
+
+def _normalise_query(query: str | None) -> str:
+    return " ".join(str(query or "").strip().lower().split())
+
+
+def _query_key(query: str | None) -> str:
+    return hashlib.sha256(_normalise_query(query).encode("utf-8")).hexdigest()
+
+
+def _to_ms(value: float | int) -> int:
+    return int(round(float(value) * 1000.0))
+
+
+def _to_seconds(value: int) -> float:
+    return round(value / 1000.0, 3)
+
+
+class VideoBlackboard:
+    """SQLite-backed blackboard with a context-local in-memory fallback."""
+
+    def __init__(
+        self,
+        board_key: str,
+        *,
+        db_path: Path | None = None,
+        evidence_dir: Path | None = None,
+        lease_seconds: float = 300.0,
+    ) -> None:
+        self.board_key = board_key
+        self.db_path = Path(db_path) if db_path is not None else None
+        self.evidence_dir = Path(evidence_dir) if evidence_dir is not None else None
+        self.lease_seconds = max(0.01, float(lease_seconds))
+        self._lock = threading.RLock()
+        self._memory_observations: dict[str, dict[str, Any]] = {}
+        self._memory_segments: dict[tuple[str, str, int, int], dict[str, Any]] = {}
+
+        if self.db_path is not None:
+            self.db_path.parent.mkdir(parents=True, exist_ok=True)
+            self._init_db()
+        if self.evidence_dir is not None:
+            self.evidence_dir.mkdir(parents=True, exist_ok=True)
+
+    @contextmanager
+    def _connection(self) -> Iterator[sqlite3.Connection]:
+        if self.db_path is None:
+            raise RuntimeError("SQLite connection requested for in-memory blackboard")
+        with self._lock:
+            conn = sqlite3.connect(self.db_path, timeout=30.0)
+            conn.row_factory = sqlite3.Row
+            try:
+                yield conn
+            finally:
+                conn.close()
+
+    def _init_db(self) -> None:
+        with self._connection() as conn:
+            conn.execute("PRAGMA journal_mode=WAL")
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS video_analysis_segments (
+                    board_key TEXT NOT NULL,
+                    query_key TEXT NOT NULL,
+                    query_text TEXT NOT NULL,
+                    modality TEXT NOT NULL,
+                    start_ms INTEGER NOT NULL,
+                    end_ms INTEGER NOT NULL,
+                    status TEXT NOT NULL,
+                    lease_owner TEXT,
+                    lease_expires_at REAL,
+                    attempt_count INTEGER NOT NULL DEFAULT 0,
+                    summary TEXT,
+                    analysis TEXT,
+                    error TEXT,
+                    error_category TEXT,
+                    model_name TEXT,
+                    source_generation INTEGER,
+                    created_at REAL NOT NULL,
+                    updated_at REAL NOT NULL,
+                    PRIMARY KEY (board_key, query_key, modality, start_ms, end_ms)
+                )
+                """
+            )
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS video_analysis_observations (
+                    observation_id TEXT PRIMARY KEY,
+                    board_key TEXT NOT NULL,
+                    start_ms INTEGER NOT NULL,
+                    end_ms INTEGER,
+                    query_text TEXT NOT NULL,
+                    summary TEXT NOT NULL,
+                    confidence REAL,
+                    modality TEXT NOT NULL,
+                    screenshot_path TEXT,
+                    extra_json TEXT,
+                    created_at REAL NOT NULL
+                )
+                """
+            )
+            columns = {
+                row["name"]
+                for row in conn.execute("PRAGMA table_info(video_analysis_segments)").fetchall()
+            }
+            for name, sql_type in (
+                ("error_category", "TEXT"),
+                ("model_name", "TEXT"),
+                ("source_generation", "INTEGER"),
+            ):
+                if name not in columns:
+                    try:
+                        conn.execute(
+                            f"ALTER TABLE video_analysis_segments ADD COLUMN {name} {sql_type}"
+                        )
+                    except sqlite3.OperationalError as error:
+                        if "duplicate column" not in str(error).lower():
+                            raise
+            conn.execute(
+                """
+                CREATE INDEX IF NOT EXISTS idx_video_analysis_segments_lookup
+                ON video_analysis_segments (board_key, query_key, modality, status, start_ms, end_ms)
+                """
+            )
+            conn.execute(
+                """
+                CREATE INDEX IF NOT EXISTS idx_video_analysis_observations_board
+                ON video_analysis_observations (board_key, start_ms, end_ms)
+                """
+            )
+            conn.commit()
+
+    def _persist_evidence(self, entry: dict[str, Any], observation_id: str) -> str | None:
+        candidate = entry.get("screenshot") or entry.get("screenshot_path")
+        if not candidate:
+            return None
+        source = Path(str(candidate))
+        if not source.exists() or not source.is_file() or self.evidence_dir is None:
+            return str(source) if source.exists() else None
+        try:
+            if source.stat().st_size <= 0:
+                return None
+        except OSError:
+            return None
+
+        suffix = source.suffix.lower() or ".jpg"
+        destination = self.evidence_dir / f"{observation_id}{suffix}"
+        if source.resolve() != destination.resolve() and not destination.exists():
+            try:
+                shutil.copy2(source, destination)
+            except OSError as exc:
+                logger.warning(f"Failed to persist video evidence {source}: {exc}")
+                return str(source)
+        return str(destination)
+
+    def add_observation(self, entry: dict[str, Any], *, modality: str = "video") -> dict[str, Any]:
+        """Idempotently persist one evidence-backed observation."""
+
+        start = float(entry.get("start", 0.0))
+        raw_end = entry.get("end")
+        end = None if raw_end in (None, "unknown") else float(raw_end)
+        query_text = str(entry.get("target", ""))
+        summary = str(entry.get("summary", "")).strip()
+        identity = json.dumps(
+            {
+                "board_key": self.board_key,
+                "start_ms": _to_ms(start),
+                "end_ms": _to_ms(end) if end is not None else None,
+                "query": _normalise_query(query_text),
+                "summary": summary,
+                "modality": modality,
+            },
+            sort_keys=True,
+            separators=(",", ":"),
+        )
+        observation_id = hashlib.sha256(identity.encode("utf-8")).hexdigest()
+        screenshot_path = self._persist_evidence(entry, observation_id)
+
+        stored = dict(entry)
+        if screenshot_path:
+            stored["screenshot"] = screenshot_path
+        stored["modality"] = modality
+        stored["observation_id"] = observation_id
+
+        excluded = {
+            "start",
+            "end",
+            "target",
+            "summary",
+            "confidence_score",
+            "screenshot",
+            "screenshot_path",
+            "modality",
+            "observation_id",
+        }
+        extra = {key: value for key, value in stored.items() if key not in excluded}
+        now = time.time()
+
+        if self.db_path is None:
+            with self._lock:
+                self._memory_observations.setdefault(observation_id, stored)
+            return dict(self._memory_observations[observation_id])
+
+        with self._connection() as conn:
+            conn.execute(
+                """
+                INSERT OR IGNORE INTO video_analysis_observations (
+                    observation_id, board_key, start_ms, end_ms, query_text,
+                    summary, confidence, modality, screenshot_path, extra_json, created_at
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    observation_id,
+                    self.board_key,
+                    _to_ms(start),
+                    _to_ms(end) if end is not None else None,
+                    query_text,
+                    summary,
+                    float(entry.get("confidence_score", 0.0)),
+                    modality,
+                    screenshot_path,
+                    json.dumps(extra, default=str),
+                    now,
+                ),
+            )
+            conn.commit()
+        return stored
+
+    def list_observations(self) -> list[dict[str, Any]]:
+        """Return all observations for this recording in timeline order."""
+
+        if self.db_path is None:
+            with self._lock:
+                entries = [dict(value) for value in self._memory_observations.values()]
+            return sorted(entries, key=lambda value: float(value.get("start", 0.0)))
+
+        with self._connection() as conn:
+            rows = conn.execute(
+                """
+                SELECT * FROM video_analysis_observations
+                WHERE board_key = ?
+                ORDER BY start_ms, created_at
+                """,
+                (self.board_key,),
+            ).fetchall()
+
+        entries: list[dict[str, Any]] = []
+        for row in rows:
+            entry: dict[str, Any] = {
+                "start": _to_seconds(row["start_ms"]),
+                "end": _to_seconds(row["end_ms"]) if row["end_ms"] is not None else "unknown",
+                "target": row["query_text"],
+                "summary": row["summary"],
+                "confidence_score": row["confidence"],
+                "modality": row["modality"],
+                "observation_id": row["observation_id"],
+            }
+            if row["screenshot_path"]:
+                entry["screenshot"] = row["screenshot_path"]
+            if row["extra_json"]:
+                try:
+                    entry.update(json.loads(row["extra_json"]))
+                except (TypeError, json.JSONDecodeError):
+                    pass
+            entries.append(entry)
+        return entries
+
+    def list_ledger_entries(self) -> list[dict[str, Any]]:
+        """Return observations plus successful query-specific segment summaries."""
+
+        entries = self.list_observations()
+        if self.db_path is None:
+            with self._lock:
+                for (query_key, modality, start_ms, end_ms), value in self._memory_segments.items():
+                    if value.get("status") != "succeeded":
+                        continue
+                    entries.append(
+                        {
+                            "start": _to_seconds(start_ms),
+                            "end": _to_seconds(end_ms),
+                            "target": value.get("query_text", ""),
+                            "summary": value.get("summary") or "No summary provided.",
+                            "confidence_score": 0.5,
+                            "modality": modality,
+                            "kind": "segment_result",
+                            "query_key": query_key,
+                        }
+                    )
+        else:
+            with self._connection() as conn:
+                rows = conn.execute(
+                    """
+                    SELECT query_key, query_text, modality, start_ms, end_ms, summary
+                    FROM video_analysis_segments
+                    WHERE board_key = ? AND status = 'succeeded'
+                    ORDER BY start_ms, updated_at
+                    """,
+                    (self.board_key,),
+                ).fetchall()
+            entries.extend(
+                {
+                    "start": _to_seconds(row["start_ms"]),
+                    "end": _to_seconds(row["end_ms"]),
+                    "target": row["query_text"],
+                    "summary": row["summary"] or "No summary provided.",
+                    "confidence_score": 0.5,
+                    "modality": row["modality"],
+                    "kind": "segment_result",
+                    "query_key": row["query_key"],
+                }
+                for row in rows
+            )
+        return sorted(entries, key=lambda value: float(value.get("start", 0.0)))
+
+    def claim_segment(
+        self,
+        start: float,
+        end: float,
+        query: str,
+        *,
+        modality: str = "video",
+        model_name: str | None = None,
+        source_generation: int | None = None,
+    ) -> SegmentClaim:
+        """Atomically acquire work, or return a cached/in-progress result."""
+
+        start_ms = _to_ms(start)
+        end_ms = _to_ms(end)
+        if end_ms <= start_ms:
+            raise ValueError("Video analysis segment end must be greater than start")
+        q_key = _query_key(query)
+        key = (q_key, modality, start_ms, end_ms)
+        now = time.time()
+        owner = str(uuid4())
+        expires = now + self.lease_seconds
+
+        if self.db_path is None:
+            with self._lock:
+                existing = self._memory_segments.get(key)
+                if existing and existing["status"] == "succeeded":
+                    return SegmentClaim(
+                        "cached", summary=existing.get("summary"), analysis=existing.get("analysis")
+                    )
+                if (
+                    existing
+                    and existing["status"] == "running"
+                    and float(existing.get("lease_expires_at") or 0.0) > now
+                ):
+                    return SegmentClaim("in_progress")
+                attempts = int(existing.get("attempt_count", 0)) + 1 if existing else 1
+                self._memory_segments[key] = {
+                    "query_text": query,
+                    "status": "running",
+                    "lease_owner": owner,
+                    "lease_expires_at": expires,
+                    "attempt_count": attempts,
+                    "model_name": model_name,
+                    "source_generation": source_generation,
+                    "start_ms": start_ms,
+                    "end_ms": end_ms,
+                    "updated_at": now,
+                }
+            return SegmentClaim("claimed", lease_owner=owner)
+
+        with self._connection() as conn:
+            conn.execute("BEGIN IMMEDIATE")
+            row = conn.execute(
+                """
+                SELECT * FROM video_analysis_segments
+                WHERE board_key = ? AND query_key = ? AND modality = ?
+                  AND start_ms = ? AND end_ms = ?
+                """,
+                (self.board_key, q_key, modality, start_ms, end_ms),
+            ).fetchone()
+            if row is not None and row["status"] == "succeeded":
+                conn.commit()
+                return SegmentClaim("cached", summary=row["summary"], analysis=row["analysis"])
+            if (
+                row is not None
+                and row["status"] == "running"
+                and float(row["lease_expires_at"] or 0.0) > now
+            ):
+                conn.commit()
+                return SegmentClaim("in_progress")
+
+            if row is None:
+                conn.execute(
+                    """
+                    INSERT INTO video_analysis_segments (
+                        board_key, query_key, query_text, modality, start_ms, end_ms,
+                        status, lease_owner, lease_expires_at, attempt_count,
+                        model_name, source_generation, created_at, updated_at
+                    ) VALUES (?, ?, ?, ?, ?, ?, 'running', ?, ?, 1, ?, ?, ?, ?)
+                    """,
+                    (
+                        self.board_key,
+                        q_key,
+                        query,
+                        modality,
+                        start_ms,
+                        end_ms,
+                        owner,
+                        expires,
+                        model_name,
+                        source_generation,
+                        now,
+                        now,
+                    ),
+                )
+            else:
+                conn.execute(
+                    """
+                    UPDATE video_analysis_segments
+                    SET status = 'running', lease_owner = ?, lease_expires_at = ?,
+                        attempt_count = attempt_count + 1, error = NULL,
+                        error_category = NULL, model_name = ?, source_generation = ?,
+                        updated_at = ?
+                    WHERE board_key = ? AND query_key = ? AND modality = ?
+                      AND start_ms = ? AND end_ms = ?
+                    """,
+                    (
+                        owner,
+                        expires,
+                        model_name,
+                        source_generation,
+                        now,
+                        self.board_key,
+                        q_key,
+                        modality,
+                        start_ms,
+                        end_ms,
+                    ),
+                )
+            conn.commit()
+        return SegmentClaim("claimed", lease_owner=owner)
+
+    def complete_segment(
+        self,
+        start: float,
+        end: float,
+        query: str,
+        lease_owner: str,
+        summary: str,
+        analysis: str | None = None,
+        *,
+        modality: str = "video",
+    ) -> None:
+        """Commit a complete result while preserving lease ownership."""
+
+        key = (_query_key(query), modality, _to_ms(start), _to_ms(end))
+        now = time.time()
+        if self.db_path is None:
+            with self._lock:
+                row = self._memory_segments.get(key)
+                if row is None or row.get("lease_owner") != lease_owner:
+                    return
+                row.update(
+                    {
+                        "status": "succeeded",
+                        "summary": summary,
+                        "analysis": analysis,
+                        "lease_expires_at": None,
+                        "updated_at": now,
+                    }
+                )
+            return
+
+        with self._connection() as conn:
+            conn.execute(
+                """
+                UPDATE video_analysis_segments
+                SET status = 'succeeded', summary = ?, analysis = ?, error = NULL,
+                    lease_expires_at = NULL, updated_at = ?
+                WHERE board_key = ? AND query_key = ? AND modality = ?
+                  AND start_ms = ? AND end_ms = ? AND lease_owner = ?
+                """,
+                (
+                    summary,
+                    analysis,
+                    now,
+                    self.board_key,
+                    key[0],
+                    modality,
+                    key[2],
+                    key[3],
+                    lease_owner,
+                ),
+            )
+            conn.commit()
+
+    def fail_segment(
+        self,
+        start: float,
+        end: float,
+        query: str,
+        lease_owner: str,
+        error: str,
+        *,
+        modality: str = "video",
+        retryable: bool = True,
+        error_category: str | None = None,
+    ) -> None:
+        """Persist a failed attempt without deleting prior observations."""
+
+        key = (_query_key(query), modality, _to_ms(start), _to_ms(end))
+        status = "retryable_failed" if retryable else "permanent_failed"
+        now = time.time()
+        if self.db_path is None:
+            with self._lock:
+                row = self._memory_segments.get(key)
+                if row is None or row.get("lease_owner") != lease_owner:
+                    return
+                row.update(
+                    {
+                        "status": status,
+                        "error": str(error),
+                        "error_category": error_category,
+                        "lease_expires_at": None,
+                        "updated_at": now,
+                    }
+                )
+            return
+
+        with self._connection() as conn:
+            conn.execute(
+                """
+                UPDATE video_analysis_segments
+                SET status = ?, error = ?, error_category = ?,
+                    lease_expires_at = NULL, updated_at = ?
+                WHERE board_key = ? AND query_key = ? AND modality = ?
+                  AND start_ms = ? AND end_ms = ? AND lease_owner = ?
+                """,
+                (
+                    status,
+                    str(error),
+                    error_category,
+                    now,
+                    self.board_key,
+                    key[0],
+                    modality,
+                    key[2],
+                    key[3],
+                    lease_owner,
+                ),
+            )
+            conn.commit()
+
+    def metrics(self) -> dict[str, Any]:
+        """Return a compact operational snapshot for tracing and diagnostics."""
+
+        if self.db_path is None:
+            with self._lock:
+                rows = [dict(value) for value in self._memory_segments.values()]
+                observation_count = len(self._memory_observations)
+        else:
+            with self._connection() as conn:
+                rows = [
+                    dict(row)
+                    for row in conn.execute(
+                        """
+                        SELECT status, attempt_count, start_ms, end_ms, error_category
+                        FROM video_analysis_segments WHERE board_key = ?
+                        """,
+                        (self.board_key,),
+                    ).fetchall()
+                ]
+                observation_count = int(
+                    conn.execute(
+                        "SELECT COUNT(*) FROM video_analysis_observations WHERE board_key = ?",
+                        (self.board_key,),
+                    ).fetchone()[0]
+                )
+
+        statuses: dict[str, int] = {}
+        failure_categories: dict[str, int] = {}
+        successful_seconds = 0.0
+        attempts = 0
+        for row in rows:
+            status = str(row.get("status", "unknown"))
+            statuses[status] = statuses.get(status, 0) + 1
+            attempts += int(row.get("attempt_count", 0) or 0)
+            category = row.get("error_category")
+            if category:
+                failure_categories[str(category)] = failure_categories.get(str(category), 0) + 1
+            if status == "succeeded" and row.get("start_ms") is not None:
+                successful_seconds += max(
+                    0.0,
+                    (float(row.get("end_ms", 0)) - float(row.get("start_ms", 0))) / 1000.0,
+                )
+        return {
+            "board_key": self.board_key,
+            "segments": statuses,
+            "attempts": attempts,
+            "failure_categories": failure_categories,
+            "successful_seconds": round(successful_seconds, 3),
+            "observations": observation_count,
+        }
+
+    def successful_segments(
+        self, start: float, end: float, query: str, *, modality: str = "video"
+    ) -> list[dict[str, Any]]:
+        """Return successful segments overlapping a requested interval."""
+
+        start_ms, end_ms = _to_ms(start), _to_ms(end)
+        q_key = _query_key(query)
+        if self.db_path is None:
+            with self._lock:
+                rows = []
+                for (
+                    row_query,
+                    row_modality,
+                    row_start,
+                    row_end,
+                ), value in self._memory_segments.items():
+                    if (
+                        row_query == q_key
+                        and row_modality == modality
+                        and value.get("status") == "succeeded"
+                        and row_end > start_ms
+                        and row_start < end_ms
+                    ):
+                        rows.append(
+                            {
+                                **value,
+                                "start": _to_seconds(row_start),
+                                "end": _to_seconds(row_end),
+                            }
+                        )
+            return sorted(rows, key=lambda value: value["start"])
+
+        with self._connection() as conn:
+            db_rows = conn.execute(
+                """
+                SELECT start_ms, end_ms, summary, analysis, status
+                FROM video_analysis_segments
+                WHERE board_key = ? AND query_key = ? AND modality = ?
+                  AND status = 'succeeded' AND end_ms > ? AND start_ms < ?
+                ORDER BY start_ms
+                """,
+                (self.board_key, q_key, modality, start_ms, end_ms),
+            ).fetchall()
+        return [
+            {
+                "start": _to_seconds(row["start_ms"]),
+                "end": _to_seconds(row["end_ms"]),
+                "summary": row["summary"],
+                "analysis": row["analysis"],
+                "status": row["status"],
+            }
+            for row in db_rows
+        ]
+
+    def missing_intervals(
+        self, start: float, end: float, query: str, *, modality: str = "video"
+    ) -> list[tuple[float, float]]:
+        """Subtract successful exact-query coverage from a requested interval."""
+
+        start_ms, end_ms = _to_ms(start), _to_ms(end)
+        if end_ms <= start_ms:
+            return []
+        covered = []
+        for row in self.successful_segments(start, end, query, modality=modality):
+            covered.append((max(start_ms, _to_ms(row["start"])), min(end_ms, _to_ms(row["end"]))))
+        covered = sorted((left, right) for left, right in covered if right > left)
+
+        merged: list[tuple[int, int]] = []
+        for left, right in covered:
+            if not merged or left > merged[-1][1]:
+                merged.append((left, right))
+            else:
+                merged[-1] = (merged[-1][0], max(merged[-1][1], right))
+
+        missing: list[tuple[float, float]] = []
+        cursor = start_ms
+        for left, right in merged:
+            if left > cursor:
+                missing.append((_to_seconds(cursor), _to_seconds(left)))
+            cursor = max(cursor, right)
+        if cursor < end_ms:
+            missing.append((_to_seconds(cursor), _to_seconds(end_ms)))
+        return missing
+
+    def format_cached_segments(
+        self, start: float, end: float, query: str, *, modality: str = "video"
+    ) -> list[str]:
+        """Format cached segment results in the child-agent response shape."""
+
+        results = []
+        for row in self.successful_segments(start, end, query, modality=modality):
+            text = (
+                f"[from {row['start']:.1f}s to {row['end']:.1f}s] "
+                f"Summary: {row.get('summary') or 'No summary provided.'}"
+            )
+            if row.get("analysis"):
+                text += f" Analysis: {row['analysis']}"
+            results.append(text)
+        return results
+
+
+def _safe_path(value: Any) -> Path | None:
+    if isinstance(value, Path):
+        return value
+    if isinstance(value, str) and value:
+        return Path(value)
+    return None
+
+
+def _latest_recorded_video_key(db_path: Path | None, session_id: Any) -> str | None:
+    """Resolve a stopped recording so a restarted analyzer keeps its memory key."""
+    if db_path is None or not session_id or not db_path.exists():
+        return None
+    try:
+        conn = sqlite3.connect(db_path, timeout=5.0)
+        try:
+            row = conn.execute(
+                """
+                SELECT video_id FROM video_recordings
+                WHERE session_id = ?
+                ORDER BY start_time DESC LIMIT 1
+                """,
+                (str(session_id),),
+            ).fetchone()
+        finally:
+            conn.close()
+    except sqlite3.Error:
+        return None
+    return f"video:{row[0]}" if row and row[0] else None
+
+
+def get_video_blackboard(ctx: ArtemisContext) -> VideoBlackboard:
+    """Return the blackboard shared by every analyzer in an Artemis context."""
+
+    existing = getattr(ctx, "_video_blackboard", None)
+    if isinstance(existing, VideoBlackboard):
+        return existing
+
+    device_id = str(getattr(getattr(ctx, "device", None), "device_id", "default"))
+    data_engine = getattr(ctx, "data_engine", None)
+    db_path = None
+    base_dir = None
+    storage = getattr(data_engine, "storage", None)
+    if storage is not None:
+        db_path = _safe_path(getattr(storage, "db_path", None))
+    if data_engine is not None:
+        base_dir = _safe_path(getattr(data_engine, "base_dir", None))
+
+    if base_dir is None:
+        execution_setup = getattr(ctx, "execution_setup", None)
+        traces_path = _safe_path(getattr(execution_setup, "traces_path", None))
+        trace_name = getattr(execution_setup, "trace_name", None)
+        if traces_path is not None:
+            base_dir = traces_path / str(trace_name) if trace_name else traces_path
+
+    if db_path is None and base_dir is not None:
+        db_path = base_dir / "video_blackboard.db"
+    evidence_dir = base_dir / "video_blackboard" / "evidence" if base_dir else None
+
+    session = get_active_session(device_id)
+    current_session_id = getattr(data_engine, "current_session_id", None)
+    if session is not None:
+        board_key = f"video:{session.video_id}"
+    else:
+        board_key = _latest_recorded_video_key(db_path, current_session_id) or (
+            f"session:{current_session_id}" if current_session_id else f"device:{device_id}:runtime"
+        )
+
+    board = VideoBlackboard(board_key, db_path=db_path, evidence_dir=evidence_dir)
+    try:
+        setattr(ctx, "_video_blackboard", board)
+    except (AttributeError, ValueError):
+        logger.debug(
+            "Context does not allow caching video_blackboard; SQLite sharing remains active"
+        )
+    return board

artemis/agents/video_analyzer/reliability.py

@@ -0,0 +1,157 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Failure classification and circuit breaking for video model calls."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+import threading
+import time
+from typing import Any
+
+
+class VideoFailureCategory(StrEnum):
+    RATE_LIMIT = "rate_limit"
+    PROVIDER_UNAVAILABLE = "provider_unavailable"
+    TIMEOUT = "timeout"
+    CONNECTION = "connection"
+    AUTHENTICATION = "authentication"
+    BAD_REQUEST = "bad_request"
+    MEDIA_PROCESSING = "media_processing"
+    CANCELLED = "cancelled"
+    UNKNOWN = "unknown"
+
+
+@dataclass(frozen=True)
+class VideoFailure:
+    category: VideoFailureCategory
+    retryable: bool
+    should_split: bool
+    should_fallback: bool
+
+
+def classify_video_failure(error: BaseException) -> VideoFailure:
+    """Convert provider/transport/media exceptions into recovery decisions."""
+
+    if isinstance(error, KeyboardInterrupt) or type(error).__name__ == "CancelledError":
+        return VideoFailure(VideoFailureCategory.CANCELLED, False, False, False)
+
+    code = getattr(error, "code", None) or getattr(error, "status_code", None)
+    try:
+        code = int(code) if code is not None else None
+    except (TypeError, ValueError):
+        code = None
+
+    message = str(error).lower()
+    if code == 429 or any(marker in message for marker in ("rate limit", "quota", "resource_exhausted")):
+        return VideoFailure(VideoFailureCategory.RATE_LIMIT, True, False, True)
+    if code in {500, 502, 503, 504} or any(
+        marker in message
+        for marker in ("unavailable", "overloaded", "high demand", "service unavailable")
+    ):
+        return VideoFailure(VideoFailureCategory.PROVIDER_UNAVAILABLE, True, False, True)
+    if isinstance(error, TimeoutError) or "timed out" in message or "timeout" in message:
+        return VideoFailure(VideoFailureCategory.TIMEOUT, True, True, True)
+    if isinstance(error, (ConnectionError, OSError)) and not any(
+        marker in message for marker in ("ffmpeg", "video", "audio", "codec", "media")
+    ):
+        return VideoFailure(VideoFailureCategory.CONNECTION, True, False, True)
+    if code in {401, 403} or any(
+        marker in message for marker in ("unauthorized", "forbidden", "api key", "credential")
+    ):
+        return VideoFailure(VideoFailureCategory.AUTHENTICATION, False, False, True)
+    if any(
+        marker in message
+        for marker in ("ffmpeg", "codec", "corrupt", "invalid video", "invalid audio", "media")
+    ):
+        return VideoFailure(VideoFailureCategory.MEDIA_PROCESSING, False, True, True)
+    if code in {400, 404, 413, 415, 422}:
+        return VideoFailure(VideoFailureCategory.BAD_REQUEST, False, code == 413, True)
+    return VideoFailure(VideoFailureCategory.UNKNOWN, True, True, True)
+
+
+@dataclass
+class _CircuitState:
+    consecutive_failures: int = 0
+    open_until: float = 0.0
+    half_open_in_flight: bool = False
+
+
+class VideoCircuitBreaker:
+    """Small context-scoped circuit breaker keyed by provider/model identity."""
+
+    def __init__(self, threshold: int = 3, cooldown_seconds: float = 60.0) -> None:
+        self.threshold = max(1, int(threshold))
+        self.cooldown_seconds = max(0.1, float(cooldown_seconds))
+        self._states: dict[str, _CircuitState] = {}
+        self._lock = threading.RLock()
+
+    def allow(self, key: str, *, now: float | None = None) -> bool:
+        current = time.time() if now is None else now
+        with self._lock:
+            state = self._states.get(key)
+            if state is None:
+                return True
+            if state.open_until:
+                if current < state.open_until or state.half_open_in_flight:
+                    return False
+                # Half-open: permit exactly one trial. Other concurrent calls
+                # continue to bypass the primary until it succeeds or fails.
+                state.half_open_in_flight = True
+                state.consecutive_failures = max(0, self.threshold - 1)
+                return True
+            return not state.half_open_in_flight
+
+    def record_success(self, key: str) -> None:
+        with self._lock:
+            self._states.pop(key, None)
+
+    def record_failure(
+        self,
+        key: str,
+        failure: VideoFailure,
+        *,
+        now: float | None = None,
+    ) -> None:
+        if failure.category not in {
+            VideoFailureCategory.RATE_LIMIT,
+            VideoFailureCategory.PROVIDER_UNAVAILABLE,
+            VideoFailureCategory.TIMEOUT,
+            VideoFailureCategory.CONNECTION,
+        }:
+            with self._lock:
+                state = self._states.get(key)
+                if state is not None and state.half_open_in_flight:
+                    self._states.pop(key, None)
+            return
+        current = time.time() if now is None else now
+        with self._lock:
+            state = self._states.setdefault(key, _CircuitState())
+            state.half_open_in_flight = False
+            state.consecutive_failures += 1
+            if state.consecutive_failures >= self.threshold:
+                state.open_until = current + self.cooldown_seconds
+
+    def snapshot(self) -> dict[str, dict[str, Any]]:
+        with self._lock:
+            return {
+                key: {
+                    "consecutive_failures": state.consecutive_failures,
+                    "open_until": state.open_until,
+                    "half_open_in_flight": state.half_open_in_flight,
+                }
+                for key, state in self._states.items()
+            }

artemis/config/agent.py

@@ -46,6 +46,69 @@ class VideoAnalyzerConfig(BaseModel):
         default=True,
         description="Whether to enable video action ledger state tracking.",
     )
+    chunk_size_seconds: float = Field(
+        default=60.0,
+        ge=5.0,
+        le=300.0,
+        description="Maximum duration of an initial video-analysis chunk.",
+    )
+    min_chunk_seconds: float = Field(
+        default=4.0,
+        ge=1.0,
+        le=30.0,
+        description="Smallest chunk produced by failure-driven bisection.",
+    )
+    max_split_depth: int = Field(
+        default=4,
+        ge=0,
+        le=8,
+        description="Maximum recursive chunk bisection depth after failures.",
+    )
+    circuit_breaker_threshold: int = Field(
+        default=3,
+        ge=1,
+        le=20,
+        description="Consecutive transient model failures before opening the circuit.",
+    )
+    circuit_breaker_cooldown_seconds: float = Field(
+        default=60.0,
+        ge=1.0,
+        le=900.0,
+        description="How long an unhealthy video model remains bypassed.",
+    )
+    action_window_seconds: float = Field(
+        default=2.0,
+        ge=0.0,
+        le=10.0,
+        description="Dense-perception window on each side of a recorded mobile action.",
+    )
+    dense_action_fps: float = Field(
+        default=4.0,
+        ge=1.0,
+        le=15.0,
+        description="Frame sampling rate around recorded mobile actions.",
+    )
+    max_dense_action_frames: int = Field(
+        default=24,
+        ge=0,
+        le=120,
+        description="Maximum additional action-proximal keyframes per chunk.",
+    )
+    native_max_retries: int = Field(
+        default=1,
+        ge=0,
+        le=5,
+        description=(
+            "Retries against the native video provider before switching to the "
+            "configured universal fallback. A value of 1 means two total attempts."
+        ),
+    )
+    model_call_timeout_seconds: float = Field(
+        default=120.0,
+        ge=15.0,
+        le=600.0,
+        description="Hard timeout for one native or universal video-model response.",
+    )
     model_config = {"extra": "allow"}
 
 

artemis/context.py

@@ -171,6 +171,9 @@ class ArtemisContext(BaseModel):
     task_plan_content_before: str | None = None
 
     _genai_client: Any | None = PrivateAttr(default=None)
+    _video_blackboard: Any | None = PrivateAttr(default=None)
+    _video_circuit_breaker: Any | None = PrivateAttr(default=None)
+    _mobile_controller: Any | None = PrivateAttr(default=None)
     mcp_client_ctx: Any | None = None
     mcp_session: Any | None = None
 

artemis/controllers/android_controller.py

@@ -78,7 +78,9 @@ class AndroidDeviceController(MobileDeviceController):
         self.data_engine_start_time = data_engine_start_time
         self.data_engine = data_engine
         self._device: AdbDevice | None = None
-        self._segment_cache: dict[tuple[float, float | None], VideoRecordingResult] = {}
+        self._segment_cache: dict[
+            tuple[str, int, float, float], VideoRecordingResult
+        ] = {}
 
     @property
     def device(self) -> AdbDevice:
@@ -734,19 +736,6 @@ class AndroidDeviceController(MobileDeviceController):
         end_relative_time: float | None = None,
     ) -> VideoRecordingResult:
         """Get a video segment for a specific time range (relative to video start)."""
-        cache_key = (
-            round(start_relative_time, 1),
-            round(end_relative_time, 1) if end_relative_time is not None else None,
-        )
-        if cache_key in self._segment_cache:
-            cached_res = self._segment_cache[cache_key]
-            if cached_res.success and cached_res.video_path and cached_res.video_path.exists():
-                logger.info(
-                    "Reusing cached trimmed video segment for range"
-                    f" {cache_key[0]}s to {cache_key[1]}s"
-                )
-                return cached_res
-
         session = get_active_session(self.device_id)
         if not session:
             return VideoRecordingResult(
@@ -754,6 +743,22 @@ class AndroidDeviceController(MobileDeviceController):
                 message=f"No active recording for device {self.device_id}",
             )
 
+        cache_key = None
+        if end_relative_time is not None:
+            cache_key = (
+                str(session.video_id),
+                session.generation,
+                round(start_relative_time, 1),
+                round(end_relative_time, 1),
+            )
+            cached_res = self._segment_cache.get(cache_key)
+            if cached_res and cached_res.success and cached_res.video_path and cached_res.video_path.exists():
+                logger.info(
+                    "Reusing generation-scoped trimmed video segment for range"
+                    f" {cache_key[2]}s to {cache_key[3]}s"
+                )
+                return cached_res
+
         try:
             mkv_path = session.local_video_path
             if not mkv_path or not mkv_path.exists():
@@ -776,6 +781,7 @@ class AndroidDeviceController(MobileDeviceController):
                 # Try to restart scrcpy
                 try:
                     session.android_segment_index += 1
+                    session.generation = session.android_segment_index
                     output_dir = mkv_path.parent
                     new_video_path = output_dir / f"recording_{session.android_segment_index}.mkv"
 
@@ -890,8 +896,15 @@ class AndroidDeviceController(MobileDeviceController):
                 duration_seconds=round(duration, 2),
                 actual_start_relative_time=actual_start,
                 warning=truncation_warning,
+                video_id=session.video_id,
+                generation=session.generation,
+                sealed_until=session.sealed_until,
+                source_revision=(
+                    f"{session.video_id}:{session.generation}:{round(actual_end, 3)}"
+                ),
             )
-            self._segment_cache[cache_key] = res
+            if cache_key is not None:
+                self._segment_cache[cache_key] = res
             return res
 
         except Exception as e:

artemis/controllers/controller_factory.py

@@ -25,5 +25,17 @@ def create_device_controller(ctx: ArtemisContext) -> UnifiedMobileController:
 
 
 def get_controller(ctx: ArtemisContext) -> UnifiedMobileController:
-    """Backward-compatible alias for create_device_controller."""
-    return create_device_controller(ctx)
+    """Return the controller shared by one execution context.
+
+    Sharing preserves recording-segment caches and prevents every tool call
+    from constructing a new driver/controller pair.
+    """
+    existing = getattr(ctx, "_mobile_controller", None)
+    if isinstance(existing, UnifiedMobileController):
+        return existing
+    controller = create_device_controller(ctx)
+    try:
+        ctx._mobile_controller = controller
+    except (AttributeError, ValueError):
+        pass
+    return controller

artemis/controllers/unified_controller.py

@@ -61,7 +61,9 @@ class UnifiedMobileController:
     def __init__(self, ctx: ArtemisContext):
         self.ctx = ctx
         self._driver: BaseDeviceDriver = get_driver(ctx)
-        self._segment_cache: dict[tuple[float, float | None], VideoRecordingResult] = {}
+        self._segment_cache: dict[
+            tuple[str, int, float, float], VideoRecordingResult
+        ] = {}
 
     @property
     def driver(self) -> BaseDeviceDriver:
@@ -290,18 +292,6 @@ class UnifiedMobileController:
         output_path: Path | None = None,
     ) -> VideoRecordingResult:
         """Get a video segment for a specific time range (relative to video start)."""
-        cache_key = (
-            round(start_time, 1),
-            round(end_time, 1) if end_time is not None else None,
-        )
-        if cache_key in self._segment_cache:
-            cached_res = self._segment_cache[cache_key]
-            if cached_res.success and cached_res.video_path and cached_res.video_path.exists():
-                logger.info(
-                    f"Reusing cached trimmed video segment for range {cache_key[0]}s to {cache_key[1]}s"
-                )
-                return cached_res
-
         device_id = self._get_device_id()
 
         # Handle mock driver
@@ -324,6 +314,22 @@ class UnifiedMobileController:
                 message=f"No active recording for device {device_id}",
             )
 
+        cache_key = None
+        if end_time is not None:
+            cache_key = (
+                str(session.video_id),
+                session.generation,
+                round(start_time, 1),
+                round(end_time, 1),
+            )
+            cached_res = self._segment_cache.get(cache_key)
+            if cached_res and cached_res.success and cached_res.video_path and cached_res.video_path.exists():
+                logger.info(
+                    "Reusing generation-scoped trimmed video segment for range "
+                    f"{cache_key[2]}s to {cache_key[3]}s"
+                )
+                return cached_res
+
         try:
             mkv_path = session.local_video_path
             if not mkv_path:
@@ -425,8 +431,15 @@ class UnifiedMobileController:
                 duration_seconds=round(duration, 2),
                 actual_start_relative_time=actual_start,
                 warning=truncation_warning,
+                video_id=session.video_id,
+                generation=session.generation,
+                sealed_until=session.sealed_until,
+                source_revision=(
+                    f"{session.video_id}:{session.generation}:{round(actual_end, 3)}"
+                ),
             )
-            self._segment_cache[cache_key] = res
+            if cache_key is not None:
+                self._segment_cache[cache_key] = res
             return res
 
         except Exception as e:
@@ -469,12 +482,17 @@ class UnifiedMobileController:
                 "start": max(0.0, start_time - session.start_time),
                 "end": max(0.0, end_time - session.start_time),
                 "rotation": session.android_rotation,
+                "generation": session.generation,
                 "conversion_scheduled": True,
             }
         )
         session.android_conversion_tasks.append(
             asyncio.create_task(self._remux_segment_record(session.android_segment_records[-1]))
         )
+        session.sealed_until = max(
+            session.sealed_until,
+            max(0.0, end_time - session.start_time),
+        )
 
     def _record_recording_failure(self, session: RecordingSession, message: str) -> None:
         if self.ctx and self.ctx.data_engine:
@@ -490,6 +508,7 @@ class UnifiedMobileController:
         self, session: RecordingSession, display_state: tuple[int, int, int] | None
     ) -> bool:
         session.android_segment_index += 1
+        session.generation = session.android_segment_index
         output_dir = session.local_video_path.parent
         new_video_path = output_dir / f"recording_{session.android_segment_index:03d}.mkv"
         process = await self._spawn_scrcpy(
@@ -635,6 +654,10 @@ class UnifiedMobileController:
             return VideoRecordingResult(
                 success=True,
                 message=f"Recording started on {device_id}",
+                video_id=session.video_id,
+                generation=session.generation,
+                sealed_until=session.sealed_until,
+                source_revision=f"{session.video_id}:{session.generation}:active",
             )
 
         except Exception as e:
@@ -743,6 +766,10 @@ class UnifiedMobileController:
                 success=True,
                 message=f"Recording stopped, saved {len(mp4_paths)} orientation-safe segments",
                 video_path=mp4_paths[0],
+                video_id=session.video_id,
+                generation=session.generation,
+                sealed_until=session.sealed_until,
+                source_revision=f"{session.video_id}:{session.generation}:sealed",
             )
 
         except Exception as e:

artemis/core/diagnostics/engine.py

@@ -48,6 +48,8 @@ logger = get_logger(__name__)
 class ReadinessEngine:
     """Central orchestration engine executing modular readiness probes."""
 
+    _REPORT_CACHE_TTL_SECONDS = 2.0
+
     def __init__(self):
         self._probes: dict[str, BaseProbe] = {}
         self._active_device_serial: str | None = None
@@ -57,6 +59,11 @@ class ReadinessEngine:
         self._credentials_probe = LLMCredentialsProbe()
         self._ocr_probe = VisionOCRProbe()
         self._adb_probe = AdbDeviceProbe()
+        self._report_cache: SystemReadinessReport | None = None
+        self._report_cache_time = 0.0
+        self._report_cache_generation = -1
+        self._cache_generation = 0
+        self._report_lock = asyncio.Lock()
 
         # Register default core probes in logical lifecycle order
         self.register_probe(self._python_probe)
@@ -77,6 +84,7 @@ class ReadinessEngine:
     def set_active_device_serial(self, serial: str | None) -> None:
         """Set user-selected active target device serial."""
         self._active_device_serial = serial
+        self.invalidate_cache()
         if hasattr(self._adb_probe, "set_target_serial"):
             self._adb_probe.set_target_serial(serial)
 
@@ -95,9 +103,74 @@ class ReadinessEngine:
         """Run the bounded device gate used by task submission."""
         return await self._adb_probe.probe_submission_readiness()
 
-    async def run_all(self, categories: list[ProbeCategory] | None = None) -> SystemReadinessReport:
-        """Concurrently run all registered diagnostic probes and compile report."""
-        toolchain.clear_cache()
+    def invalidate_cache(self) -> None:
+        """Invalidate the UI readiness snapshot after an explicit configuration change."""
+        self._cache_generation += 1
+        self._report_cache = None
+        self._report_cache_time = 0.0
+        self._report_cache_generation = -1
+
+    def _cached_report(self, max_age_seconds: float) -> SystemReadinessReport | None:
+        if self._report_cache is None:
+            return None
+        if self._report_cache_generation != self._cache_generation:
+            return None
+        if time.monotonic() - self._report_cache_time > max_age_seconds:
+            return None
+        return self._report_cache.model_copy(deep=True)
+
+    async def run_all(
+        self,
+        categories: list[ProbeCategory] | None = None,
+        *,
+        force_refresh: bool = False,
+    ) -> SystemReadinessReport:
+        """Return a coalesced readiness snapshot.
+
+        The dashboard polls frequently, so allowing every HTTP request to launch a
+        complete toolchain and ADB scan creates a request storm. Full reports are
+        cached briefly and all concurrent refreshes share one execution. The task
+        submission gate remains independent and always performs its own bounded
+        device check.
+        """
+        cacheable = categories is None
+        request_started = time.monotonic()
+        if cacheable and not force_refresh:
+            cached = self._cached_report(self._REPORT_CACHE_TTL_SECONDS)
+            if cached is not None:
+                return cached
+
+        async with self._report_lock:
+            # A refresh that completed while this caller waited satisfies even a
+            # forced request that began before it, coalescing concurrent clicks.
+            if cacheable and self._report_cache_time >= request_started:
+                cached = self._cached_report(float("inf"))
+                if cached is not None:
+                    return cached
+            if cacheable and not force_refresh:
+                cached = self._cached_report(self._REPORT_CACHE_TTL_SECONDS)
+                if cached is not None:
+                    return cached
+
+            if force_refresh:
+                toolchain.clear_cache()
+                self._adb_probe.invalidate_enrichment_cache()
+
+            build_generation = self._cache_generation
+            report = await self._build_report(categories)
+            # If a device/configuration change happened during the scan, return
+            # this result only to its original caller and never publish it as the
+            # shared snapshot for later requests.
+            if cacheable and build_generation == self._cache_generation:
+                self._report_cache = report.model_copy(deep=True)
+                self._report_cache_time = time.monotonic()
+                self._report_cache_generation = build_generation
+            return report
+
+    async def _build_report(
+        self, categories: list[ProbeCategory] | None = None
+    ) -> SystemReadinessReport:
+        """Execute the underlying probes and compile an uncached report."""
         target_probes = [
             probe
             for probe in self._probes.values()

artemis/core/diagnostics/probes/adb_probe.py

@@ -20,6 +20,7 @@ from pathlib import Path
 import re
 import shutil
 import subprocess
+import time
 from typing import Any
 
 from artemis.core.diagnostics.probes.base import BaseProbe
@@ -40,8 +41,14 @@ logger = get_logger(__name__)
 class AdbDeviceProbe(BaseProbe):
     """Deep inspection probe for Android Debug Bridge, connected mobile devices, and local AVD emulators."""
 
+    _ENRICHMENT_CACHE_TTL_SECONDS = 60.0
+    _LAST_LOCK_STATE_TTL_SECONDS = 15.0
+
     def __init__(self, target_serial: str | None = None):
         self._target_serial = target_serial
+        self._device_enrichment_cache: dict[str, tuple[float, dict[str, Any]]] = {}
+        self._last_lock_states: dict[str, tuple[float, bool]] = {}
+        self._lock_state_sources: dict[str, str] = {}
 
     @property
     def probe_id(self) -> str:
@@ -59,9 +66,13 @@ class AdbDeviceProbe(BaseProbe):
         """Set or update preferred active device serial."""
         self._target_serial = serial
 
+    def invalidate_enrichment_cache(self) -> None:
+        """Refresh static device metadata on an explicit full re-check."""
+        self._device_enrichment_cache.clear()
+
     def _locate_adb(self) -> str | None:
         """Find the adb binary path from ToolchainResolver."""
-        return toolchain.resolve("adb", force_refresh=True)
+        return toolchain.resolve("adb")
 
     def _locate_emulator(self) -> str | None:
         """Find the emulator binary path from PATH or standard SDK environments."""
@@ -170,10 +181,19 @@ class AdbDeviceProbe(BaseProbe):
             trust_match.group(1).lower() in {"true", "1"} if trust_match else None
         )
 
-        states = [state for state in [policy_showing, trust_locked, *legacy_states] if state is not None]
-        if any(states):
+        # Prefer the modern, current-user signals. Legacy fields can coexist in
+        # dumpsys output as stale or display-specific diagnostics and must not
+        # override an explicit modern unlocked result.
+        modern_states = [
+            state for state in [policy_showing, trust_locked] if state is not None
+        ]
+        if any(modern_states):
             return True
-        if states:
+        if modern_states:
+            return False
+        if any(legacy_states):
+            return True
+        if legacy_states:
             return False
         return None
 
@@ -218,6 +238,85 @@ class AdbDeviceProbe(BaseProbe):
             logger.debug(f"Failed to query lock state for {serial}: {exc}")
             return None
 
+    async def _get_confirmed_device_lock_state(
+        self,
+        adb_path: str,
+        serial: str,
+        timeout_seconds: float = 2.0,
+    ) -> bool | None:
+        """Confirm a positive lock result so one transient sample cannot flap the UI."""
+        first = await self._get_device_lock_state(
+            adb_path, serial, timeout_seconds=timeout_seconds
+        )
+        if first is not True:
+            return first
+
+        second = await self._get_device_lock_state(
+            adb_path, serial, timeout_seconds=timeout_seconds
+        )
+        if second is True:
+            return True
+        if second is False:
+            return False
+        return None
+
+    async def _get_dashboard_lock_state(
+        self,
+        adb_path: str,
+        serial: str,
+        timeout_seconds: float = 2.0,
+    ) -> bool | None:
+        """Use a recent confirmed state when one dashboard sample times out.
+
+        This smoothing is intentionally dashboard-only. Task submission uses the
+        fresh confirmed probe and fails closed when the state is unknown.
+        """
+        observed = await self._get_confirmed_device_lock_state(
+            adb_path, serial, timeout_seconds=timeout_seconds
+        )
+        now = time.monotonic()
+        if observed is not None:
+            self._last_lock_states[serial] = (now, observed)
+            self._lock_state_sources[serial] = "observed"
+            return observed
+
+        previous = self._last_lock_states.get(serial)
+        if previous and now - previous[0] <= self._LAST_LOCK_STATE_TTL_SECONDS:
+            self._lock_state_sources[serial] = "recent_confirmed"
+            return previous[1]
+
+        self._lock_state_sources[serial] = "unknown"
+        return None
+
+    @staticmethod
+    async def _run_adb_shell(
+        adb_path: str,
+        serial: str,
+        *args: str,
+        timeout_seconds: float = 2.5,
+    ) -> str:
+        """Run a bounded ADB shell command and always reap timed-out children."""
+        proc = await asyncio.create_subprocess_exec(
+            adb_path,
+            "-s",
+            serial,
+            "shell",
+            *args,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        try:
+            stdout, _ = await asyncio.wait_for(
+                proc.communicate(), timeout=timeout_seconds
+            )
+        except TimeoutError:
+            proc.kill()
+            await proc.communicate()
+            return ""
+        if proc.returncode != 0:
+            return ""
+        return stdout.decode(errors="replace").strip()
+
     async def _get_device_states(
         self,
         adb_path: str,
@@ -322,7 +421,7 @@ class AdbDeviceProbe(BaseProbe):
             if self._target_serial in ready_serials
             else ready_serials[0]
         )
-        is_locked = await self._get_device_lock_state(
+        is_locked = await self._get_confirmed_device_lock_state(
             adb_path,
             serial,
             timeout_seconds=1.0,
@@ -412,66 +511,60 @@ class AdbDeviceProbe(BaseProbe):
         except Exception as e:
             logger.error(f"Error querying adb devices: {e}")
 
-        # For authorized devices, attempt non-blocking quick property enrichment
+        # Lock state stays live, while static device properties and the installed
+        # package inventory are cached to keep frequent dashboard probes cheap.
         for dev in devices:
             if dev.state == "device":
                 try:
-                    p_prop = await asyncio.create_subprocess_exec(
-                        adb_path,
-                        "-s",
-                        dev.serial,
-                        "shell",
-                        "getprop ro.build.version.release",
-                        stdout=subprocess.PIPE,
-                        stderr=subprocess.PIPE,
-                    )
-                    p_size = await asyncio.create_subprocess_exec(
-                        adb_path,
-                        "-s",
-                        dev.serial,
-                        "shell",
-                        "wm size",
-                        stdout=subprocess.PIPE,
-                        stderr=subprocess.PIPE,
-                    )
                     lock_state_task = asyncio.create_task(
-                        self._get_device_lock_state(adb_path, dev.serial)
+                        self._get_dashboard_lock_state(adb_path, dev.serial)
                     )
-                    out_prop, _ = await p_prop.communicate()
-                    out_size, _ = await p_size.communicate()
-                    dev.is_locked = await lock_state_task
-
-                    if out_prop:
-                        dev.android_version = out_prop.decode(errors="replace").strip()
-                    if out_size:
-                        size_str = out_size.decode(errors="replace").strip()
-                        if "Physical size:" in size_str:
-                            dev.screen_resolution = size_str.split("Physical size:")[-1].strip()
-
-                    # Query installed application packages with timeout protection
-                    try:
-                        p_pkg = await asyncio.create_subprocess_exec(
+                    cached = self._device_enrichment_cache.get(dev.serial)
+                    now = time.monotonic()
+                    if cached and now - cached[0] <= self._ENRICHMENT_CACHE_TTL_SECONDS:
+                        enrichment = cached[1]
+                    else:
+                        prop_task = self._run_adb_shell(
                             adb_path,
-                            "-s",
                             dev.serial,
-                            "shell",
-                            "pm list packages",
-                            stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE,
+                            "getprop",
+                            "ro.build.version.release",
                         )
-                        out_pkg, _ = await asyncio.wait_for(p_pkg.communicate(), timeout=2.5)
-                        if out_pkg:
-                            pkgs = [
+                        size_task = self._run_adb_shell(
+                            adb_path, dev.serial, "wm", "size"
+                        )
+                        packages_task = self._run_adb_shell(
+                            adb_path,
+                            dev.serial,
+                            "pm",
+                            "list",
+                            "packages",
+                        )
+                        android_version, size_str, packages_output = await asyncio.gather(
+                            prop_task, size_task, packages_task
+                        )
+                        enrichment = {
+                            "android_version": android_version or None,
+                            "screen_resolution": (
+                                size_str.split("Physical size:")[-1].strip()
+                                if "Physical size:" in size_str
+                                else None
+                            ),
+                            "installed_packages": [
                                 line.split("package:", 1)[1].strip()
-                                for line in out_pkg.decode(errors="replace").splitlines()
+                                for line in packages_output.splitlines()
                                 if line.startswith("package:")
                                 and line.split("package:", 1)[1].strip()
-                            ]
-                            dev.installed_packages = pkgs
-                    except Exception as e_pkg:
-                        logger.debug(f"Failed to query packages for {dev.serial}: {e_pkg}")
-                except Exception:
-                    pass
+                            ],
+                        }
+                        self._device_enrichment_cache[dev.serial] = (now, enrichment)
+
+                    dev.android_version = enrichment["android_version"]
+                    dev.screen_resolution = enrichment["screen_resolution"]
+                    dev.installed_packages = list(enrichment["installed_packages"])
+                    dev.is_locked = await lock_state_task
+                except Exception as exc:
+                    logger.debug(f"Failed to enrich device {dev.serial}: {exc}")
 
         return devices
 
@@ -714,6 +807,9 @@ class AdbDeviceProbe(BaseProbe):
             display_name = f"{display_name} ({active_dev.screen_resolution})"
 
         metadata["active_device"] = active_dev.model_dump()
+        metadata["lock_state_source"] = self._lock_state_sources.get(
+            active_dev.serial, "unknown"
+        )
 
         if active_dev.is_locked is not False:
             is_locked = active_dev.is_locked is True

artemis/core/diagnostics/probes/toolchain_probe.py

@@ -41,8 +41,8 @@ class ToolchainProbe(BaseProbe):
         return False
 
     async def probe(self) -> ProbeResult:
-        ffmpeg_path = toolchain.resolve("ffmpeg", force_refresh=True)
-        scrcpy_path = toolchain.resolve("scrcpy", force_refresh=True)
+        ffmpeg_path = toolchain.resolve("ffmpeg")
+        scrcpy_path = toolchain.resolve("scrcpy")
 
         tools_installed = []
         tools_missing = []

artemis/data_engine/engine.py

@@ -84,6 +84,8 @@ class DataEngine:
         self.ipc_socket = None
         self._ipc_socket_lock = threading.Lock()
         self._ipc_shutdown = False
+        self._ipc_retry_after = 0.0
+        self._ipc_failure_count = 0
         with self._ipc_socket_lock:
             self._connect_ipc_locked()
 
@@ -110,25 +112,35 @@ class DataEngine:
                 logger.debug(f"Could not read refreshed IPC port from {port_file}: {exc}")
         return candidates
 
-    def _connect_ipc_locked(self) -> bool:
+    def _connect_ipc_locked(self, *, force: bool = False) -> bool:
         """Connect to the UI event bridge while holding ``_ipc_socket_lock``."""
         if self._ipc_shutdown:
             return False
         if self.ipc_socket is not None:
             return True
+        if not force and time.monotonic() < self._ipc_retry_after:
+            return False
 
         for ipc_port in self._ipc_port_candidates():
             try:
-                sock = socket.create_connection(("127.0.0.1", ipc_port), timeout=1.0)
+                # A localhost listener either accepts immediately or is absent.
+                # A short connect cap prevents telemetry from stalling the
+                # automation/data worker when the desktop event bridge is stale.
+                sock = socket.create_connection(("127.0.0.1", ipc_port), timeout=0.2)
                 sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
                 sock.settimeout(None)
                 self.ipc_socket = sock
+                self._ipc_failure_count = 0
+                self._ipc_retry_after = 0.0
                 logger.info(f"Connected to IPC server on port {ipc_port} with TCP_NODELAY")
                 return True
             except OSError as exc:
                 logger.warning(f"Failed to connect to IPC server on port {ipc_port}: {exc}")
 
         self.ipc_socket = None
+        self._ipc_failure_count += 1
+        retry_delay = min(30.0, 0.5 * (2 ** min(self._ipc_failure_count - 1, 6)))
+        self._ipc_retry_after = time.monotonic() + retry_delay
         return False
 
     def _send_ipc_event(self, event_type: str, data: Any) -> None:
@@ -143,8 +155,8 @@ class DataEngine:
         encoded_payload = payload.encode("utf-8")
 
         with self._ipc_socket_lock:
-            for _attempt in range(2):
-                if not self._connect_ipc_locked():
+            for attempt in range(2):
+                if not self._connect_ipc_locked(force=attempt > 0):
                     return
                 try:
                     self.ipc_socket.sendall(encoded_payload)
@@ -156,6 +168,7 @@ class DataEngine:
                     except Exception:
                         pass
                     self.ipc_socket = None
+                    self._ipc_retry_after = 0.0
 
     @property
     def base_dir(self) -> Path:

artemis/data_engine/storage.py

@@ -895,6 +895,8 @@ class StorageManager:
         with self._get_connection() as conn:
             conn.execute("PRAGMA foreign_keys = OFF")
             tables = [
+                "video_analysis_observations",
+                "video_analysis_segments",
                 "failed_outputs",
                 "traces",
                 "background_tasks",
@@ -951,13 +953,16 @@ class StorageManager:
 
         # 1. Get video paths before deleting from DB
         video_paths = []
+        video_ids: list[str] = []
         try:
             with self._get_connection() as conn:
                 cursor = conn.execute(
-                    "SELECT local_video_path FROM video_recordings WHERE session_id = ?",
+                    "SELECT video_id, local_video_path FROM video_recordings WHERE session_id = ?",
                     (session_id_str,),
                 )
                 for row in cursor.fetchall():
+                    if row["video_id"]:
+                        video_ids.append(str(row["video_id"]))
                     if row["local_video_path"]:
                         video_paths.append(Path(row["local_video_path"]))
         except sqlite3.OperationalError as e:
@@ -966,6 +971,20 @@ class StorageManager:
         # 2. Delete from DB tables
         with self._get_connection() as conn:
             conn.execute("PRAGMA foreign_keys = OFF")
+            board_keys = [f"session:{session_id_str}"] + [
+                f"video:{video_id}" for video_id in video_ids
+            ]
+            for table in (
+                "video_analysis_observations",
+                "video_analysis_segments",
+            ):
+                try:
+                    conn.executemany(
+                        f"DELETE FROM {table} WHERE board_key = ?",
+                        [(key,) for key in board_keys],
+                    )
+                except sqlite3.OperationalError as e:
+                    logger.warning(f"Failed to delete video memory from {table}: {e}")
             tables = [
                 "failed_outputs",
                 "traces",

artemis/drivers/base.py

@@ -23,6 +23,7 @@ from abc import ABC, abstractmethod
 import asyncio
 from enum import Enum
 from pathlib import Path
+import re
 import time
 from typing import Any, Literal
 from pydantic import BaseModel, Field
@@ -249,9 +250,34 @@ class BaseDeviceDriver(ABC):
 
         elem = matched[index]
         bounds = elem.get("bounds")
-        center = elem.get("center")
-        if not center and bounds and len(bounds) == 4:
-            center = [(bounds[0] + bounds[2]) // 2, (bounds[1] + bounds[3]) // 2]
+        parsed_bounds = elem.get("parsed_bounds")
+        numeric_bounds: list[int] | None = None
+        if isinstance(parsed_bounds, dict):
+            keys = ("left", "top", "right", "bottom")
+            if all(isinstance(parsed_bounds.get(key), (int, float)) for key in keys):
+                numeric_bounds = [int(parsed_bounds[key]) for key in keys]
+        if numeric_bounds is None and isinstance(bounds, (list, tuple)) and len(bounds) == 4:
+            if all(isinstance(value, (int, float)) for value in bounds):
+                numeric_bounds = [int(value) for value in bounds]
+        if numeric_bounds is None and isinstance(bounds, str):
+            match = re.fullmatch(
+                r"\[\s*(-?\d+)\s*,\s*(-?\d+)\s*\]\[\s*(-?\d+)\s*,\s*(-?\d+)\s*\]",
+                bounds,
+            )
+            if match:
+                numeric_bounds = [int(value) for value in match.groups()]
+        # Bounds belong to the freshly captured hierarchy. Some UIAutomator
+        # adapters also attach a precomputed center which can lag one layout
+        # transition behind (for example when an expanding drawer shifts a
+        # keypad). Always derive the tap point from live bounds when possible;
+        # retain center only for adapters that do not expose bounds.
+        if numeric_bounds is not None:
+            center = [
+                (numeric_bounds[0] + numeric_bounds[2]) // 2,
+                (numeric_bounds[1] + numeric_bounds[3]) // 2,
+            ]
+        else:
+            center = elem.get("center")
         return elem, center, None
 
     async def tap_element(

artemis/sdk/builders/agent_config_builder.py

@@ -74,6 +74,7 @@ class AgentConfigBuilder:
         self._explorer = agent_cfg.explorer
         self._explorer_versions = agent_cfg.explorer_versions
         self._denylisted_tools = agent_cfg.denylisted_tools
+        self._video_analyzer = agent_cfg.video_analyzer
         self._enable_video_ledger = agent_cfg.video_analyzer.enable_ledger
         if agent_cfg.video_analyzer.enabled is not None:
             self._video_recording_tools_enabled = agent_cfg.video_analyzer.enabled
@@ -507,6 +508,9 @@ class AgentConfigBuilder:
             explorer_versions=self._explorer_versions,
             denylisted_tools=self._denylisted_tools,
             enable_video_ledger=self._enable_video_ledger,
+            video_analyzer=self._video_analyzer.model_copy(
+                update={"enable_ledger": self._enable_video_ledger}
+            ),
         )
 
 

artemis/sdk/types/agent.py

@@ -21,6 +21,7 @@ from artemis.config import (
     FlashProfileConfig,
     OutputterConfig,
     ProProfileConfig,
+    VideoAnalyzerConfig,
 )
 from artemis.context import DevicePlatform
 from artemis.sdk.types.task import AgentProfile, TaskRequestCommon
@@ -127,6 +128,7 @@ class AgentConfig(BaseModel):
     explorer_versions: dict[str, Literal["flash", "pro", "ultra"]]
     denylisted_tools: dict[str, list[str]]
     enable_video_ledger: bool = True
+    video_analyzer: VideoAnalyzerConfig = Field(default_factory=VideoAnalyzerConfig)
 
     model_config = {"arbitrary_types_allowed": True}
 

artemis/utils/video.py

@@ -118,6 +118,8 @@ class RecordingSession(BaseModel):
     android_segment_started_at: float | None = None
     android_segment_records: list[dict[str, Any]] = []
     android_conversion_tasks: list[asyncio.Task] = []
+    generation: int = 0
+    sealed_until: float = 0.0
     is_active: bool = True
     errors: list[str] = []
 
@@ -132,6 +134,10 @@ class VideoRecordingResult(BaseModel):
     duration_seconds: float | None = None
     actual_start_relative_time: float | None = None
     warning: str | None = None
+    video_id: UUID | None = None
+    generation: int | None = None
+    sealed_until: float | None = None
+    source_revision: str | None = None
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
@@ -798,6 +804,27 @@ async def extract_audio_from_video(input_path: Path) -> Path:
     if not input_path.exists():
         raise FileNotFoundError(f"Video file not found: {input_path}")
 
+    probe_cmd = [
+        get_ffprobe_path(),
+        "-v",
+        "error",
+        "-select_streams",
+        "a:0",
+        "-show_entries",
+        "stream=index",
+        "-of",
+        "csv=p=0",
+        str(input_path),
+    ]
+    probe = await asyncio.create_subprocess_exec(
+        *probe_cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    probe_stdout, _ = await probe.communicate()
+    if probe.returncode != 0 or not probe_stdout.strip():
+        raise ValueError("Video has no audio stream")
+
     output_path = input_path.parent / f"audio_{input_path.stem}.mp3"
     logger.info(f"Extracting audio from {input_path} to {output_path}")
 
@@ -823,17 +850,17 @@ async def extract_audio_from_video(input_path: Path) -> Path:
         _, stderr = await proc.communicate()
 
         if proc.returncode != 0:
-            err_msg = stderr.decode().strip()
-            logger.error(f"ffmpeg audio extraction failed: {err_msg}")
+            err_msg = stderr.decode(errors="replace").strip()
+            concise_error = "\n".join(err_msg.splitlines()[-8:])
+            logger.error(f"ffmpeg audio extraction failed: {concise_error}")
             raise RuntimeError(
-                f"ffmpeg audio extraction failed (code {proc.returncode}): {err_msg}"
+                f"ffmpeg audio extraction failed (code {proc.returncode}): {concise_error}"
             )
 
         return output_path
 
-    except Exception as e:
-        logger.error(f"Audio extraction failed: {e}")
-        raise e
+    except Exception:
+        raise
 
 
 def extract_keyframes_from_video(
@@ -880,3 +907,54 @@ def extract_keyframes_from_video(
 
     cap.release()
     return frames
+
+
+def extract_frames_at_timestamps(
+    video_path: Path | str,
+    timestamps: list[float],
+    *,
+    max_frames: int = 30,
+    max_dimension: int = 1080,
+) -> list[tuple[float, bytes]]:
+    """Seek to exact video-relative timestamps and return JPEG evidence frames."""
+
+    if max_frames <= 0:
+        return []
+    requested = sorted(
+        {
+            round(float(timestamp), 3)
+            for timestamp in timestamps
+            if isinstance(timestamp, (int, float)) and float(timestamp) >= 0.0
+        }
+    )[:max_frames]
+    if not requested:
+        return []
+
+    cap = cv2.VideoCapture(str(video_path))
+    if not cap.isOpened():
+        logger.error(f"Failed to open video file for targeted frame extraction: {video_path}")
+        return []
+
+    frames: list[tuple[float, bytes]] = []
+    try:
+        for timestamp in requested:
+            cap.set(cv2.CAP_PROP_POS_MSEC, timestamp * 1000.0)
+            ret, frame = cap.read()
+            if not ret:
+                continue
+            h, w = frame.shape[:2]
+            if max(h, w) > max_dimension:
+                scale = max_dimension / max(h, w)
+                frame = cv2.resize(
+                    frame,
+                    (max(1, int(w * scale)), max(1, int(h * scale))),
+                    interpolation=cv2.INTER_AREA,
+                )
+            encoded, buffer = cv2.imencode(
+                ".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 85]
+            )
+            if encoded:
+                frames.append((timestamp, buffer.tobytes()))
+    finally:
+        cap.release()
+    return frames