feat: settle mobile observations before summarizing steps

artemis/agents/flash/flash_runner.md

@@ -55,5 +55,5 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
 ---
 
 # 6. TERMINATION PROTOCOL (`report_task_status`)
-- **Completion (`status="completed"`)**: ONLY call `report_task_status(status="completed", ...)` after you have **visually verified** from the latest screen that the objective is fully achieved.
+- **Completion (`status="completed"`)**: ONLY call `report_task_status(status="completed", ...)` after the latest screenshot and UI elements support the claimed result. If those sources disagree during a transition, wait and observe again before concluding. Tie each claimed page title or value to an observed screenshot or UI element; mark any missing evidence as unverified.
 - **Failure (`status="failed"`)**: If the task is genuinely impossible due to paywalls, unrecoverable crashes, or hitting an absolute dead end after sensible fallbacks, call `report_task_status(status="failed", ...)` with the exact blocking reason.

artemis/agents/flash/flash_summarizer.md

@@ -24,6 +24,7 @@ You are the Step Summarizer for an Android UI automation agent. Your task is to
    - **Target & Action**: Identify what control/button was targeted (referencing the red visual indicator on the BEFORE screen) and the physical action performed.
    - **Objective Transition**: Describe the physical screen change (e.g., page transitioned, modal dialog opened, list scrolled revealing new items, checkbox/toggle toggled, keyboard appeared).
    - **Preserve Critical Data & Verifications**: Faithfully transcribe visible text, toast alerts, error banners, verification codes, prices, account names, or tracking numbers that appeared on the AFTER screen.
+3. **Reconcile both observation sources**: The optional AFTER ACTION UI ELEMENTS block is device hierarchy evidence from the same observation request. During a page animation its contents may be newer than the screenshot. If the image still shows the old page but the hierarchy names the new page, state both observations and that the transition is not yet visually settled; never conclude that the tap did nothing. Do not invent details absent from both sources. A controller's "Tapped" outcome only confirms input dispatch.
 
 ---
 

artemis/agents/flash/runner.py

@@ -842,6 +842,8 @@ class FlashRunner:
                     pre_img_bytes=pre_screenshot_bytes,
                     post_img_bytes=post_img_bytes,
                     exec_outcome=exec_result.text_summary,
+                    post_ui_elements_text=exec_result.ui_elements_text,
+                    observation_settled=(exec_result.metadata or {}).get("observation_settled"),
                     action_key=str(tc_id),
                     data_engine_step_id=recorded_step_id,
                     # The model's own reasoning for this turn steers the lens's

artemis/agents/flash/summarizer.py

@@ -227,6 +227,8 @@ class VisualStepSummarizer(StepMemoryService):
         action_key: str | None = None,
         data_engine_step_id: UUID | str | None = None,
         focus: dict[str, str] | None = None,
+        post_ui_elements_text: str | None = None,
+        observation_settled: bool | None = None,
     ) -> None:
         """Dispatches an asynchronous summarization task without blocking the caller.
 
@@ -257,6 +259,8 @@ class VisualStepSummarizer(StepMemoryService):
             "exec_outcome": exec_outcome,
             "data_engine_step_id": data_engine_step_id,
             "focus": focus,
+            "post_ui_elements_text": post_ui_elements_text,
+            "observation_settled": observation_settled,
         }
         self.submit(key, payload, aliases=aliases)
 
@@ -409,6 +413,10 @@ class VisualStepSummarizer(StepMemoryService):
             if verbatim and not self._phrase_carries(phrase, verbatim):
                 lead_lines.append(f"Action arguments (verbatim): {action_name}({verbatim})")
             lead_lines.append(f"Controller Outcome: {exec_outcome}")
+            if input_data.get("observation_settled") is False:
+                lead_lines.append(
+                    "Observation status: screen still changing; page state is not confirmed."
+                )
             focus = input_data.get("focus")
             if isinstance(focus, dict) and focus:
                 lead_lines.extend(["", render_focus_block(focus, dual=dual)])
@@ -457,6 +465,15 @@ class VisualStepSummarizer(StepMemoryService):
                     }
                 )
 
+            post_ui = input_data.get("post_ui_elements_text")
+            if post_bytes and isinstance(post_ui, str) and post_ui.strip():
+                content_blocks.append(
+                    {
+                        "type": "text",
+                        "text": "--- [3] AFTER ACTION UI ELEMENTS ---\n" + post_ui[:8000],
+                    }
+                )
+
             messages: list[BaseMessage] = [
                 SystemMessage(content=rendered_prompt),
                 HumanMessage(content=content_blocks),

artemis/mcp/action_executor.py

@@ -44,6 +44,7 @@ from artemis.mcp.action_manifest import OPTIONAL_ACTIONS, REQUIRED_ACTIONS
 from artemis.mcp.action_specs import exception_prefix
 from artemis.mcp.action_session import ActionSession, get_action_session
 from artemis.mcp.action_types import ActionCode, ActionResult
+from artemis.mcp.action_types import ObserveResult
 from artemis.mcp.actuators.adb import AdbActuator
 from artemis.agents.validator.tool_declarations import (
     ToolExecutionResult,
@@ -56,6 +57,7 @@ from artemis.utils.coordinates import (
     parse_swipe_parameters,
 )
 from artemis.utils.logger import get_logger
+from artemis.utils.image_hash import dhash_hex, hamming_distance_hex
 from artemis.utils.notes import (
     format_list_notes_failure,
     format_list_notes_success,
@@ -76,6 +78,9 @@ AGENT_TOOL_NAMES: frozenset[str] = (
 
 #: Actions whose ``target`` is a single point: an element index or an [x, y] pair.
 _POINT_TARGET_ACTIONS: frozenset[str] = frozenset({"click", "long_press", "input_text"})
+_SCREEN_CHANGING_ACTIONS: frozenset[str] = frozenset(
+    {"click", "click_sequence", "long_press", "input_text", "press_key", "swipe", "manage_app"}
+)
 
 
 class _ArgError(ValueError):
@@ -111,6 +116,99 @@ class McpActionExecutor:
             self._session = await get_action_session(self.ctx, actuator=self.actuator)
         return self._session
 
+    @staticmethod
+    def _image_hash(observation: ObserveResult) -> str | None:
+        path = observation.screenshot_path
+        if not path:
+            return None
+        try:
+            return dhash_hex(Path(path).read_bytes())
+        except OSError:
+            return None
+
+    @classmethod
+    def _same_screen(cls, first: ObserveResult, second: ObserveResult) -> bool:
+        """Compare two observations without treating a dispatched tap as navigation."""
+        if not first.ok or not second.ok:
+            return False
+        checks: list[bool] = []
+        first_hash, second_hash = cls._image_hash(first), cls._image_hash(second)
+        distance = hamming_distance_hex(first_hash, second_hash)
+        if distance is not None:
+            checks.append(distance <= 4)
+        if first.hierarchy_ok and second.hierarchy_ok:
+            checks.append(first.elements_text == second.elements_text)
+        return bool(checks) and all(checks)
+
+    @staticmethod
+    def _element_signature(elements: list[dict[str, Any]]) -> list[tuple[Any, Any, Any]]:
+        return [(e.get("text"), e.get("resource_id"), e.get("bounds")) for e in elements]
+
+    @classmethod
+    def _stale_image_with_new_hierarchy(
+        cls,
+        observation: ObserveResult,
+        previous_hash: str | None,
+        previous_elements: list[dict[str, Any]] | None,
+    ) -> bool:
+        if previous_hash is None or previous_elements is None or not observation.hierarchy_ok:
+            return False
+        distance = hamming_distance_hex(previous_hash, cls._image_hash(observation))
+        return (
+            distance is not None
+            and distance <= 4
+            and cls._element_signature(previous_elements)
+            != cls._element_signature(observation.elements)
+        )
+
+    async def _observe_after_action(
+        self,
+        session: ActionSession,
+        raw_name: str,
+        previous_screenshot: str | None,
+        previous_elements: list[dict[str, Any]] | None = None,
+    ) -> tuple[ObserveResult, bool]:
+        """Prefer a settled image/tree pair after actions that can change screens.
+
+        The Android helper captures its bitmap and hierarchy concurrently. During a
+        transition the bitmap can still show the old page while the hierarchy already
+        describes the new one. Bounded re-observation waits for adjacent stable
+        frames and rejects a new hierarchy paired with the previous page's bitmap.
+        """
+        first = await session.observe(settle_ms=400)
+        if not first.ok or raw_name not in _SCREEN_CHANGING_ACTIONS:
+            return first, first.ok
+
+        second = await session.observe(settle_ms=250)
+        if not second.ok:
+            return first, False
+        previous_hash = None
+        if previous_screenshot:
+            try:
+                previous_hash = dhash_hex(Path(previous_screenshot).read_bytes())
+            except OSError:
+                pass
+        if self._same_screen(first, second):
+            distance = hamming_distance_hex(previous_hash, self._image_hash(second))
+            if (distance is None or distance > 4) and not self._stale_image_with_new_hierarchy(
+                second, previous_hash, previous_elements
+            ):
+                return second, True
+
+        third = await session.observe(settle_ms=250)
+        if not third.ok:
+            return second, False
+        if self._same_screen(second, third) and not self._stale_image_with_new_hierarchy(
+            third, previous_hash, previous_elements
+        ):
+            return third, True
+        fourth = await session.observe(settle_ms=250)
+        if not fourth.ok:
+            return third, False
+        return fourth, self._same_screen(
+            third, fourth
+        ) and not self._stale_image_with_new_hierarchy(fourth, previous_hash, previous_elements)
+
     # --- Public entry ----------------------------------------------------------------
 
     async def execute(
@@ -171,6 +269,7 @@ class McpActionExecutor:
         xml_list: str | None = None
         target_semantics: dict[str, Any] = {}
         target_coordinates: list[int] | None = None
+        observation_settled: bool | None = None
         with span:
             try:
                 session = await self._session_or_start()
@@ -196,7 +295,12 @@ class McpActionExecutor:
                     message = finalize(res) if finalize else res.message
 
                 if res.ok or self._observe_despite_failure(raw_name, res):
-                    obs = await session.observe(settle_ms=400)
+                    obs, observation_settled = await self._observe_after_action(
+                        session,
+                        raw_name,
+                        getattr(state, "latest_screenshot", None),
+                        getattr(state, "indexed_elements", None),
+                    )
                     if obs.ok:
                         shot_path = obs.screenshot_path
                         xml_list = obs.elements_text
@@ -217,6 +321,8 @@ class McpActionExecutor:
 
             if shot_path:
                 state.latest_screenshot = shot_path
+            if observation_settled is False:
+                message = f"{message} Post-action screen is still changing; observe again before concluding."
 
             status = "success" if res.ok else "error"
             post_image_name = None
@@ -247,6 +353,7 @@ class McpActionExecutor:
                 "normalized_coordinates": res.normalized_coordinates,
                 "target_semantics": target_semantics,
                 "target_coordinates": target_coordinates,
+                "observation_settled": observation_settled,
             },
         )