feat: settle mobile observations before summarizing steps

apps/admin_console/services/model_settings_service.py

@@ -64,7 +64,12 @@ def get_model_settings() -> dict:
         "max_retries": settings.OPENAI_MAX_RETRIES,
         "retry_interval_seconds": settings.OPENAI_RETRY_INTERVAL_SECONDS,
         "reasoning_effort": settings.OPENAI_REASONING_EFFORT,
-        "model_presets": ["gpt-5.6-luna", "mimo-v2.6-flash"],
+        "model_presets": [
+            "gpt-5.6-luna",
+            "gpt-6-luna",
+            "mimo-v2.6-flash",
+            "mimo-v2.6-pro",
+        ],
         "env_path": str(get_env_file()),
     }
 

apps/showcase_ui/src/app/components/model-settings/model-settings.component.spec.ts

@@ -7,7 +7,8 @@ import { CapabilitySettingsComponent } from './capability-settings.component';
 describe('持久化模型与功能设置', () => {
   let http: HttpTestingController;
   const config = { base_url: 'https://example.test/v1', model: 'mimo-v2.6-flash',
-    api_key_configured: true, timeout_seconds: 300, max_retries: 5, retry_interval_seconds: 10, reasoning_effort: 'high', env_path: '/app/.env' };
+    api_key_configured: true, timeout_seconds: 300, max_retries: 5, retry_interval_seconds: 10, reasoning_effort: 'high',
+    model_presets: ['gpt-5.6-luna', 'gpt-6-luna', 'mimo-v2.6-flash', 'mimo-v2.6-pro'], env_path: '/app/.env' };
   beforeEach(() => {
     TestBed.configureTestingModule({ providers: [provideHttpClient(), provideHttpClientTesting()] });
     http = TestBed.inject(HttpTestingController);
@@ -44,13 +45,23 @@ describe('持久化模型与功能设置', () => {
     await fixture.whenStable();
     const root: HTMLElement = fixture.nativeElement;
     const presets = root.querySelectorAll<HTMLButtonElement>('.model-presets button');
-    expect(presets.length).toBe(2);
+    expect(Array.from(presets, button => button.textContent?.trim())).toEqual([
+      'gpt-5.6-luna', 'gpt-6-luna', 'mimo-v2.6-flash', 'mimo-v2.6-pro'
+    ]);
     presets[0].click();
     fixture.detectChanges();
     await fixture.whenStable();
     const input = root.querySelector<HTMLInputElement>('#model-name')!;
     expect(input.value).toBe('gpt-5.6-luna');
     expect(presets[0].getAttribute('aria-pressed')).toBe('true');
+    presets[1].click();
+    fixture.detectChanges();
+    await fixture.whenStable();
+    expect(input.value).toBe('gpt-6-luna');
+    presets[3].click();
+    fixture.detectChanges();
+    await fixture.whenStable();
+    expect(input.value).toBe('mimo-v2.6-pro');
     expect(input.getAttribute('list')).toBeNull();
     expect(input.autocomplete).toBe('off');
     input.value = 'custom/model-v2';

apps/showcase_ui/src/app/components/model-settings/model-settings.component.ts

@@ -11,6 +11,7 @@ interface ModelSettings {
   max_retries: number;
   retry_interval_seconds: number;
   reasoning_effort: string;
+  model_presets: string[];
   env_path: string;
   message?: string;
 }
@@ -90,7 +91,7 @@ interface ModelSettings {
 export class ModelSettingsComponent implements OnInit {
   private http = inject(HttpClient);
   saved = output<void>();
-  readonly modelPresets = ['gpt-5.6-luna', 'mimo-v2.6-flash'];
+  modelPresets: string[] = [];
   baseUrl = ''; model = ''; apiKey = ''; timeout = 300; retries = 5; retryInterval = 10; reasoning = 'high';
   busy = signal<'save' | 'test' | null>(null);
   loading = signal(true); keyConfigured = signal(false); envPath = signal('');
@@ -112,6 +113,7 @@ export class ModelSettingsComponent implements OnInit {
   }
   private apply(config: ModelSettings): void {
     this.baseUrl = config.base_url; this.model = config.model; this.timeout = config.timeout_seconds;
+    this.modelPresets = config.model_presets;
     this.retries = config.max_retries; this.retryInterval = config.retry_interval_seconds ?? 10; this.reasoning = config.reasoning_effort ?? 'high'; this.keyConfigured.set(config.api_key_configured); this.envPath.set(config.env_path);
   }
   save(): void {

artemis/agents/flash/flash_runner.md

@@ -55,5 +55,5 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
 ---
 
 # 6. TERMINATION PROTOCOL (`report_task_status`)
-- **Completion (`status="completed"`)**: ONLY call `report_task_status(status="completed", ...)` after you have **visually verified** from the latest screen that the objective is fully achieved.
+- **Completion (`status="completed"`)**: ONLY call `report_task_status(status="completed", ...)` after the latest screenshot and UI elements support the claimed result. If those sources disagree during a transition, wait and observe again before concluding. Tie each claimed page title or value to an observed screenshot or UI element; mark any missing evidence as unverified.
 - **Failure (`status="failed"`)**: If the task is genuinely impossible due to paywalls, unrecoverable crashes, or hitting an absolute dead end after sensible fallbacks, call `report_task_status(status="failed", ...)` with the exact blocking reason.

artemis/agents/flash/flash_summarizer.md

@@ -24,6 +24,7 @@ You are the Step Summarizer for an Android UI automation agent. Your task is to
    - **Target & Action**: Identify what control/button was targeted (referencing the red visual indicator on the BEFORE screen) and the physical action performed.
    - **Objective Transition**: Describe the physical screen change (e.g., page transitioned, modal dialog opened, list scrolled revealing new items, checkbox/toggle toggled, keyboard appeared).
    - **Preserve Critical Data & Verifications**: Faithfully transcribe visible text, toast alerts, error banners, verification codes, prices, account names, or tracking numbers that appeared on the AFTER screen.
+3. **Reconcile both observation sources**: The optional AFTER ACTION UI ELEMENTS block is device hierarchy evidence from the same observation request. During a page animation its contents may be newer than the screenshot. If the image still shows the old page but the hierarchy names the new page, state both observations and that the transition is not yet visually settled; never conclude that the tap did nothing. Do not invent details absent from both sources. A controller's "Tapped" outcome only confirms input dispatch.
 
 ---
 

artemis/agents/flash/runner.py

@@ -842,6 +842,8 @@ class FlashRunner:
                     pre_img_bytes=pre_screenshot_bytes,
                     post_img_bytes=post_img_bytes,
                     exec_outcome=exec_result.text_summary,
+                    post_ui_elements_text=exec_result.ui_elements_text,
+                    observation_settled=(exec_result.metadata or {}).get("observation_settled"),
                     action_key=str(tc_id),
                     data_engine_step_id=recorded_step_id,
                     # The model's own reasoning for this turn steers the lens's

artemis/agents/flash/summarizer.py

@@ -227,6 +227,8 @@ class VisualStepSummarizer(StepMemoryService):
         action_key: str | None = None,
         data_engine_step_id: UUID | str | None = None,
         focus: dict[str, str] | None = None,
+        post_ui_elements_text: str | None = None,
+        observation_settled: bool | None = None,
     ) -> None:
         """Dispatches an asynchronous summarization task without blocking the caller.
 
@@ -257,6 +259,8 @@ class VisualStepSummarizer(StepMemoryService):
             "exec_outcome": exec_outcome,
             "data_engine_step_id": data_engine_step_id,
             "focus": focus,
+            "post_ui_elements_text": post_ui_elements_text,
+            "observation_settled": observation_settled,
         }
         self.submit(key, payload, aliases=aliases)
 
@@ -409,6 +413,10 @@ class VisualStepSummarizer(StepMemoryService):
             if verbatim and not self._phrase_carries(phrase, verbatim):
                 lead_lines.append(f"Action arguments (verbatim): {action_name}({verbatim})")
             lead_lines.append(f"Controller Outcome: {exec_outcome}")
+            if input_data.get("observation_settled") is False:
+                lead_lines.append(
+                    "Observation status: screen still changing; page state is not confirmed."
+                )
             focus = input_data.get("focus")
             if isinstance(focus, dict) and focus:
                 lead_lines.extend(["", render_focus_block(focus, dual=dual)])
@@ -457,6 +465,15 @@ class VisualStepSummarizer(StepMemoryService):
                     }
                 )
 
+            post_ui = input_data.get("post_ui_elements_text")
+            if post_bytes and isinstance(post_ui, str) and post_ui.strip():
+                content_blocks.append(
+                    {
+                        "type": "text",
+                        "text": "--- [3] AFTER ACTION UI ELEMENTS ---\n" + post_ui[:8000],
+                    }
+                )
+
             messages: list[BaseMessage] = [
                 SystemMessage(content=rendered_prompt),
                 HumanMessage(content=content_blocks),

artemis/mcp/action_executor.py

@@ -44,6 +44,7 @@ from artemis.mcp.action_manifest import OPTIONAL_ACTIONS, REQUIRED_ACTIONS
 from artemis.mcp.action_specs import exception_prefix
 from artemis.mcp.action_session import ActionSession, get_action_session
 from artemis.mcp.action_types import ActionCode, ActionResult
+from artemis.mcp.action_types import ObserveResult
 from artemis.mcp.actuators.adb import AdbActuator
 from artemis.agents.validator.tool_declarations import (
     ToolExecutionResult,
@@ -56,6 +57,7 @@ from artemis.utils.coordinates import (
     parse_swipe_parameters,
 )
 from artemis.utils.logger import get_logger
+from artemis.utils.image_hash import dhash_hex, hamming_distance_hex
 from artemis.utils.notes import (
     format_list_notes_failure,
     format_list_notes_success,
@@ -76,6 +78,9 @@ AGENT_TOOL_NAMES: frozenset[str] = (
 
 #: Actions whose ``target`` is a single point: an element index or an [x, y] pair.
 _POINT_TARGET_ACTIONS: frozenset[str] = frozenset({"click", "long_press", "input_text"})
+_SCREEN_CHANGING_ACTIONS: frozenset[str] = frozenset(
+    {"click", "click_sequence", "long_press", "input_text", "press_key", "swipe", "manage_app"}
+)
 
 
 class _ArgError(ValueError):
@@ -111,6 +116,99 @@ class McpActionExecutor:
             self._session = await get_action_session(self.ctx, actuator=self.actuator)
         return self._session
 
+    @staticmethod
+    def _image_hash(observation: ObserveResult) -> str | None:
+        path = observation.screenshot_path
+        if not path:
+            return None
+        try:
+            return dhash_hex(Path(path).read_bytes())
+        except OSError:
+            return None
+
+    @classmethod
+    def _same_screen(cls, first: ObserveResult, second: ObserveResult) -> bool:
+        """Compare two observations without treating a dispatched tap as navigation."""
+        if not first.ok or not second.ok:
+            return False
+        checks: list[bool] = []
+        first_hash, second_hash = cls._image_hash(first), cls._image_hash(second)
+        distance = hamming_distance_hex(first_hash, second_hash)
+        if distance is not None:
+            checks.append(distance <= 4)
+        if first.hierarchy_ok and second.hierarchy_ok:
+            checks.append(first.elements_text == second.elements_text)
+        return bool(checks) and all(checks)
+
+    @staticmethod
+    def _element_signature(elements: list[dict[str, Any]]) -> list[tuple[Any, Any, Any]]:
+        return [(e.get("text"), e.get("resource_id"), e.get("bounds")) for e in elements]
+
+    @classmethod
+    def _stale_image_with_new_hierarchy(
+        cls,
+        observation: ObserveResult,
+        previous_hash: str | None,
+        previous_elements: list[dict[str, Any]] | None,
+    ) -> bool:
+        if previous_hash is None or previous_elements is None or not observation.hierarchy_ok:
+            return False
+        distance = hamming_distance_hex(previous_hash, cls._image_hash(observation))
+        return (
+            distance is not None
+            and distance <= 4
+            and cls._element_signature(previous_elements)
+            != cls._element_signature(observation.elements)
+        )
+
+    async def _observe_after_action(
+        self,
+        session: ActionSession,
+        raw_name: str,
+        previous_screenshot: str | None,
+        previous_elements: list[dict[str, Any]] | None = None,
+    ) -> tuple[ObserveResult, bool]:
+        """Prefer a settled image/tree pair after actions that can change screens.
+
+        The Android helper captures its bitmap and hierarchy concurrently. During a
+        transition the bitmap can still show the old page while the hierarchy already
+        describes the new one. Bounded re-observation waits for adjacent stable
+        frames and rejects a new hierarchy paired with the previous page's bitmap.
+        """
+        first = await session.observe(settle_ms=400)
+        if not first.ok or raw_name not in _SCREEN_CHANGING_ACTIONS:
+            return first, first.ok
+
+        second = await session.observe(settle_ms=250)
+        if not second.ok:
+            return first, False
+        previous_hash = None
+        if previous_screenshot:
+            try:
+                previous_hash = dhash_hex(Path(previous_screenshot).read_bytes())
+            except OSError:
+                pass
+        if self._same_screen(first, second):
+            distance = hamming_distance_hex(previous_hash, self._image_hash(second))
+            if (distance is None or distance > 4) and not self._stale_image_with_new_hierarchy(
+                second, previous_hash, previous_elements
+            ):
+                return second, True
+
+        third = await session.observe(settle_ms=250)
+        if not third.ok:
+            return second, False
+        if self._same_screen(second, third) and not self._stale_image_with_new_hierarchy(
+            third, previous_hash, previous_elements
+        ):
+            return third, True
+        fourth = await session.observe(settle_ms=250)
+        if not fourth.ok:
+            return third, False
+        return fourth, self._same_screen(
+            third, fourth
+        ) and not self._stale_image_with_new_hierarchy(fourth, previous_hash, previous_elements)
+
     # --- Public entry ----------------------------------------------------------------
 
     async def execute(
@@ -171,6 +269,7 @@ class McpActionExecutor:
         xml_list: str | None = None
         target_semantics: dict[str, Any] = {}
         target_coordinates: list[int] | None = None
+        observation_settled: bool | None = None
         with span:
             try:
                 session = await self._session_or_start()
@@ -196,7 +295,12 @@ class McpActionExecutor:
                     message = finalize(res) if finalize else res.message
 
                 if res.ok or self._observe_despite_failure(raw_name, res):
-                    obs = await session.observe(settle_ms=400)
+                    obs, observation_settled = await self._observe_after_action(
+                        session,
+                        raw_name,
+                        getattr(state, "latest_screenshot", None),
+                        getattr(state, "indexed_elements", None),
+                    )
                     if obs.ok:
                         shot_path = obs.screenshot_path
                         xml_list = obs.elements_text
@@ -217,6 +321,8 @@ class McpActionExecutor:
 
             if shot_path:
                 state.latest_screenshot = shot_path
+            if observation_settled is False:
+                message = f"{message} Post-action screen is still changing; observe again before concluding."
 
             status = "success" if res.ok else "error"
             post_image_name = None
@@ -247,6 +353,7 @@ class McpActionExecutor:
                 "normalized_coordinates": res.normalized_coordinates,
                 "target_semantics": target_semantics,
                 "target_coordinates": target_coordinates,
+                "observation_settled": observation_settled,
             },
         )
 

tests/unit/admin_console/test_model_settings.py

@@ -70,6 +70,12 @@ def test_new_install_uses_playones_endpoint_without_overriding_saved_url(monkeyp
     assert service.get_model_settings()["base_url"] == "https://saved.example/v1"
 
 
+def test_model_presets_include_flash_pro_and_luna_without_restricting_custom_names():
+    presets = service.get_model_settings()["model_presets"]
+    assert presets == ["gpt-5.6-luna", "gpt-6-luna", "mimo-v2.6-flash", "mimo-v2.6-pro"]
+    assert service.ModelSettingsUpdate(model="custom/model-v2").model == "custom/model-v2"
+
+
 @pytest.mark.parametrize(
     "url",
     [

tests/unit/agents/flash/test_turn_index_snapshot.py

@@ -60,20 +60,21 @@ REFRESHED = [_el(1, "Back", 100, 100), _el(2, "Wi-Fi toggle", 900, 300)]
 
 
 class _FakeSession:
-    """An action session whose observations replay a scripted list sequence."""
+    """An action session whose screen advances after each device action."""
 
     started = True
 
     def __init__(self, observations: list[list[dict]]):
         self.calls: list[tuple[str, dict]] = []
-        self._observations = list(observations)
+        self._observations = observations
 
     async def call(self, name, args):
         self.calls.append((name, dict(args)))
         return ActionResult.success(name, f"{name} dispatched")
 
     async def observe(self, settle_ms=0):
-        elements = self._observations.pop(0) if self._observations else []
+        index = min(len(self.calls) - 1, len(self._observations) - 1)
+        elements = self._observations[index] if self._observations else []
         return ObserveResult(
             ok=True,
             elements_text="\n".join(f"[{e['index']}] {e['text']}" for e in elements),

tests/unit/agents/test_flash_step_summarizer.py

@@ -468,6 +468,8 @@ async def test_dual_frame_dispatch_keeps_transition_prompt(mock_context):
         pre_img_bytes=b"pre",
         post_img_bytes=b"post",
         exec_outcome="executed",
+        post_ui_elements_text="[1] Text: '存储空间' | Bounds: [0,0][100,100]",
+        observation_settled=False,
     )
     await summarizer.flush()
 
@@ -477,6 +479,10 @@ async def test_dual_frame_dispatch_keeps_transition_prompt(mock_context):
     assert "Before/After screenshots" in system_text
     assert "--- [1] BEFORE ACTION SCREEN (Action Marked Visually in Red) ---" in human_text
     assert "--- [2] AFTER ACTION SCREEN ---" in human_text
+    assert "--- [3] AFTER ACTION UI ELEMENTS ---" in human_text
+    assert "Text: '存储空间'" in human_text
+    assert "screen still changing" in human_text
+    assert "hierarchy names the new page" in system_text
     assert "DECISION FRAME" not in human_text
 
 

tests/unit/mcp/test_action_observation_settling.py

@@ -0,0 +1,104 @@
+"""Page transitions are recaptured until screenshot and hierarchy agree."""
+
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+from PIL import Image
+
+from artemis.mcp.action_executor import McpActionExecutor
+from artemis.mcp.action_types import ObserveResult
+
+
+def _screen(tmp_path, name: str, *, reverse: bool = False) -> str:
+    image = Image.new("L", (64, 64))
+    image.putdata([255 - x * 4 if reverse else x * 4 for _y in range(64) for x in range(64)])
+    path = tmp_path / name
+    image.save(path, format="JPEG")
+    return str(path)
+
+
+def _observation(path: str, page: str) -> ObserveResult:
+    return ObserveResult(
+        ok=True,
+        screenshot_path=path,
+        elements_text=f"[1] {page}",
+        elements=[{"text": page, "resource_id": "page/title", "bounds": [0, 0, 64, 64]}],
+    )
+
+
+@pytest.mark.asyncio
+async def test_navigation_recaptures_when_image_lags_behind_hierarchy(tmp_path):
+    home = _screen(tmp_path, "home.jpg")
+    detail = _screen(tmp_path, "detail.jpg", reverse=True)
+    session = Mock()
+    session.observe = AsyncMock(
+        side_effect=[
+            _observation(home, "存储空间"),
+            _observation(detail, "存储空间"),
+            _observation(detail, "存储空间"),
+        ]
+    )
+    executor = McpActionExecutor(Mock(), actuator=Mock())
+
+    observed, settled = await executor._observe_after_action(
+        session, "click", home, _observation(home, "设置首页").elements
+    )
+
+    assert observed.screenshot_path == detail
+    assert observed.elements_text == "[1] 存储空间"
+    assert settled is True
+    assert session.observe.await_count == 3
+
+
+@pytest.mark.asyncio
+async def test_navigation_waits_until_new_page_has_two_stable_frames(tmp_path):
+    home = _screen(tmp_path, "home.jpg")
+    detail = _screen(tmp_path, "detail.jpg", reverse=True)
+    session = Mock()
+    session.observe = AsyncMock(
+        side_effect=[
+            _observation(home, "设置首页"),
+            _observation(home, "设置首页"),
+            _observation(detail, "存储空间"),
+            _observation(detail, "存储空间"),
+        ]
+    )
+    executor = McpActionExecutor(Mock(), actuator=Mock())
+
+    observed, settled = await executor._observe_after_action(
+        session, "click", home, _observation(home, "设置首页").elements
+    )
+
+    assert observed.screenshot_path == detail
+    assert settled is True
+    assert session.observe.await_count == 4
+
+
+@pytest.mark.asyncio
+async def test_old_bitmap_with_new_hierarchy_is_never_marked_settled(tmp_path):
+    home = _screen(tmp_path, "home.jpg")
+    session = Mock()
+    session.observe = AsyncMock(return_value=_observation(home, "存储空间"))
+    executor = McpActionExecutor(Mock(), actuator=Mock())
+
+    observed, settled = await executor._observe_after_action(
+        session, "click", home, _observation(home, "设置首页").elements
+    )
+
+    assert observed.elements_text == "[1] 存储空间"
+    assert settled is False
+    assert session.observe.await_count == 4
+
+
+@pytest.mark.asyncio
+async def test_read_only_action_keeps_single_observation(tmp_path):
+    home = _screen(tmp_path, "home.jpg")
+    session = Mock()
+    session.observe = AsyncMock(return_value=_observation(home, "设置首页"))
+    executor = McpActionExecutor(Mock(), actuator=Mock())
+
+    observed, settled = await executor._observe_after_action(session, "wait_for_delay", home)
+
+    assert observed.screenshot_path == home
+    assert settled is True
+    session.observe.assert_awaited_once_with(settle_ms=400)