diff --git a/apps/admin_console/services/model_settings_service.py b/apps/admin_console/services/model_settings_service.py index c348a1f..95ad6ca 100644 --- a/apps/admin_console/services/model_settings_service.py +++ b/apps/admin_console/services/model_settings_service.py @@ -64,7 +64,12 @@ def get_model_settings() -> dict: "max_retries": settings.OPENAI_MAX_RETRIES, "retry_interval_seconds": settings.OPENAI_RETRY_INTERVAL_SECONDS, "reasoning_effort": settings.OPENAI_REASONING_EFFORT, - "model_presets": ["gpt-5.6-luna", "mimo-v2.6-flash"], + "model_presets": [ + "gpt-5.6-luna", + "gpt-6-luna", + "mimo-v2.6-flash", + "mimo-v2.6-pro", + ], "env_path": str(get_env_file()), } diff --git a/apps/showcase_ui/src/app/components/model-settings/model-settings.component.spec.ts b/apps/showcase_ui/src/app/components/model-settings/model-settings.component.spec.ts index 6ef496e..2ac8b69 100644 --- a/apps/showcase_ui/src/app/components/model-settings/model-settings.component.spec.ts +++ b/apps/showcase_ui/src/app/components/model-settings/model-settings.component.spec.ts @@ -7,7 +7,8 @@ import { CapabilitySettingsComponent } from './capability-settings.component'; describe('持久化模型与功能设置', () => { let http: HttpTestingController; const config = { base_url: 'https://example.test/v1', model: 'mimo-v2.6-flash', - api_key_configured: true, timeout_seconds: 300, max_retries: 5, retry_interval_seconds: 10, reasoning_effort: 'high', env_path: '/app/.env' }; + api_key_configured: true, timeout_seconds: 300, max_retries: 5, retry_interval_seconds: 10, reasoning_effort: 'high', + model_presets: ['gpt-5.6-luna', 'gpt-6-luna', 'mimo-v2.6-flash', 'mimo-v2.6-pro'], env_path: '/app/.env' }; beforeEach(() => { TestBed.configureTestingModule({ providers: [provideHttpClient(), provideHttpClientTesting()] }); http = TestBed.inject(HttpTestingController); @@ -44,13 +45,23 @@ describe('持久化模型与功能设置', () => { await fixture.whenStable(); const root: HTMLElement = fixture.nativeElement; const presets = root.querySelectorAll('.model-presets button'); - expect(presets.length).toBe(2); + expect(Array.from(presets, button => button.textContent?.trim())).toEqual([ + 'gpt-5.6-luna', 'gpt-6-luna', 'mimo-v2.6-flash', 'mimo-v2.6-pro' + ]); presets[0].click(); fixture.detectChanges(); await fixture.whenStable(); const input = root.querySelector('#model-name')!; expect(input.value).toBe('gpt-5.6-luna'); expect(presets[0].getAttribute('aria-pressed')).toBe('true'); + presets[1].click(); + fixture.detectChanges(); + await fixture.whenStable(); + expect(input.value).toBe('gpt-6-luna'); + presets[3].click(); + fixture.detectChanges(); + await fixture.whenStable(); + expect(input.value).toBe('mimo-v2.6-pro'); expect(input.getAttribute('list')).toBeNull(); expect(input.autocomplete).toBe('off'); input.value = 'custom/model-v2'; diff --git a/apps/showcase_ui/src/app/components/model-settings/model-settings.component.ts b/apps/showcase_ui/src/app/components/model-settings/model-settings.component.ts index 750e179..baa0b0a 100644 --- a/apps/showcase_ui/src/app/components/model-settings/model-settings.component.ts +++ b/apps/showcase_ui/src/app/components/model-settings/model-settings.component.ts @@ -11,6 +11,7 @@ interface ModelSettings { max_retries: number; retry_interval_seconds: number; reasoning_effort: string; + model_presets: string[]; env_path: string; message?: string; } @@ -90,7 +91,7 @@ interface ModelSettings { export class ModelSettingsComponent implements OnInit { private http = inject(HttpClient); saved = output(); - readonly modelPresets = ['gpt-5.6-luna', 'mimo-v2.6-flash']; + modelPresets: string[] = []; baseUrl = ''; model = ''; apiKey = ''; timeout = 300; retries = 5; retryInterval = 10; reasoning = 'high'; busy = signal<'save' | 'test' | null>(null); loading = signal(true); keyConfigured = signal(false); envPath = signal(''); @@ -112,6 +113,7 @@ export class ModelSettingsComponent implements OnInit { } private apply(config: ModelSettings): void { this.baseUrl = config.base_url; this.model = config.model; this.timeout = config.timeout_seconds; + this.modelPresets = config.model_presets; this.retries = config.max_retries; this.retryInterval = config.retry_interval_seconds ?? 10; this.reasoning = config.reasoning_effort ?? 'high'; this.keyConfigured.set(config.api_key_configured); this.envPath.set(config.env_path); } save(): void { diff --git a/artemis/agents/flash/flash_runner.md b/artemis/agents/flash/flash_runner.md index 438e1ad..252eb9a 100644 --- a/artemis/agents/flash/flash_runner.md +++ b/artemis/agents/flash/flash_runner.md @@ -55,5 +55,5 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi --- # 6. TERMINATION PROTOCOL (`report_task_status`) -- **Completion (`status="completed"`)**: ONLY call `report_task_status(status="completed", ...)` after you have **visually verified** from the latest screen that the objective is fully achieved. +- **Completion (`status="completed"`)**: ONLY call `report_task_status(status="completed", ...)` after the latest screenshot and UI elements support the claimed result. If those sources disagree during a transition, wait and observe again before concluding. Tie each claimed page title or value to an observed screenshot or UI element; mark any missing evidence as unverified. - **Failure (`status="failed"`)**: If the task is genuinely impossible due to paywalls, unrecoverable crashes, or hitting an absolute dead end after sensible fallbacks, call `report_task_status(status="failed", ...)` with the exact blocking reason. diff --git a/artemis/agents/flash/flash_summarizer.md b/artemis/agents/flash/flash_summarizer.md index 85311b5..865a8da 100644 --- a/artemis/agents/flash/flash_summarizer.md +++ b/artemis/agents/flash/flash_summarizer.md @@ -24,6 +24,7 @@ You are the Step Summarizer for an Android UI automation agent. Your task is to - **Target & Action**: Identify what control/button was targeted (referencing the red visual indicator on the BEFORE screen) and the physical action performed. - **Objective Transition**: Describe the physical screen change (e.g., page transitioned, modal dialog opened, list scrolled revealing new items, checkbox/toggle toggled, keyboard appeared). - **Preserve Critical Data & Verifications**: Faithfully transcribe visible text, toast alerts, error banners, verification codes, prices, account names, or tracking numbers that appeared on the AFTER screen. +3. **Reconcile both observation sources**: The optional AFTER ACTION UI ELEMENTS block is device hierarchy evidence from the same observation request. During a page animation its contents may be newer than the screenshot. If the image still shows the old page but the hierarchy names the new page, state both observations and that the transition is not yet visually settled; never conclude that the tap did nothing. Do not invent details absent from both sources. A controller's "Tapped" outcome only confirms input dispatch. --- diff --git a/artemis/agents/flash/runner.py b/artemis/agents/flash/runner.py index 61fba03..cd467e6 100644 --- a/artemis/agents/flash/runner.py +++ b/artemis/agents/flash/runner.py @@ -842,6 +842,8 @@ class FlashRunner: pre_img_bytes=pre_screenshot_bytes, post_img_bytes=post_img_bytes, exec_outcome=exec_result.text_summary, + post_ui_elements_text=exec_result.ui_elements_text, + observation_settled=(exec_result.metadata or {}).get("observation_settled"), action_key=str(tc_id), data_engine_step_id=recorded_step_id, # The model's own reasoning for this turn steers the lens's diff --git a/artemis/agents/flash/summarizer.py b/artemis/agents/flash/summarizer.py index 2db238a..36fb2c6 100644 --- a/artemis/agents/flash/summarizer.py +++ b/artemis/agents/flash/summarizer.py @@ -227,6 +227,8 @@ class VisualStepSummarizer(StepMemoryService): action_key: str | None = None, data_engine_step_id: UUID | str | None = None, focus: dict[str, str] | None = None, + post_ui_elements_text: str | None = None, + observation_settled: bool | None = None, ) -> None: """Dispatches an asynchronous summarization task without blocking the caller. @@ -257,6 +259,8 @@ class VisualStepSummarizer(StepMemoryService): "exec_outcome": exec_outcome, "data_engine_step_id": data_engine_step_id, "focus": focus, + "post_ui_elements_text": post_ui_elements_text, + "observation_settled": observation_settled, } self.submit(key, payload, aliases=aliases) @@ -409,6 +413,10 @@ class VisualStepSummarizer(StepMemoryService): if verbatim and not self._phrase_carries(phrase, verbatim): lead_lines.append(f"Action arguments (verbatim): {action_name}({verbatim})") lead_lines.append(f"Controller Outcome: {exec_outcome}") + if input_data.get("observation_settled") is False: + lead_lines.append( + "Observation status: screen still changing; page state is not confirmed." + ) focus = input_data.get("focus") if isinstance(focus, dict) and focus: lead_lines.extend(["", render_focus_block(focus, dual=dual)]) @@ -457,6 +465,15 @@ class VisualStepSummarizer(StepMemoryService): } ) + post_ui = input_data.get("post_ui_elements_text") + if post_bytes and isinstance(post_ui, str) and post_ui.strip(): + content_blocks.append( + { + "type": "text", + "text": "--- [3] AFTER ACTION UI ELEMENTS ---\n" + post_ui[:8000], + } + ) + messages: list[BaseMessage] = [ SystemMessage(content=rendered_prompt), HumanMessage(content=content_blocks), diff --git a/artemis/mcp/action_executor.py b/artemis/mcp/action_executor.py index b3e565e..a0a2631 100644 --- a/artemis/mcp/action_executor.py +++ b/artemis/mcp/action_executor.py @@ -44,6 +44,7 @@ from artemis.mcp.action_manifest import OPTIONAL_ACTIONS, REQUIRED_ACTIONS from artemis.mcp.action_specs import exception_prefix from artemis.mcp.action_session import ActionSession, get_action_session from artemis.mcp.action_types import ActionCode, ActionResult +from artemis.mcp.action_types import ObserveResult from artemis.mcp.actuators.adb import AdbActuator from artemis.agents.validator.tool_declarations import ( ToolExecutionResult, @@ -56,6 +57,7 @@ from artemis.utils.coordinates import ( parse_swipe_parameters, ) from artemis.utils.logger import get_logger +from artemis.utils.image_hash import dhash_hex, hamming_distance_hex from artemis.utils.notes import ( format_list_notes_failure, format_list_notes_success, @@ -76,6 +78,9 @@ AGENT_TOOL_NAMES: frozenset[str] = ( #: Actions whose ``target`` is a single point: an element index or an [x, y] pair. _POINT_TARGET_ACTIONS: frozenset[str] = frozenset({"click", "long_press", "input_text"}) +_SCREEN_CHANGING_ACTIONS: frozenset[str] = frozenset( + {"click", "click_sequence", "long_press", "input_text", "press_key", "swipe", "manage_app"} +) class _ArgError(ValueError): @@ -111,6 +116,99 @@ class McpActionExecutor: self._session = await get_action_session(self.ctx, actuator=self.actuator) return self._session + @staticmethod + def _image_hash(observation: ObserveResult) -> str | None: + path = observation.screenshot_path + if not path: + return None + try: + return dhash_hex(Path(path).read_bytes()) + except OSError: + return None + + @classmethod + def _same_screen(cls, first: ObserveResult, second: ObserveResult) -> bool: + """Compare two observations without treating a dispatched tap as navigation.""" + if not first.ok or not second.ok: + return False + checks: list[bool] = [] + first_hash, second_hash = cls._image_hash(first), cls._image_hash(second) + distance = hamming_distance_hex(first_hash, second_hash) + if distance is not None: + checks.append(distance <= 4) + if first.hierarchy_ok and second.hierarchy_ok: + checks.append(first.elements_text == second.elements_text) + return bool(checks) and all(checks) + + @staticmethod + def _element_signature(elements: list[dict[str, Any]]) -> list[tuple[Any, Any, Any]]: + return [(e.get("text"), e.get("resource_id"), e.get("bounds")) for e in elements] + + @classmethod + def _stale_image_with_new_hierarchy( + cls, + observation: ObserveResult, + previous_hash: str | None, + previous_elements: list[dict[str, Any]] | None, + ) -> bool: + if previous_hash is None or previous_elements is None or not observation.hierarchy_ok: + return False + distance = hamming_distance_hex(previous_hash, cls._image_hash(observation)) + return ( + distance is not None + and distance <= 4 + and cls._element_signature(previous_elements) + != cls._element_signature(observation.elements) + ) + + async def _observe_after_action( + self, + session: ActionSession, + raw_name: str, + previous_screenshot: str | None, + previous_elements: list[dict[str, Any]] | None = None, + ) -> tuple[ObserveResult, bool]: + """Prefer a settled image/tree pair after actions that can change screens. + + The Android helper captures its bitmap and hierarchy concurrently. During a + transition the bitmap can still show the old page while the hierarchy already + describes the new one. Bounded re-observation waits for adjacent stable + frames and rejects a new hierarchy paired with the previous page's bitmap. + """ + first = await session.observe(settle_ms=400) + if not first.ok or raw_name not in _SCREEN_CHANGING_ACTIONS: + return first, first.ok + + second = await session.observe(settle_ms=250) + if not second.ok: + return first, False + previous_hash = None + if previous_screenshot: + try: + previous_hash = dhash_hex(Path(previous_screenshot).read_bytes()) + except OSError: + pass + if self._same_screen(first, second): + distance = hamming_distance_hex(previous_hash, self._image_hash(second)) + if (distance is None or distance > 4) and not self._stale_image_with_new_hierarchy( + second, previous_hash, previous_elements + ): + return second, True + + third = await session.observe(settle_ms=250) + if not third.ok: + return second, False + if self._same_screen(second, third) and not self._stale_image_with_new_hierarchy( + third, previous_hash, previous_elements + ): + return third, True + fourth = await session.observe(settle_ms=250) + if not fourth.ok: + return third, False + return fourth, self._same_screen( + third, fourth + ) and not self._stale_image_with_new_hierarchy(fourth, previous_hash, previous_elements) + # --- Public entry ---------------------------------------------------------------- async def execute( @@ -171,6 +269,7 @@ class McpActionExecutor: xml_list: str | None = None target_semantics: dict[str, Any] = {} target_coordinates: list[int] | None = None + observation_settled: bool | None = None with span: try: session = await self._session_or_start() @@ -196,7 +295,12 @@ class McpActionExecutor: message = finalize(res) if finalize else res.message if res.ok or self._observe_despite_failure(raw_name, res): - obs = await session.observe(settle_ms=400) + obs, observation_settled = await self._observe_after_action( + session, + raw_name, + getattr(state, "latest_screenshot", None), + getattr(state, "indexed_elements", None), + ) if obs.ok: shot_path = obs.screenshot_path xml_list = obs.elements_text @@ -217,6 +321,8 @@ class McpActionExecutor: if shot_path: state.latest_screenshot = shot_path + if observation_settled is False: + message = f"{message} Post-action screen is still changing; observe again before concluding." status = "success" if res.ok else "error" post_image_name = None @@ -247,6 +353,7 @@ class McpActionExecutor: "normalized_coordinates": res.normalized_coordinates, "target_semantics": target_semantics, "target_coordinates": target_coordinates, + "observation_settled": observation_settled, }, ) diff --git a/tests/unit/admin_console/test_model_settings.py b/tests/unit/admin_console/test_model_settings.py index eb2f7c1..3de5387 100644 --- a/tests/unit/admin_console/test_model_settings.py +++ b/tests/unit/admin_console/test_model_settings.py @@ -70,6 +70,12 @@ def test_new_install_uses_playones_endpoint_without_overriding_saved_url(monkeyp assert service.get_model_settings()["base_url"] == "https://saved.example/v1" +def test_model_presets_include_flash_pro_and_luna_without_restricting_custom_names(): + presets = service.get_model_settings()["model_presets"] + assert presets == ["gpt-5.6-luna", "gpt-6-luna", "mimo-v2.6-flash", "mimo-v2.6-pro"] + assert service.ModelSettingsUpdate(model="custom/model-v2").model == "custom/model-v2" + + @pytest.mark.parametrize( "url", [ diff --git a/tests/unit/agents/flash/test_turn_index_snapshot.py b/tests/unit/agents/flash/test_turn_index_snapshot.py index 0c6e422..f30efba 100644 --- a/tests/unit/agents/flash/test_turn_index_snapshot.py +++ b/tests/unit/agents/flash/test_turn_index_snapshot.py @@ -60,20 +60,21 @@ REFRESHED = [_el(1, "Back", 100, 100), _el(2, "Wi-Fi toggle", 900, 300)] class _FakeSession: - """An action session whose observations replay a scripted list sequence.""" + """An action session whose screen advances after each device action.""" started = True def __init__(self, observations: list[list[dict]]): self.calls: list[tuple[str, dict]] = [] - self._observations = list(observations) + self._observations = observations async def call(self, name, args): self.calls.append((name, dict(args))) return ActionResult.success(name, f"{name} dispatched") async def observe(self, settle_ms=0): - elements = self._observations.pop(0) if self._observations else [] + index = min(len(self.calls) - 1, len(self._observations) - 1) + elements = self._observations[index] if self._observations else [] return ObserveResult( ok=True, elements_text="\n".join(f"[{e['index']}] {e['text']}" for e in elements), diff --git a/tests/unit/agents/test_flash_step_summarizer.py b/tests/unit/agents/test_flash_step_summarizer.py index 572eaff..55086ff 100644 --- a/tests/unit/agents/test_flash_step_summarizer.py +++ b/tests/unit/agents/test_flash_step_summarizer.py @@ -468,6 +468,8 @@ async def test_dual_frame_dispatch_keeps_transition_prompt(mock_context): pre_img_bytes=b"pre", post_img_bytes=b"post", exec_outcome="executed", + post_ui_elements_text="[1] Text: '存储空间' | Bounds: [0,0][100,100]", + observation_settled=False, ) await summarizer.flush() @@ -477,6 +479,10 @@ async def test_dual_frame_dispatch_keeps_transition_prompt(mock_context): assert "Before/After screenshots" in system_text assert "--- [1] BEFORE ACTION SCREEN (Action Marked Visually in Red) ---" in human_text assert "--- [2] AFTER ACTION SCREEN ---" in human_text + assert "--- [3] AFTER ACTION UI ELEMENTS ---" in human_text + assert "Text: '存储空间'" in human_text + assert "screen still changing" in human_text + assert "hierarchy names the new page" in system_text assert "DECISION FRAME" not in human_text diff --git a/tests/unit/mcp/test_action_observation_settling.py b/tests/unit/mcp/test_action_observation_settling.py new file mode 100644 index 0000000..91eb148 --- /dev/null +++ b/tests/unit/mcp/test_action_observation_settling.py @@ -0,0 +1,104 @@ +"""Page transitions are recaptured until screenshot and hierarchy agree.""" + +from unittest.mock import AsyncMock, Mock + +import pytest +from PIL import Image + +from artemis.mcp.action_executor import McpActionExecutor +from artemis.mcp.action_types import ObserveResult + + +def _screen(tmp_path, name: str, *, reverse: bool = False) -> str: + image = Image.new("L", (64, 64)) + image.putdata([255 - x * 4 if reverse else x * 4 for _y in range(64) for x in range(64)]) + path = tmp_path / name + image.save(path, format="JPEG") + return str(path) + + +def _observation(path: str, page: str) -> ObserveResult: + return ObserveResult( + ok=True, + screenshot_path=path, + elements_text=f"[1] {page}", + elements=[{"text": page, "resource_id": "page/title", "bounds": [0, 0, 64, 64]}], + ) + + +@pytest.mark.asyncio +async def test_navigation_recaptures_when_image_lags_behind_hierarchy(tmp_path): + home = _screen(tmp_path, "home.jpg") + detail = _screen(tmp_path, "detail.jpg", reverse=True) + session = Mock() + session.observe = AsyncMock( + side_effect=[ + _observation(home, "存储空间"), + _observation(detail, "存储空间"), + _observation(detail, "存储空间"), + ] + ) + executor = McpActionExecutor(Mock(), actuator=Mock()) + + observed, settled = await executor._observe_after_action( + session, "click", home, _observation(home, "设置首页").elements + ) + + assert observed.screenshot_path == detail + assert observed.elements_text == "[1] 存储空间" + assert settled is True + assert session.observe.await_count == 3 + + +@pytest.mark.asyncio +async def test_navigation_waits_until_new_page_has_two_stable_frames(tmp_path): + home = _screen(tmp_path, "home.jpg") + detail = _screen(tmp_path, "detail.jpg", reverse=True) + session = Mock() + session.observe = AsyncMock( + side_effect=[ + _observation(home, "设置首页"), + _observation(home, "设置首页"), + _observation(detail, "存储空间"), + _observation(detail, "存储空间"), + ] + ) + executor = McpActionExecutor(Mock(), actuator=Mock()) + + observed, settled = await executor._observe_after_action( + session, "click", home, _observation(home, "设置首页").elements + ) + + assert observed.screenshot_path == detail + assert settled is True + assert session.observe.await_count == 4 + + +@pytest.mark.asyncio +async def test_old_bitmap_with_new_hierarchy_is_never_marked_settled(tmp_path): + home = _screen(tmp_path, "home.jpg") + session = Mock() + session.observe = AsyncMock(return_value=_observation(home, "存储空间")) + executor = McpActionExecutor(Mock(), actuator=Mock()) + + observed, settled = await executor._observe_after_action( + session, "click", home, _observation(home, "设置首页").elements + ) + + assert observed.elements_text == "[1] 存储空间" + assert settled is False + assert session.observe.await_count == 4 + + +@pytest.mark.asyncio +async def test_read_only_action_keeps_single_observation(tmp_path): + home = _screen(tmp_path, "home.jpg") + session = Mock() + session.observe = AsyncMock(return_value=_observation(home, "设置首页")) + executor = McpActionExecutor(Mock(), actuator=Mock()) + + observed, settled = await executor._observe_after_action(session, "wait_for_delay", home) + + assert observed.screenshot_path == home + assert settled is True + session.observe.assert_awaited_once_with(settle_ms=400)