fix: improve cross-device evidence, request diagnostics and prompt efficiency

artemis/agents/flash/context_compressor.py

@@ -143,7 +143,11 @@ class ScrubEdgeCompressor:
         self._xml_candidates: list[int] = []
 
     def compress(
-        self, messages: list[BaseMessage], *, image_scrub_depth: int | None = None
+        self,
+        messages: list[BaseMessage],
+        *,
+        image_scrub_depth: int | None = None,
+        protected_message_ids: set[int] | None = None,
     ) -> None:
         """Advance the scrub edge over newly appended messages in-place.
 
@@ -156,7 +160,7 @@ class ScrubEdgeCompressor:
         self._discover(messages)
         if not self._tracked and not self._xml_candidates:
             return
-        self._scrub_edge(messages, image_scrub_depth)
+        self._scrub_edge(messages, image_scrub_depth, protected_message_ids)
 
     # ------------------------------------------------------------------
     # Discovery (incremental; frozen prefix is never rescanned)
@@ -211,7 +215,10 @@ class ScrubEdgeCompressor:
     # ------------------------------------------------------------------
 
     def _scrub_edge(
-        self, messages: list[BaseMessage], image_scrub_depth: int | None = None
+        self,
+        messages: list[BaseMessage],
+        image_scrub_depth: int | None = None,
+        protected_message_ids: set[int] | None = None,
     ) -> None:
         total = len(self._tracked)
         scrub_depth = max(1, int(image_scrub_depth or self._image_scrub_depth))
@@ -219,6 +226,8 @@ class ScrubEdgeCompressor:
         text_depth = self._xml_scrub_depth
 
         for pos, rec in enumerate(self._tracked):
+            if protected_message_ids and id(messages[rec["idx"]]) in protected_message_ids:
+                continue  # A newly delivered tool result must be seen before it can be scrubbed.
             if rec["idx"] in self._frozen:
                 continue
             # 1 == most recent image-bearing message; a live tail outside the

artemis/agents/flash/flash_runner.md

@@ -57,4 +57,5 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
 
 # 6. TERMINATION PROTOCOL (`report_task_status`)
 - **Completion (`status="completed"`)**: ONLY call `report_task_status(status="completed", ...)` after the latest screenshot and UI elements support the claimed result. If those sources disagree during a transition, wait and observe again before concluding. Tie each claimed page title or value to an observed screenshot or UI element; mark any missing evidence as unverified.
+- **Deliver the requested output**: Completing device actions does not complete a requested report. Before finishing, check the original goal and user guidance for required deliverables. Put the actual report/answer in the explanation field, covering the requested sections with observed findings, source-specific evidence and untested or blocked scope. Do not replace it with "completed" or a promise to provide it later. The short-paragraph rule above applies to turn commentary, not to this final deliverable. Use the existing record; retrieve only missing evidence instead of repeating the whole test. Do not invent findings or treat finding a product defect as execution failure when the goal is to investigate and report it.
 - **Failure (`status="failed"`)**: If the task is genuinely impossible due to paywalls, unrecoverable crashes, or hitting an absolute dead end after sensible fallbacks, call `report_task_status(status="failed", ...)` with the exact blocking reason.

artemis/agents/flash/runner.py

@@ -147,6 +147,7 @@ class FlashRunner:
         self.ctx = ctx
         self.goal = goal
         self.final_report_received = False
+        self._android_observation_stale = False
         try:
             cfg = load_agent_config()
             self.max_turns = max_turns if max_turns is not None else cfg.flash.max_turns
@@ -746,6 +747,9 @@ class FlashRunner:
                         "error_code",
                         "attachments",
                         "recording",
+                        "pre_image_name",
+                        "post_image_name",
+                        "pre_image_valid",
                     )
                     if key in metadata
                 }
@@ -865,6 +869,13 @@ class FlashRunner:
                 # A phone observation cannot serve as the before-image for a desktop action.
                 pre_screenshot_bytes = plugins.previous_image(name, args)
                 xml_list = ""
+                if getattr(self.ctx, "execution_target", "android") != "computer":
+                    # Invalidate before dispatch: even a failed/interrupted PC
+                    # action may have changed the environment or the phone.
+                    self._android_observation_stale = True
+                    state.indexed_elements = []
+                    state.indexed_points = []
+                    state.latest_screenshot = None
             control = getattr(self.ctx, "task_control", None)
             if plugins and name in plugins.tool_names and isinstance(control, TaskControl):
                 interrupted, exec_result = await control.run_interruptibly(
@@ -936,6 +947,11 @@ class FlashRunner:
                     exec_outcome=exec_result.text_summary,
                     post_ui_elements_text=exec_result.ui_elements_text,
                     observation_settled=(exec_result.metadata or {}).get("observation_settled"),
+                    receipt_only=bool(
+                        plugin_action
+                        and exec_result.metadata.get("read_only") is True
+                        and exec_result.status == "success"
+                    ),
                     action_key=str(tc_id),
                     data_engine_step_id=recorded_step_id,
                     # The model's own reasoning for this turn steers the lens's
@@ -988,16 +1004,41 @@ class FlashRunner:
                 )
             )
         if plugin_action and getattr(self.ctx, "execution_target", "android") != "computer":
-            # The loop's observation always belongs to Android in a mixed task.
-            # Plugin images travel in their labelled tool result and step record;
-            # never pair a desktop bitmap with the phone's element indexes.
-            shot_path, observation_image, observation_text = await capture_screenshot_and_parse_ui(
-                self.ctx, state, self.controller
-            )
-            state.latest_screenshot = shot_path
-            return observation_image, observation_text, action_sequence
+            # The labelled tool result already carries the PC evidence. Do not
+            # capture an idle phone or duplicate the PC bitmap in the phone tail.
+            return None, self._plugin_observation_notice(), action_sequence
         return pre_screenshot_bytes, xml_list, action_sequence
 
+    @staticmethod
+    def _plugin_observation_notice() -> str:
+        return (
+            "当前使用插件观察；实际画面和控件见对应工具返回，不是安卓画面。"
+            "安卓旧截图、元素编号和坐标已失效。连续操作电脑无需重复观察手机；"
+            "切回安卓时先单独调用 wait_for_delay(time_in_ms=0) 读取最新手机画面，"
+            "收到后再决定手机动作。不要将电脑控件编号用于手机。"
+        )
+
+    async def _refresh_observation(self, state: State):
+        shot_path, image, elements = await capture_screenshot_and_parse_ui(
+            self.ctx, state, self.controller, skip_settling=False
+        )
+        state.latest_screenshot = shot_path
+        self._android_observation_stale = False
+        return image, elements
+
+    @staticmethod
+    def _defer_tool_calls(messages: list[BaseMessage], tool_calls: list) -> None:
+        """Close every pending call without executing decisions made on stale evidence."""
+        for tc in tool_calls:
+            messages.append(
+                ToolMessage(
+                    tool_call_id=tc.get("id") or str(uuid.uuid4()),
+                    name=tc["name"],
+                    content="未执行此调用：已切回安卓并重新观察。请先读取最新手机画面和控件，再重新决定动作或结论。",
+                    status="error",
+                )
+            )
+
     async def _invoke_model(self, llm, current_tools: list, messages: list[BaseMessage]):
         """Binds the active tools and invokes the model through the LLM gateway."""
         # Bind active tools
@@ -1048,16 +1089,31 @@ class FlashRunner:
         final_report is non-None only when 'report_task_status' was called.
         """
         action_names = self.executor.action_tool_names | {"android_file"}
+        from artemis.plugins.runtime import get_plugin_runtime
+
+        plugins = get_plugin_runtime(self.ctx)
+        android_tools = action_names | {"ask_explorer"}
+        if plugins:
+            android_tools -= plugins.tool_names
         index_elements = self._turn_index_snapshot(state)
-        for tc in tool_calls:
+        for call_index, tc in enumerate(tool_calls):
             control = getattr(self.ctx, "task_control", None)
             if isinstance(control, TaskControl) and has_pending_instruction(control.directory):
                 return None, pre_screenshot_bytes, xml_list, action_sequence
             name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
             args = tc.get("args") or {}
             tc_id = tc.get("id") or str(uuid.uuid4())
-            logger.info(f"Executing Flash tool: {name}({args})")
 
+            returning_to_android = (
+                getattr(self, "_android_observation_stale", False) and name in android_tools
+            )
+            if returning_to_android and name != "wait_for_delay":
+                logger.info(f"Flash 暂缓 {name}：切回安卓，先刷新画面并重新决定动作。")
+                pre_screenshot_bytes, xml_list = await self._refresh_observation(state)
+                self._defer_tool_calls(messages, tool_calls[call_index:])
+                return None, pre_screenshot_bytes, xml_list, action_sequence
+
+            logger.info(f"Executing Flash tool: {name}({args})")
             if name == "report_task_status":
                 final_report = await self._finalize_task_report(
                     name,
@@ -1090,6 +1146,15 @@ class FlashRunner:
                 native_text=native_text,
                 index_elements=index_elements,
             )
+            if returning_to_android:
+                # wait_for_delay has no stale target and its normal post-action
+                # observation is fresh. The model must see it before any further
+                # calls (including a report) from this pre-observation batch.
+                if pre_screenshot_bytes is None:
+                    pre_screenshot_bytes, xml_list = await self._refresh_observation(state)
+                self._android_observation_stale = False
+                self._defer_tool_calls(messages, tool_calls[call_index + 1 :])
+                return None, pre_screenshot_bytes, xml_list, action_sequence
             if name not in action_names:
                 index_elements = self._extend_index_snapshot(state, index_elements, before_len)
         return None, pre_screenshot_bytes, xml_list, action_sequence
@@ -1135,10 +1200,7 @@ class FlashRunner:
         )
 
         # Capture Initial State (Screenshot + UI Tree)
-        shot_path, img_bytes, xml_list = await capture_screenshot_and_parse_ui(
-            self.ctx, state, self.controller, skip_settling=False
-        )
-        state.latest_screenshot = shot_path
+        img_bytes, xml_list = await self._refresh_observation(state)
         return ledger, img_bytes, xml_list
 
     @staticmethod
@@ -1196,6 +1258,7 @@ class FlashRunner:
         fallback_llm = None
         using_tool_fallback = False
         action_failure_attempts = 0
+        refreshed_for_guidance = False
 
         while limit is None or turns < limit:
             control = getattr(self.ctx, "task_control", None)
@@ -1210,13 +1273,10 @@ class FlashRunner:
                     action_failure_attempts = 0
                     self._llm = llm
                     (
-                        shot_path,
                         current_pre_screenshot_bytes,
                         current_xml_list,
-                    ) = await capture_screenshot_and_parse_ui(
-                        self.ctx, state, self.controller, skip_settling=False
-                    )
-                    state.latest_screenshot = shot_path
+                    ) = await self._refresh_observation(state)
+                    refreshed_for_guidance = True
                     pending_notices.append(
                         "设备在暂停期间可能被其他任务操作过。已重新观察画面；"
                         "请根据当前状态和新指导重新判断操作，勿沿用旧坐标。"
@@ -1236,6 +1296,11 @@ class FlashRunner:
             injected, injected_guidance = await self._read_injected_instruction(
                 ledger.elapsed_label()
             )
+            if injected and not refreshed_for_guidance:
+                current_pre_screenshot_bytes, current_xml_list = await self._refresh_observation(
+                    state
+                )
+            refreshed_for_guidance = False
 
             # Tool restriction on the final turn (bounded loops only)
             is_final = limit is not None and turns == limit
@@ -1283,14 +1348,10 @@ class FlashRunner:
                 no_tool_attempts = 0
                 action_failure_attempts = 0
                 self._llm = llm
-                (
-                    shot_path,
-                    current_pre_screenshot_bytes,
-                    current_xml_list,
-                ) = await capture_screenshot_and_parse_ui(
-                    self.ctx, state, self.controller, skip_settling=False
+                current_pre_screenshot_bytes, current_xml_list = await self._refresh_observation(
+                    state
                 )
-                state.latest_screenshot = shot_path
+                refreshed_for_guidance = True
                 pending_notices.append(
                     "新指导已打断旧模型步骤。请重新观察画面并按最新指导决定下一步。"
                 )
@@ -1326,14 +1387,10 @@ class FlashRunner:
                 using_tool_fallback = False
                 no_tool_attempts = 0
                 self._llm = llm
-                (
-                    shot_path,
-                    current_pre_screenshot_bytes,
-                    current_xml_list,
-                ) = await capture_screenshot_and_parse_ui(
-                    self.ctx, state, self.controller, skip_settling=False
+                current_pre_screenshot_bytes, current_xml_list = await self._refresh_observation(
+                    state
                 )
-                state.latest_screenshot = shot_path
+                refreshed_for_guidance = True
                 pending_notices.append("新指导已收到；忽略旧模型动作，重新观察画面。")
                 turns -= 1
                 continue
@@ -1421,12 +1478,9 @@ class FlashRunner:
                     break
                 # A fresh, valid observation is required before spending any
                 # fallback model calls. Hardware recovery is not model retry.
-                (
-                    shot_path,
-                    current_pre_screenshot_bytes,
-                    current_xml_list,
-                ) = await capture_screenshot_and_parse_ui(self.ctx, state, self.controller)
-                state.latest_screenshot = shot_path
+                current_pre_screenshot_bytes, current_xml_list = await self._refresh_observation(
+                    state
+                )
                 fallback_llm = get_llm(self.ctx, name="operator", use_fallback=True)
                 using_tool_fallback = True
                 self._llm = fallback_llm

artemis/agents/flash/summarizer.py

@@ -29,6 +29,7 @@ import asyncio
 from artemis.config.settings import settings
 import base64
 from pathlib import Path
+import sqlite3
 from typing import Any
 from uuid import UUID
 
@@ -231,6 +232,7 @@ class VisualStepSummarizer(StepMemoryService):
         focus: dict[str, str] | None = None,
         post_ui_elements_text: str | None = None,
         observation_settled: bool | None = None,
+        receipt_only: bool = False,
     ) -> None:
         """Dispatches an asynchronous summarization task without blocking the caller.
 
@@ -252,6 +254,16 @@ class VisualStepSummarizer(StepMemoryService):
         else:
             key = action_key if action_key is not None else step_number
 
+        if action_name.startswith("plug_"):
+            from artemis.config.constants import PLUGIN_SUMMARY_TEXT_CHARS
+            from artemis.plugins.prompt_budget import bounded_result_text
+
+            exec_outcome = bounded_result_text(exec_outcome, limit=PLUGIN_SUMMARY_TEXT_CHARS)
+            if post_ui_elements_text:
+                post_ui_elements_text = bounded_result_text(
+                    post_ui_elements_text, limit=PLUGIN_SUMMARY_TEXT_CHARS
+                )
+
         payload = {
             "step_number": step_number,
             "action_name": action_name,
@@ -264,6 +276,30 @@ class VisualStepSummarizer(StepMemoryService):
             "post_ui_elements_text": post_ui_elements_text,
             "observation_settled": observation_settled,
         }
+        if receipt_only:
+            from artemis.config.constants import PLUGIN_SUMMARY_TEXT_CHARS
+            from artemis.plugins.prompt_budget import bounded_result_text
+
+            summary = (
+                f"步骤 {step_number}：只读观察 {action_name}；未执行写操作。"
+                "以下为工具返回摘录，截图仍在本步骤证据中；不代表视觉验收通过。\n"
+                + bounded_result_text(exec_outcome, limit=PLUGIN_SUMMARY_TEXT_CHARS)
+            )
+            # Evidence has already been persisted by the caller. As with a
+            # completed visual summary, do not retain every frame in memory.
+            payload.update(pre_img_bytes=None, post_img_bytes=None, focus=None)
+            self._step_inputs[key] = payload
+            self._summaries[key] = summary
+            for alias in aliases:
+                self._aliases[alias] = key
+            if data_engine_step_id is not None and self.ctx.data_engine:
+                try:
+                    self.ctx.data_engine.update_step_summary(
+                        data_engine_step_id, summary, status="ready", source="tool_receipt"
+                    )
+                except (OSError, ValueError, sqlite3.Error) as de_err:
+                    logger.debug(f"DataEngine receipt summary update skipped: {de_err}")
+            return
         self.submit(key, payload, aliases=aliases)
 
     @staticmethod