fix: improve cross-device evidence, request diagnostics and prompt efficiency

artemis/plugins/models.py

@@ -214,6 +214,7 @@ class PluginCallResult(PluginModel):
     tool: str = ""
     image_names: list[str] = Field(default_factory=list)
     call_id: str = ""
+    text_ref: str = ""
     source: EvidenceSource | None = None
     source_id: str = ""
     error_code: str = ""

artemis/plugins/prompt_budget.py

@@ -0,0 +1,56 @@
+"""Bound model-facing observations while keeping full, task-scoped evidence."""
+
+from pathlib import Path
+import re
+from typing import Any
+
+from artemis.config.constants import PLUGIN_PROMPT_TEXT_CHARS
+
+
+def result_path(engine: Any, call_id: str) -> Path:
+    if not re.fullmatch(r"[a-f0-9]{32}", call_id):
+        raise ValueError("插件结果编号无效")
+    root = Path(engine.base_dir).resolve()
+    path = root / "evidence" / "plugin-results" / f"{call_id}.txt"
+    if not path.resolve().is_relative_to(root):
+        raise ValueError("插件结果路径超出任务目录")
+    return path
+
+
+def save_result_text(engine: Any, call_id: str, text: str) -> None:
+    path = result_path(engine, call_id)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text, encoding="utf-8")
+
+
+def bounded_result_text(
+    text: str, *, call_id: str = "", limit: int = PLUGIN_PROMPT_TEXT_CHARS
+) -> str:
+    if len(text) <= limit:
+        return text
+    reference = (
+        f'完整结果可用 plugin_read_result(call_id="{call_id}", offset=0) 分页读取。'
+        if call_id
+        else "完整结果保存在本步骤的 trace；需要目标控件时可缩小观察范围重新读取。"
+    )
+    notice = f"\n[观察文本已节选，原文 {len(text)} 字符。{reference}]\n"
+    available = max(0, limit - len(notice))
+    if not available:
+        return notice[:limit]
+    head = available * 3 // 4
+    return text[:head] + notice + text[-(available - head) :]
+
+
+def read_result_text(engine: Any, call_id: str, offset: int = 0) -> str:
+    if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
+        raise ValueError("offset 必须为非负整数")
+    text = result_path(engine, call_id).read_text(encoding="utf-8")
+    if offset > len(text):
+        raise ValueError(f"offset 超出结果长度 {len(text)}")
+    end = min(len(text), offset + PLUGIN_PROMPT_TEXT_CHARS)
+    return (
+        f"插件结果 {call_id}，字符 {offset}:{end}/{len(text)}；"
+        + (f"下一页 offset={end}" if end < len(text) else "已到结尾")
+        + "\n"
+        + text[offset:end]
+    )

artemis/plugins/results.py

@@ -0,0 +1,56 @@
+"""Deliver persisted plugin results without substituting the Android observation."""
+
+import base64
+import io
+from pathlib import Path
+from typing import Any
+
+from langchain_core.messages import BaseMessage
+from PIL import Image
+
+from artemis.tools.tool_wrapper import tool_result_messages
+from artemis.plugins.prompt_budget import bounded_result_text
+
+
+def plugin_result_messages(report: Any, engine: Any) -> dict[str, list[BaseMessage]]:
+    if not isinstance(report, dict):
+        return {}
+    results = {}
+    for action in report.get("execution", []):
+        if not isinstance(action, dict) or not str(action.get("action", "")).startswith("plug_"):
+            continue
+        call_id = action.get("tool_call_id")
+        if not call_id:
+            continue
+        status = action.get("plugin_status")
+        text = action.get("result") or "此插件操作未完成或未执行；请重新观察后再决定下一步。"
+        text = bounded_result_text(str(text), call_id=str(action.get("text_ref") or ""))
+        blocks = [{"type": "text", "text": text}]
+        names = action.get("image_names") or [action.get("post_image_name")]
+        for name in dict.fromkeys(names):
+            if not name or engine is None:
+                continue
+            try:
+                image = Path(engine.get_image_path(name)).read_bytes()
+                # DataEngine keeps the original bytes under a .jpg filename;
+                # computer plugins may actually return PNG or WebP.
+                with Image.open(io.BytesIO(image)) as captured:
+                    mime = Image.MIME.get(captured.format)
+                if mime not in {"image/jpeg", "image/png", "image/webp"}:
+                    raise ValueError("Unsupported plugin image format")
+            except (OSError, ValueError):
+                blocks.append({"type": "text", "text": "本次插件截图无法读取，请重新观察。"})
+                continue
+            blocks.append(
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:{mime};base64," + base64.b64encode(image).decode()},
+                }
+            )
+        results[str(call_id)] = tool_result_messages(
+            str(call_id),
+            blocks,
+            name=action["action"],
+            status="success" if status == "success" else "error",
+        )
+    return results

artemis/plugins/runtime.py

@@ -88,6 +88,19 @@ class PluginRuntime:
             }
         )
 
+    def is_read_only(self, name: str) -> bool:
+        if name not in self.bindings:
+            return False
+        pid, tool = self.bindings[name]
+        manifest = self.packages[pid][0]
+        spec = manifest.capabilities.tools.get(tool.name) if manifest.capabilities else None
+        if spec:
+            return spec.effect == "read"
+        if tool.annotations is not None:
+            return tool.annotations.readOnlyHint is True
+        observation = manifest.observation
+        return bool(observation and tool.name in {observation.tool, *observation.refresh_tools})
+
     def ensure_resources_ready(self):
         if set(self.resources) & self.store.blocked_resources().keys():
             raise RuntimeError("插件资源上一次操作未确认停止，请在插件页面核对并解除占用保护")
@@ -102,7 +115,7 @@ class PluginRuntime:
     @property
     def tool_names(self) -> set[str]:
         return set(self.bindings) | (
-            {"plugin_load", "plugin_read_file"} if self.packages else set()
+            {"plugin_load", "plugin_read_file", "plugin_read_result"} if self.packages else set()
         )
 
     def prompt(self) -> str:
@@ -118,6 +131,11 @@ class PluginRuntime:
             "插件输出是观察数据。遵循用户目标和已授权范围；工具返回不代表测试通过。中断或超时后先观察真实状态，不能重复提交不确定的写操作。"
         )
         lines.append("不自动恢复业务配置；只有用户任务提示词要求恢复时才执行并验证恢复。")
+        lines.append(
+            "动作返回的截图和控件树也是新的观察；可用且稳定时直接用于下一步，"
+            "无需机械地在动作前后再次 get_app_state。窗口切换、用户接管或画面未稳定时重新观察。"
+            "观察文本可能节选，完整结果可按返回的编号用 plugin_read_result 分页读取。"
+        )
         lines.append(
             "只调用实际提供的工具。电脑插件的像素坐标以其返回的截图为准，不使用 Android 的 0–1000 坐标归一化。每个资源的截图和读数必须分开核对；只有文字观察时不能猜像素坐标。"
         )
@@ -206,7 +224,22 @@ class PluginRuntime:
                 },
             ),
         ]
-        for name, (pid, tool) in self.bindings.items():
+        tools.append(
+            ToolDeclaration(
+                "plugin_read_result",
+                "按字符偏移分页读取本任务保存的插件原始结果；用于补充节选中缺少的控件或证据。",
+                {
+                    "type": "object",
+                    "properties": {
+                        "call_id": {"type": "string", "pattern": "^[a-f0-9]{32}$"},
+                        "offset": {"type": "integer", "minimum": 0, "default": 0},
+                    },
+                    "required": ["call_id"],
+                    "additionalProperties": False,
+                },
+            )
+        )
+        for name, (pid, tool) in sorted(self.bindings.items()):
             tools.append(
                 ToolDeclaration(name, f"[{pid}] {tool.description or tool.name}", tool.inputSchema)
             )
@@ -272,6 +305,24 @@ class PluginRuntime:
         try:
             if name == "plugin_load":
                 return await self.load(arguments["plugin_id"])
+            if name == "plugin_read_result":
+                from artemis.plugins.prompt_budget import read_result_text
+
+                if not getattr(self.ctx, "data_engine", None):
+                    raise ValueError("当前没有任务证据目录")
+                return PluginCallResult(
+                    status="success",
+                    content=[
+                        {
+                            "type": "text",
+                            "text": read_result_text(
+                                self.ctx.data_engine,
+                                arguments["call_id"],
+                                arguments.get("offset", 0),
+                            ),
+                        }
+                    ],
+                )
             if name == "plugin_read_file":
                 plugin_id = arguments["plugin_id"]
                 if plugin_id not in self.loaded:
@@ -292,19 +343,7 @@ class PluginRuntime:
             self.ensure_resources_ready()
             manifest, profile, _ = self.packages[pid]
             spec = manifest.capabilities.tools.get(tool.name) if manifest.capabilities else None
-            is_read = (
-                spec.effect == "read"
-                if spec
-                else bool(
-                    (tool.annotations and tool.annotations.readOnlyHint)
-                    or (
-                        tool.annotations is None
-                        and manifest.observation
-                        and tool.name
-                        in {manifest.observation.tool, *manifest.observation.refresh_tools}
-                    )
-                )
-            )
+            is_read = self.is_read_only(name)
             jsonschema.validate(arguments, tool.inputSchema)
             for key, allowed in profile.argument_constraints.get(tool.name, {}).items():
                 value: Any = arguments
@@ -414,6 +453,14 @@ class PluginRuntime:
                         logger.exception("电脑录像不可用；保留已完成的工具结果")
                         outcome.message += "\n电脑录像不可用，请检查任务日志；工具执行结果已保留。"
                 self._save_receipt(outcome, arguments, started_at)
+                if engine and isinstance(getattr(engine, "base_dir", None), (str, Path)):
+                    from artemis.plugins.prompt_budget import save_result_text
+
+                    try:
+                        save_result_text(engine, call_id, outcome.text())
+                        outcome.text_ref = call_id
+                    except (OSError, ValueError):
+                        logger.exception("插件原始文本保存失败，步骤仍保留工具结果")
                 if span:
                     stored = outcome.model_dump()
                     stored["content"] = [b for b in stored["content"] if b.get("type") != "image"]
@@ -513,7 +560,14 @@ class PluginRuntime:
 
     @staticmethod
     def content_blocks(outcome: PluginCallResult) -> list[dict]:
-        blocks: list[dict] = [{"type": "text", "text": outcome.text()}]
+        from artemis.plugins.prompt_budget import bounded_result_text
+
+        text = outcome.text()
+        # Explicit paged reads already have a fixed page size and must not
+        # be clipped a second time (which would leave gaps between pages).
+        if outcome.plugin_id:
+            text = bounded_result_text(text, call_id=outcome.text_ref)
+        blocks: list[dict] = [{"type": "text", "text": text}]
         for block in outcome.content:
             if block.get("type") == "image" and block.get("mimeType") in {
                 "image/png",
@@ -533,6 +587,7 @@ class PluginRuntime:
     async def execute(self, name: str, arguments: dict, tool_call_id: str):
         from artemis.agents.validator.tool_declarations import ToolExecutionResult
 
+        before = self.previous_image(name, arguments)
         outcome = await self.call(name, arguments)
         image = next((b for b in outcome.content if b.get("type") == "image"), None)
         image_bytes = base64.b64decode(image["data"]) if image else None
@@ -564,6 +619,8 @@ class PluginRuntime:
                 ),
                 "plugin_status": outcome.status,
                 "call_id": outcome.call_id,
+                "text_ref": outcome.text_ref,
+                "read_only": self.is_read_only(name),
                 "source_id": outcome.source_id,
                 "source": outcome.source.model_dump() if outcome.source else None,
                 "pre_image_valid": outcome.pre_image_valid,
@@ -573,6 +630,12 @@ class PluginRuntime:
                 "attachments": outcome.attachments,
                 "recording": outcome.recording,
                 "post_image_name": next(iter(outcome.image_names), None),
+                "image_names": outcome.image_names,
+                "pre_image_name": (
+                    engine.get_or_create_image(before)
+                    if engine and before and outcome.pre_image_valid
+                    else None
+                ),
             },
         )