57 lines
2.3 KiB
Python
57 lines
2.3 KiB
Python
"""Deliver persisted plugin results without substituting the Android observation."""
|
|
|
|
import base64
|
|
import io
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from langchain_core.messages import BaseMessage
|
|
from PIL import Image
|
|
|
|
from artemis.tools.tool_wrapper import tool_result_messages
|
|
from artemis.plugins.prompt_budget import bounded_result_text
|
|
|
|
|
|
def plugin_result_messages(report: Any, engine: Any) -> dict[str, list[BaseMessage]]:
|
|
if not isinstance(report, dict):
|
|
return {}
|
|
results = {}
|
|
for action in report.get("execution", []):
|
|
if not isinstance(action, dict) or not str(action.get("action", "")).startswith("plug_"):
|
|
continue
|
|
call_id = action.get("tool_call_id")
|
|
if not call_id:
|
|
continue
|
|
status = action.get("plugin_status")
|
|
text = action.get("result") or "此插件操作未完成或未执行；请重新观察后再决定下一步。"
|
|
text = bounded_result_text(str(text), call_id=str(action.get("text_ref") or ""))
|
|
blocks = [{"type": "text", "text": text}]
|
|
names = action.get("image_names") or [action.get("post_image_name")]
|
|
for name in dict.fromkeys(names):
|
|
if not name or engine is None:
|
|
continue
|
|
try:
|
|
image = Path(engine.get_image_path(name)).read_bytes()
|
|
# DataEngine keeps the original bytes under a .jpg filename;
|
|
# computer plugins may actually return PNG or WebP.
|
|
with Image.open(io.BytesIO(image)) as captured:
|
|
mime = Image.MIME.get(captured.format)
|
|
if mime not in {"image/jpeg", "image/png", "image/webp"}:
|
|
raise ValueError("Unsupported plugin image format")
|
|
except (OSError, ValueError):
|
|
blocks.append({"type": "text", "text": "本次插件截图无法读取，请重新观察。"})
|
|
continue
|
|
blocks.append(
|
|
{
|
|
"type": "image_url",
|
|
"image_url": {"url": f"data:{mime};base64," + base64.b64encode(image).decode()},
|
|
}
|
|
)
|
|
results[str(call_id)] = tool_result_messages(
|
|
str(call_id),
|
|
blocks,
|
|
name=action["action"],
|
|
status="success" if status == "success" else "error",
|
|
)
|
|
return results
|