675 lines
31 KiB
Python
675 lines
31 KiB
Python
"""Per-task plugin loading and shared Flash/Pro tool dispatch."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
from pathlib import Path
|
|
import re
|
|
import time
|
|
import uuid
|
|
from typing import Any
|
|
|
|
import jsonschema
|
|
from langchain_core.messages import ToolMessage
|
|
from langchain_core.tools import StructuredTool
|
|
|
|
from artemis.config.settings import settings
|
|
from artemis.core.tool_declaration import ToolDeclaration
|
|
from artemis.plugins.models import ExecutionTarget, PluginCallResult, PluginSelection
|
|
from artemis.plugins.evidence import (
|
|
decode_receipt,
|
|
persist_attachments,
|
|
persist_receipt,
|
|
source_identity,
|
|
validate_tools,
|
|
)
|
|
from artemis.plugins.session import PluginSession, resolve_server
|
|
from artemis.plugins.store import PluginStore, contained_path, package_resources
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def get_plugin_runtime(ctx: Any) -> PluginRuntime | None:
|
|
runtime = getattr(ctx, "plugins", None)
|
|
return runtime if isinstance(runtime, PluginRuntime) else None
|
|
|
|
|
|
class PluginRuntime:
|
|
def __init__(
|
|
self,
|
|
selections: list[PluginSelection] | None = None,
|
|
*,
|
|
store: PluginStore | None = None,
|
|
ctx: Any = None,
|
|
target: ExecutionTarget | None = None,
|
|
):
|
|
self.store = store or PluginStore()
|
|
self.ctx = ctx
|
|
self.packages = {}
|
|
target = target or getattr(ctx, "execution_target", "mixed")
|
|
self._lease_id = f"plugin-runtime-{uuid.uuid4().hex}"
|
|
with self.store.mutation_lock:
|
|
pinned = self.store.pin(selections, target)
|
|
for selection in pinned:
|
|
self.packages[selection.id] = self.store.resolve(selection)
|
|
self.store.hold(self._lease_id, pinned)
|
|
self.sessions: dict[str, PluginSession] = {}
|
|
self.loaded: set[str] = set()
|
|
self.bindings: dict[str, tuple[str, Any]] = {}
|
|
self._load_lock = asyncio.Lock()
|
|
self._observation_calls: dict[str, tuple[str, dict]] = {}
|
|
self.last_images: dict[str, bytes] = {}
|
|
self._image_targets: dict[str, tuple[str, str]] = {}
|
|
self._recordings = None
|
|
self._capture_suspended = False
|
|
|
|
def previous_image(self, name: str, arguments: dict) -> bytes | None:
|
|
if name not in self.bindings:
|
|
return None
|
|
pid, tool = self.bindings[name]
|
|
manifest = self.packages[pid][0]
|
|
spec = manifest.capabilities.tools.get(tool.name) if manifest.capabilities else None
|
|
target = spec.target_argument if spec else "app"
|
|
key = str(arguments.get(target) or "")
|
|
cached = self._image_targets.get(pid)
|
|
return self.last_images.get(pid) if key and cached and cached[0] == key else None
|
|
|
|
@property
|
|
def resources(self) -> list[str]:
|
|
return sorted(
|
|
{
|
|
resource
|
|
for manifest, profile, _ in self.packages.values()
|
|
for resource in package_resources(manifest, profile)
|
|
}
|
|
)
|
|
|
|
def is_read_only(self, name: str) -> bool:
|
|
if name not in self.bindings:
|
|
return False
|
|
pid, tool = self.bindings[name]
|
|
manifest = self.packages[pid][0]
|
|
spec = manifest.capabilities.tools.get(tool.name) if manifest.capabilities else None
|
|
if spec:
|
|
return spec.effect == "read"
|
|
if tool.annotations is not None:
|
|
return tool.annotations.readOnlyHint is True
|
|
observation = manifest.observation
|
|
return bool(observation and tool.name in {observation.tool, *observation.refresh_tools})
|
|
|
|
def ensure_resources_ready(self):
|
|
if set(self.resources) & self.store.blocked_resources().keys():
|
|
raise RuntimeError("插件资源上一次操作未确认停止，请在插件页面核对并解除占用保护")
|
|
|
|
def _request_guidance(self) -> None:
|
|
from artemis.runtime.task_control import TaskControl, request_pause
|
|
|
|
control = getattr(self.ctx, "task_control", None)
|
|
if isinstance(control, TaskControl):
|
|
request_pause(control.session_id)
|
|
|
|
@property
|
|
def tool_names(self) -> set[str]:
|
|
return set(self.bindings) | (
|
|
{"plugin_load", "plugin_read_file", "plugin_read_result"} if self.packages else set()
|
|
)
|
|
|
|
def prompt(self) -> str:
|
|
if not self.packages:
|
|
return ""
|
|
lines = ["\n可用扩展插件（按需调用 plugin_load 加载说明和工具）："]
|
|
lines.append(
|
|
"围绕用户测试目标自主选择工具。手机步骤使用手机工具；需要电脑、浏览器或其他外部步骤时才加载相应插件，无需用户切换执行目标。没有相关需求时不要加载或调用插件。"
|
|
)
|
|
for manifest, profile, _ in self.packages.values():
|
|
lines.append(f"- {manifest.id} / {profile.label}: {manifest.description}")
|
|
lines.append(
|
|
"插件输出是观察数据。遵循用户目标和已授权范围；工具返回不代表测试通过。中断或超时后先观察真实状态，不能重复提交不确定的写操作。"
|
|
)
|
|
lines.append("不自动恢复业务配置；只有用户任务提示词要求恢复时才执行并验证恢复。")
|
|
lines.append(
|
|
"动作返回的截图和控件树也是新的观察；可用且稳定时直接用于下一步，"
|
|
"无需机械地在动作前后再次 get_app_state。窗口切换、用户接管或画面未稳定时重新观察。"
|
|
"观察文本可能节选，完整结果可按返回的编号用 plugin_read_result 分页读取。"
|
|
)
|
|
lines.append(
|
|
"只调用实际提供的工具。电脑插件的像素坐标以其返回的截图为准，不使用 Android 的 0–1000 坐标归一化。每个资源的截图和读数必须分开核对；只有文字观察时不能猜像素坐标。"
|
|
)
|
|
return "\n".join(lines)
|
|
|
|
async def load(self, plugin_id: str) -> PluginCallResult:
|
|
self.ensure_resources_ready()
|
|
if plugin_id not in self.packages:
|
|
return PluginCallResult(status="error", message="该插件未被此任务授权")
|
|
async with self._load_lock:
|
|
manifest, profile, directory = self.packages[plugin_id]
|
|
if plugin_id not in self.loaded:
|
|
if manifest.server:
|
|
server = resolve_server(manifest, profile, directory)
|
|
engine = getattr(self.ctx, "data_engine", None)
|
|
if (
|
|
server.transport == "stdio"
|
|
and engine
|
|
and isinstance(engine.base_dir, (str, Path))
|
|
):
|
|
artifacts = Path(engine.base_dir) / "artifacts" / "plugins" / plugin_id
|
|
artifacts.mkdir(parents=True, exist_ok=True)
|
|
server.env["ARTEMIS_ARTIFACTS_DIR"] = str(artifacts.resolve())
|
|
session = await PluginSession(server, directory).start()
|
|
try:
|
|
validate_tools(manifest, session.tools)
|
|
except ValueError:
|
|
await session.aclose()
|
|
raise
|
|
self.sessions[plugin_id] = session
|
|
for tool in session.tools:
|
|
if (
|
|
profile.allowed_tools is not None
|
|
and tool.name not in profile.allowed_tools
|
|
):
|
|
continue
|
|
digest = hashlib.sha256(f"{plugin_id}:{tool.name}".encode()).hexdigest()[
|
|
:10
|
|
]
|
|
label = re.sub(r"[^a-zA-Z0-9_-]", "_", tool.name)[:35]
|
|
self.bindings[f"plug_{digest}_{label}"] = (plugin_id, tool)
|
|
self.loaded.add(plugin_id)
|
|
skill = ""
|
|
if manifest.skill:
|
|
skill = contained_path(directory, manifest.skill).read_text(encoding="utf-8")
|
|
tools = {
|
|
name: tool.name for name, (pid, tool) in self.bindings.items() if pid == plugin_id
|
|
}
|
|
return PluginCallResult(
|
|
status="success",
|
|
plugin_id=plugin_id,
|
|
version=manifest.version,
|
|
content=[{"type": "text", "text": f"插件说明（仅适用于此插件）：\n{skill}"}],
|
|
structured_content={
|
|
"tools": tools,
|
|
"resources": manifest.resources + profile.resources,
|
|
"argument_constraints": profile.argument_constraints,
|
|
},
|
|
)
|
|
|
|
def declarations(self) -> list[ToolDeclaration]:
|
|
if not self.packages:
|
|
return []
|
|
tools = [
|
|
ToolDeclaration(
|
|
"plugin_load",
|
|
"按需加载已授权插件的 Skill、操作工具及控制范围。",
|
|
{
|
|
"type": "object",
|
|
"properties": {"plugin_id": {"type": "string", "enum": list(self.packages)}},
|
|
"required": ["plugin_id"],
|
|
"additionalProperties": False,
|
|
},
|
|
),
|
|
ToolDeclaration(
|
|
"plugin_read_file",
|
|
"读取已加载插件 Skill 引用的包内文档；不可读取包外文件。",
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"plugin_id": {"type": "string", "enum": list(self.packages)},
|
|
"path": {"type": "string"},
|
|
},
|
|
"required": ["plugin_id", "path"],
|
|
"additionalProperties": False,
|
|
},
|
|
),
|
|
]
|
|
tools.append(
|
|
ToolDeclaration(
|
|
"plugin_read_result",
|
|
"按字符偏移分页读取本任务保存的插件原始结果；用于补充节选中缺少的控件或证据。",
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"call_id": {"type": "string", "pattern": "^[a-f0-9]{32}$"},
|
|
"offset": {"type": "integer", "minimum": 0, "default": 0},
|
|
},
|
|
"required": ["call_id"],
|
|
"additionalProperties": False,
|
|
},
|
|
)
|
|
)
|
|
for name, (pid, tool) in sorted(self.bindings.items()):
|
|
tools.append(
|
|
ToolDeclaration(name, f"[{pid}] {tool.description or tool.name}", tool.inputSchema)
|
|
)
|
|
return tools
|
|
|
|
def langchain_tools(self) -> list[StructuredTool]:
|
|
result = []
|
|
for declaration in self.declarations():
|
|
|
|
def handler(name):
|
|
async def call(**arguments):
|
|
outcome = await self.call(name, arguments)
|
|
return ToolMessage(
|
|
content=self.content_blocks(outcome),
|
|
name=name,
|
|
tool_call_id="plugin",
|
|
status="success" if outcome.status == "success" else "error",
|
|
)
|
|
|
|
return call
|
|
|
|
result.append(
|
|
StructuredTool(
|
|
name=declaration.name,
|
|
description=declaration.description,
|
|
args_schema=declaration.parameters,
|
|
coroutine=handler(declaration.name),
|
|
)
|
|
)
|
|
return result
|
|
|
|
def _redact(self, value: str) -> str:
|
|
for _, profile, _ in self.packages.values():
|
|
for secret in profile.secrets.values():
|
|
raw = secret.get_secret_value()
|
|
if raw:
|
|
value = value.replace(raw, "[已隐藏密钥]")
|
|
return value
|
|
|
|
def _redact_payload(self, value: Any) -> Any:
|
|
if isinstance(value, list):
|
|
return [self._redact_payload(item) for item in value]
|
|
if isinstance(value, dict):
|
|
# Binary image/audio data are not text. Never corrupt base64 or JSON keys.
|
|
binary = value.get("type") in {"image", "audio"}
|
|
return {
|
|
key: item if binary and key == "data" else self._redact_payload(item)
|
|
for key, item in value.items()
|
|
}
|
|
return self._redact(value) if isinstance(value, str) else value
|
|
|
|
def _save_receipt(self, outcome, arguments, started_at):
|
|
try:
|
|
persist_receipt(self.ctx, outcome, self._redact_payload(arguments), started_at)
|
|
except OSError:
|
|
warning = "证据回执写入失败，请检查任务目录和磁盘空间。"
|
|
outcome.message = f"{outcome.message}\n{warning}".strip()
|
|
logger.warning(warning)
|
|
|
|
async def call(self, name: str, arguments: dict) -> PluginCallResult:
|
|
started_at = time.time()
|
|
call_id = uuid.uuid4().hex
|
|
try:
|
|
if name == "plugin_load":
|
|
return await self.load(arguments["plugin_id"])
|
|
if name == "plugin_read_result":
|
|
from artemis.plugins.prompt_budget import read_result_text
|
|
|
|
if not getattr(self.ctx, "data_engine", None):
|
|
raise ValueError("当前没有任务证据目录")
|
|
return PluginCallResult(
|
|
status="success",
|
|
content=[
|
|
{
|
|
"type": "text",
|
|
"text": read_result_text(
|
|
self.ctx.data_engine,
|
|
arguments["call_id"],
|
|
arguments.get("offset", 0),
|
|
),
|
|
}
|
|
],
|
|
)
|
|
if name == "plugin_read_file":
|
|
plugin_id = arguments["plugin_id"]
|
|
if plugin_id not in self.loaded:
|
|
raise ValueError("请先加载插件")
|
|
path = contained_path(self.packages[plugin_id][2], arguments["path"])
|
|
if (
|
|
path.suffix.lower() not in {".md", ".txt", ".json"}
|
|
or path.stat().st_size > 256 * 1024
|
|
):
|
|
raise ValueError("仅支持读取 256 KiB 以内的插件说明文件")
|
|
return PluginCallResult(
|
|
status="success",
|
|
content=[
|
|
{"type": "text", "text": self._redact(path.read_text(encoding="utf-8"))}
|
|
],
|
|
)
|
|
pid, tool = self.bindings[name]
|
|
self.ensure_resources_ready()
|
|
manifest, profile, _ = self.packages[pid]
|
|
spec = manifest.capabilities.tools.get(tool.name) if manifest.capabilities else None
|
|
is_read = self.is_read_only(name)
|
|
jsonschema.validate(arguments, tool.inputSchema)
|
|
for key, allowed in profile.argument_constraints.get(tool.name, {}).items():
|
|
value: Any = arguments
|
|
for part in key.split("."):
|
|
value = value.get(part) if isinstance(value, dict) else None
|
|
if not isinstance(value, (str, int, float, bool)) or str(value) not in allowed:
|
|
raise PermissionError(f"工具参数 {key} 超出授权范围，请用户补充指导或修改授权")
|
|
span = None
|
|
if self.ctx:
|
|
from artemis.data_engine.trace import TraceSpan
|
|
|
|
span = TraceSpan(name=name, trace_type="tool", ctx=self.ctx)
|
|
if span:
|
|
span.__enter__()
|
|
span.payload = {
|
|
"call_id": call_id,
|
|
"args": self._redact_payload(arguments),
|
|
"plugin_id": pid,
|
|
"plugin_version": manifest.version,
|
|
"tool_name": tool.name,
|
|
"effect": "read" if is_read else "write",
|
|
"observation": bool(spec.observation)
|
|
if spec
|
|
else bool(is_read and manifest.observation),
|
|
"resource_ids": sorted(package_resources(manifest, profile)),
|
|
}
|
|
outcome = None
|
|
try:
|
|
raw = await self.sessions[pid].call(tool.name, arguments)
|
|
payload = raw.model_dump(mode="json", by_alias=True, exclude_none=True)
|
|
encoded = json.dumps(payload, ensure_ascii=False)
|
|
if len(encoded.encode()) > settings.ARTEMIS_PLUGIN_RESULT_MAX_BYTES:
|
|
raise RuntimeError("插件结果超过大小限制，操作可能已经完成；请重新观察")
|
|
payload = self._redact_payload(payload)
|
|
outcome = PluginCallResult(
|
|
status="error" if payload.get("isError") else "success",
|
|
content=payload.get("content", []),
|
|
structured_content=payload.get("structuredContent"),
|
|
plugin_id=pid,
|
|
version=manifest.version,
|
|
tool=tool.name,
|
|
call_id=call_id,
|
|
execution="unknown" if payload.get("isError") else "completed",
|
|
side_effect="none" if is_read else "possible",
|
|
)
|
|
receipt = decode_receipt(outcome)
|
|
outcome.source_id = source_identity(pid, outcome.source)
|
|
engine = getattr(self.ctx, "data_engine", None)
|
|
if receipt.get("attachments") and engine:
|
|
outcome.attachments = persist_attachments(
|
|
Path(engine.base_dir), pid, receipt["attachments"]
|
|
)
|
|
observation = manifest.observation
|
|
image_block = next((b for b in outcome.content if b.get("type") == "image"), None)
|
|
target_argument = spec.target_argument if spec else "app"
|
|
target = str(arguments.get(target_argument) or "")
|
|
previous = self._image_targets.get(pid)
|
|
pre_image_valid = bool(previous and previous == (target, outcome.source_id))
|
|
outcome.pre_image_valid = pre_image_valid
|
|
if image_block:
|
|
self.last_images[pid] = base64.b64decode(image_block["data"], validate=True)
|
|
self._image_targets[pid] = (target, outcome.source_id)
|
|
elif not is_read:
|
|
self.last_images.pop(pid, None)
|
|
self._image_targets.pop(pid, None)
|
|
if (
|
|
outcome.status == "success"
|
|
and observation
|
|
and tool.name in observation.refresh_tools
|
|
and is_read
|
|
):
|
|
self._observation_calls[pid] = (tool.name, dict(arguments))
|
|
self._persist_images(outcome)
|
|
if (
|
|
outcome.source
|
|
and not getattr(self, "_capture_suspended", False)
|
|
and image_block
|
|
and manifest.capabilities
|
|
and manifest.capabilities.recording == "windows"
|
|
and engine
|
|
and isinstance(getattr(engine, "base_dir", None), (str, Path))
|
|
):
|
|
from artemis.plugins.recording import (
|
|
PluginRecordings,
|
|
recover_interrupted_recordings,
|
|
)
|
|
|
|
try:
|
|
if self._recordings is None:
|
|
await recover_interrupted_recordings(Path(engine.base_dir))
|
|
self._recordings = PluginRecordings(self.ctx)
|
|
await self._recordings.observe(outcome.source_id, pid, outcome.source)
|
|
track = next(
|
|
(
|
|
value
|
|
for value in self._recordings.sources.values()
|
|
if outcome.source_id in value.get("target_ids", [])
|
|
),
|
|
None,
|
|
)
|
|
if track:
|
|
outcome.recording = {
|
|
key: track.get(key)
|
|
for key in ("id", "status", "scope", "area", "message")
|
|
}
|
|
except (OSError, ValueError, RuntimeError, TimeoutError):
|
|
logger.exception("电脑录像不可用；保留已完成的工具结果")
|
|
outcome.message += "\n电脑录像不可用，请检查任务日志；工具执行结果已保留。"
|
|
self._save_receipt(outcome, arguments, started_at)
|
|
if engine and isinstance(getattr(engine, "base_dir", None), (str, Path)):
|
|
from artemis.plugins.prompt_budget import save_result_text
|
|
|
|
try:
|
|
save_result_text(engine, call_id, outcome.text())
|
|
outcome.text_ref = call_id
|
|
except (OSError, ValueError):
|
|
logger.exception("插件原始文本保存失败，步骤仍保留工具结果")
|
|
if span:
|
|
stored = outcome.model_dump()
|
|
stored["content"] = [b for b in stored["content"] if b.get("type") != "image"]
|
|
stored["post_image_name"] = next(iter(outcome.image_names), None)
|
|
span.payload["post_image_name"] = stored["post_image_name"]
|
|
span.payload["image_names"] = outcome.image_names
|
|
span.payload["source_id"] = outcome.source_id
|
|
span.payload["source"] = outcome.source.model_dump() if outcome.source else None
|
|
span.payload["pre_image_valid"] = pre_image_valid
|
|
span.result = stored
|
|
if outcome.status != "success":
|
|
span.status, span.error = "failed", outcome.text()
|
|
return outcome
|
|
except (RuntimeError, OSError, ValueError, TimeoutError, KeyError, TypeError) as exc:
|
|
if self.sessions[pid].stop_uncertain:
|
|
self.store.block_resources(
|
|
self.resources,
|
|
"远程插件操作中断，无法确认写操作是否停止；请核对远程状态后解除保护。",
|
|
)
|
|
if span:
|
|
span.status, span.error = "failed", "插件调用失败，结果不确定"
|
|
known_outcome = outcome
|
|
outcome = PluginCallResult(
|
|
status="uncertain",
|
|
plugin_id=pid,
|
|
version=manifest.version,
|
|
tool=tool.name,
|
|
call_id=call_id,
|
|
source_id=known_outcome.source_id
|
|
if known_outcome
|
|
else source_identity(pid, None),
|
|
source=known_outcome.source if known_outcome else None,
|
|
execution=known_outcome.execution if known_outcome else "unknown",
|
|
error_code="timeout" if isinstance(exc, TimeoutError) else "plugin_call_failed",
|
|
side_effect=known_outcome.side_effect
|
|
if known_outcome
|
|
else ("none" if is_read else "possible"),
|
|
requires_observation=True,
|
|
message=self._redact(
|
|
f"插件调用中断或超时：{str(exc)[:1000] or type(exc).__name__}。结果不确定，请先观察实际状态，不能盲目重放写操作。"
|
|
),
|
|
)
|
|
self._save_receipt(outcome, arguments, started_at)
|
|
if span:
|
|
span.result = outcome.model_dump()
|
|
span.error = outcome.message
|
|
return outcome
|
|
finally:
|
|
if span:
|
|
span.__exit__(None, None, None)
|
|
except asyncio.CancelledError:
|
|
if any(s.stop_uncertain for s in self.sessions.values()):
|
|
self.store.block_resources(
|
|
self.resources, "插件操作取消后未确认停止，请核对目标状态后解除保护。"
|
|
)
|
|
raise
|
|
except PermissionError:
|
|
self._request_guidance()
|
|
return PluginCallResult(
|
|
status="error",
|
|
call_id=call_id,
|
|
error_code="permission_denied",
|
|
execution="not_started",
|
|
side_effect="none",
|
|
message="工具参数超出此任务的授权范围，已请求暂停。请提供授权范围内的指导；如需扩大范围，请结束任务后修改插件配置。",
|
|
)
|
|
except (
|
|
KeyError,
|
|
ValueError,
|
|
OSError,
|
|
jsonschema.ValidationError,
|
|
jsonschema.SchemaError,
|
|
):
|
|
# Schema errors can contain the rejected value (including secrets); keep diagnostics structural.
|
|
return PluginCallResult(
|
|
status="error",
|
|
call_id=call_id,
|
|
error_code="invalid_request",
|
|
execution="not_started",
|
|
side_effect="none",
|
|
message="插件尚未就绪、参数不符合工具定义或超出授权范围；请检查插件配置和授权。",
|
|
)
|
|
|
|
def _persist_images(self, outcome: PluginCallResult) -> None:
|
|
engine = getattr(self.ctx, "data_engine", None)
|
|
if not engine:
|
|
return
|
|
for block in outcome.content:
|
|
if block.get("type") == "image" and block.get("data"):
|
|
image = base64.b64decode(block["data"], validate=True)
|
|
# Save the text from the same tool result before a later action
|
|
# can fill this image's missing tree with a different screen.
|
|
name = engine.get_or_create_image(image, ui_tree=outcome.text())
|
|
outcome.image_names.append(name)
|
|
if engine.current_session_id:
|
|
engine.storage.reference_plugin_image(engine.current_session_id, name)
|
|
|
|
@staticmethod
|
|
def content_blocks(outcome: PluginCallResult) -> list[dict]:
|
|
from artemis.plugins.prompt_budget import bounded_result_text
|
|
|
|
text = outcome.text()
|
|
# Explicit paged reads already have a fixed page size and must not
|
|
# be clipped a second time (which would leave gaps between pages).
|
|
if outcome.plugin_id:
|
|
text = bounded_result_text(text, call_id=outcome.text_ref)
|
|
blocks: list[dict] = [{"type": "text", "text": text}]
|
|
for block in outcome.content:
|
|
if block.get("type") == "image" and block.get("mimeType") in {
|
|
"image/png",
|
|
"image/jpeg",
|
|
"image/webp",
|
|
}:
|
|
blocks.append(
|
|
{
|
|
"type": "image_url",
|
|
"image_url": {"url": f"data:{block['mimeType']};base64,{block['data']}"},
|
|
}
|
|
)
|
|
elif block.get("type") not in {"image", "text"}:
|
|
blocks.append({"type": "text", "text": json.dumps(block, ensure_ascii=False)})
|
|
return blocks
|
|
|
|
async def execute(self, name: str, arguments: dict, tool_call_id: str):
|
|
from artemis.agents.validator.tool_declarations import ToolExecutionResult
|
|
|
|
before = self.previous_image(name, arguments)
|
|
outcome = await self.call(name, arguments)
|
|
image = next((b for b in outcome.content if b.get("type") == "image"), None)
|
|
image_bytes = base64.b64decode(image["data"]) if image else None
|
|
engine = getattr(self.ctx, "data_engine", None)
|
|
path = None
|
|
if engine and image_bytes:
|
|
path = str(engine.get_image_path(engine.get_or_create_image(image_bytes)))
|
|
return ToolExecutionResult(
|
|
tool_name=name,
|
|
tool_call_id=tool_call_id,
|
|
status="success" if outcome.status == "success" else "error",
|
|
text_summary=(
|
|
f"来源：插件 {outcome.plugin_id} v{outcome.version} / {outcome.tool}\n"
|
|
if outcome.plugin_id
|
|
else ""
|
|
)
|
|
+ outcome.text(),
|
|
raw_result=self.content_blocks(outcome),
|
|
screenshot_bytes=image_bytes,
|
|
screenshot_path=path,
|
|
ui_elements_text=outcome.text(),
|
|
metadata={
|
|
"plugin_id": outcome.plugin_id,
|
|
"plugin_version": outcome.version,
|
|
"resource_ids": (
|
|
sorted(package_resources(*self.packages[outcome.plugin_id][:2]))
|
|
if outcome.plugin_id in self.packages
|
|
else []
|
|
),
|
|
"plugin_status": outcome.status,
|
|
"call_id": outcome.call_id,
|
|
"text_ref": outcome.text_ref,
|
|
"read_only": self.is_read_only(name),
|
|
"source_id": outcome.source_id,
|
|
"source": outcome.source.model_dump() if outcome.source else None,
|
|
"pre_image_valid": outcome.pre_image_valid,
|
|
"error_code": outcome.error_code,
|
|
"execution": outcome.execution,
|
|
"side_effect": outcome.side_effect,
|
|
"attachments": outcome.attachments,
|
|
"recording": outcome.recording,
|
|
"post_image_name": next(iter(outcome.image_names), None),
|
|
"image_names": outcome.image_names,
|
|
"pre_image_name": (
|
|
engine.get_or_create_image(before)
|
|
if engine and before and outcome.pre_image_valid
|
|
else None
|
|
),
|
|
},
|
|
)
|
|
|
|
async def suspend_capture(self):
|
|
self._capture_suspended = True
|
|
if self._recordings is not None:
|
|
await self._recordings.stop()
|
|
|
|
def resume_capture(self):
|
|
self._capture_suspended = False
|
|
|
|
async def aclose(self, *, release: bool = True):
|
|
if self._recordings is not None:
|
|
try:
|
|
await self._recordings.stop()
|
|
except (OSError, ValueError, RuntimeError, TimeoutError):
|
|
logger.exception("电脑录像收尾失败；继续关闭插件连接")
|
|
pending = list(self.sessions.items())
|
|
errors = await asyncio.gather(
|
|
*(session.aclose() for _, session in pending), return_exceptions=True
|
|
)
|
|
for (pid, _), error in zip(pending, errors):
|
|
if not isinstance(error, BaseException):
|
|
self.sessions.pop(pid, None)
|
|
self.bindings = {
|
|
name: binding for name, binding in self.bindings.items() if binding[0] in self.sessions
|
|
}
|
|
self.loaded.intersection_update(self.sessions)
|
|
self._observation_calls.clear()
|
|
self.last_images.clear()
|
|
self._image_targets.clear()
|
|
if any(isinstance(error, BaseException) for error in errors):
|
|
self.store.block_resources(self.resources, "插件未确认停止，请核对目标状态后解除保护。")
|
|
raise RuntimeError("部分插件未确认停止，资源仍需保持占用")
|
|
if release:
|
|
self.store.release(self._lease_id)
|