397 lines
14 KiB
Python
397 lines
14 KiB
Python
"""Plugin contracts and evidence persistence, without devices or external services."""
|
|
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, Mock
|
|
from uuid import uuid4
|
|
|
|
from mcp.types import CallToolResult, ImageContent, TextContent, Tool, ToolAnnotations
|
|
import pytest
|
|
|
|
from artemis.plugins.evidence import (
|
|
decode_receipt,
|
|
persist_attachments,
|
|
persist_receipt,
|
|
source_identity,
|
|
validate_tools,
|
|
)
|
|
from artemis.plugins.models import EvidenceSource, PluginCallResult, PluginManifest
|
|
from artemis.plugins.runtime import PluginRuntime
|
|
|
|
|
|
def manifest(**updates):
|
|
return PluginManifest.model_validate(
|
|
{
|
|
"schema_version": 2,
|
|
"id": "visual-test",
|
|
"name": "Visual fixture",
|
|
"version": "1.0.0",
|
|
"skill": "SKILL.md",
|
|
"observation": {"tool": "observe", "refresh_tools": ["observe"]},
|
|
"capabilities": {
|
|
"kind": "visual",
|
|
"recording": "windows",
|
|
"tools": {
|
|
"observe": {
|
|
"effect": "read",
|
|
"observation": True,
|
|
"target_argument": "app",
|
|
}
|
|
},
|
|
"required_tools": ["observe"],
|
|
},
|
|
**updates,
|
|
}
|
|
)
|
|
|
|
|
|
def observation_tool(*, readonly=True, properties=None):
|
|
return Tool(
|
|
name="observe",
|
|
inputSchema={
|
|
"type": "object",
|
|
"properties": {"app": {}} if properties is None else properties,
|
|
},
|
|
annotations=ToolAnnotations(readOnlyHint=readonly),
|
|
)
|
|
|
|
|
|
def attachment(data=b"verified evidence", name="capture.png"):
|
|
return {
|
|
"name": name,
|
|
"data": base64.b64encode(data).decode(),
|
|
"sha256": hashlib.sha256(data).hexdigest(),
|
|
"media_type": "image/png",
|
|
}
|
|
|
|
|
|
def test_schema_one_stays_compatible_but_schema_two_requires_contract():
|
|
legacy = manifest(schema_version=1, capabilities=None)
|
|
validate_tools(legacy, [observation_tool(readonly=False)])
|
|
with pytest.raises(ValueError, match="capabilities"):
|
|
manifest(capabilities=None)
|
|
with pytest.raises(ValueError, match="schema_version=2"):
|
|
manifest(schema_version=1)
|
|
|
|
|
|
@pytest.mark.parametrize("effect,observation", [("write", True), ("read", False)])
|
|
def test_auto_observation_requires_readonly_contract(effect, observation):
|
|
with pytest.raises(ValueError, match="只读"):
|
|
manifest(
|
|
capabilities={
|
|
"kind": "visual",
|
|
"tools": {"observe": {"effect": effect, "observation": observation}},
|
|
}
|
|
)
|
|
|
|
|
|
def test_nonvisual_plugin_cannot_request_screen_recording():
|
|
with pytest.raises(ValueError, match="视觉插件"):
|
|
manifest(observation=None, capabilities={"kind": "api", "recording": "windows"})
|
|
|
|
|
|
def test_manifest_is_checked_against_actual_mcp_tools():
|
|
validate_tools(manifest(), [observation_tool()])
|
|
with pytest.raises(ValueError, match="只读声明"):
|
|
validate_tools(manifest(), [observation_tool(readonly=False)])
|
|
with pytest.raises(ValueError, match="缺少目标参数"):
|
|
validate_tools(manifest(), [observation_tool(properties={})])
|
|
with pytest.raises(ValueError, match="缺少：observe"):
|
|
validate_tools(manifest(), [])
|
|
with pytest.raises(ValueError, match="未声明：click"):
|
|
validate_tools(manifest(), [observation_tool(), Tool(name="click", inputSchema={})])
|
|
|
|
|
|
def test_source_identity_invalidates_geometry_but_not_title_or_capture_time():
|
|
source = EvidenceSource(
|
|
kind="computer",
|
|
target_id="123",
|
|
label="before",
|
|
captured_at=1,
|
|
width=800,
|
|
height=600,
|
|
window={"left": 0, "top": 0, "width": 800, "height": 600},
|
|
)
|
|
identity = source_identity("visual-test", source)
|
|
assert identity == source_identity(
|
|
"visual-test", source.model_copy(update={"label": "after", "captured_at": 2})
|
|
)
|
|
for change in (
|
|
{"target_id": "456"},
|
|
{"width": 400},
|
|
{"window": {"left": 100, "top": 0, "width": 800, "height": 600}},
|
|
):
|
|
assert identity != source_identity("visual-test", source.model_copy(update=change))
|
|
assert identity != source_identity("another-plugin", source)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"execution,retryable,observe",
|
|
[
|
|
("unknown", False, True),
|
|
("partial", False, True),
|
|
("completed", False, False),
|
|
("not_started", True, False),
|
|
],
|
|
)
|
|
def test_uncertain_writes_cannot_claim_to_be_retryable(execution, retryable, observe):
|
|
result = PluginCallResult(
|
|
status="error",
|
|
structured_content={
|
|
"artemis": {
|
|
"version": 1,
|
|
"execution": execution,
|
|
"side_effect": "possible",
|
|
"retryable": True,
|
|
"error_code": "remote_failure",
|
|
}
|
|
},
|
|
)
|
|
decode_receipt(result)
|
|
assert result.execution == execution
|
|
assert result.retryable is retryable
|
|
assert result.requires_observation is observe
|
|
assert result.error_code == "remote_failure"
|
|
|
|
|
|
@pytest.mark.parametrize("raw", ["invalid", {"version": 99}])
|
|
def test_rejects_unsupported_receipt_shapes(raw):
|
|
with pytest.raises(ValueError):
|
|
decode_receipt(PluginCallResult(status="success", structured_content={"artemis": raw}))
|
|
|
|
|
|
def test_attachments_are_read_back_and_never_overwrite_existing_evidence(tmp_path):
|
|
receipt = attachment()
|
|
saved = persist_attachments(tmp_path, "visual-test", [receipt])[0]
|
|
path = tmp_path / saved["path"]
|
|
assert path.read_bytes() == b"verified evidence"
|
|
assert saved["sha256"] == receipt["sha256"]
|
|
assert saved["bytes"] == len(b"verified evidence")
|
|
assert (
|
|
persist_attachments(
|
|
tmp_path, "visual-test", [{k: v for k, v in receipt.items() if k != "data"}]
|
|
)[0]
|
|
== saved
|
|
)
|
|
with pytest.raises(ValueError, match="不能覆盖"):
|
|
persist_attachments(tmp_path, "visual-test", [attachment(b"different")])
|
|
assert path.read_bytes() == b"verified evidence"
|
|
|
|
|
|
def test_invalid_hash_does_not_publish_attachment(tmp_path):
|
|
receipt = attachment()
|
|
receipt["sha256"] = "0" * 64
|
|
with pytest.raises(ValueError, match="校验失败"):
|
|
persist_attachments(tmp_path, "visual-test", [receipt])
|
|
assert not (tmp_path / "artifacts/plugins/visual-test/capture.png").exists()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"name",
|
|
[
|
|
"../outside.png",
|
|
"..\\outside.png",
|
|
"C:\\outside.png",
|
|
"file.png:stream",
|
|
"CON.png",
|
|
"LPT1",
|
|
"file.",
|
|
"file ",
|
|
"bad\nname",
|
|
],
|
|
)
|
|
def test_attachment_names_cannot_escape_or_alias_windows_files(tmp_path, name):
|
|
with pytest.raises(ValueError, match="普通文件名"):
|
|
persist_attachments(tmp_path, "visual-test", [attachment(name=name)])
|
|
|
|
|
|
def test_receipt_index_has_references_instead_of_binary_content(tmp_path):
|
|
result = PluginCallResult(
|
|
status="success",
|
|
call_id="call-1",
|
|
image_names=["image-1"],
|
|
content=[{"type": "image", "data": "large-binary"}],
|
|
structured_content={"artemis": {"large": "payload"}},
|
|
)
|
|
persist_receipt(
|
|
SimpleNamespace(data_engine=SimpleNamespace(base_dir=tmp_path)),
|
|
result,
|
|
{"app": "fixture"},
|
|
123,
|
|
)
|
|
receipt = json.loads((tmp_path / "evidence/calls.jsonl").read_text(encoding="utf-8"))
|
|
assert receipt["call_id"] == "call-1"
|
|
assert receipt["image_names"] == ["image-1"]
|
|
assert receipt["started_at"] == 123
|
|
assert "content" not in receipt
|
|
assert "structured_content" not in receipt
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_receipt_disk_failure_preserves_successful_write_result(monkeypatch):
|
|
runtime = PluginRuntime.__new__(PluginRuntime)
|
|
runtime.ctx = None
|
|
runtime.bindings = {
|
|
"write": (
|
|
"visual-test",
|
|
SimpleNamespace(name="write", inputSchema={"type": "object"}, annotations=None),
|
|
)
|
|
}
|
|
runtime.packages = {
|
|
"visual-test": (
|
|
SimpleNamespace(capabilities=None, observation=None, version="1.0.0"),
|
|
SimpleNamespace(argument_constraints={}, secrets={}),
|
|
None,
|
|
)
|
|
}
|
|
session = SimpleNamespace(
|
|
call=AsyncMock(
|
|
return_value=CallToolResult(
|
|
content=[TextContent(type="text", text="changed")], isError=False
|
|
)
|
|
),
|
|
stop_uncertain=False,
|
|
)
|
|
runtime.sessions = {"visual-test": session}
|
|
runtime.ensure_resources_ready = lambda: None
|
|
runtime._image_targets = {}
|
|
runtime.last_images = {}
|
|
runtime._observation_calls = {}
|
|
|
|
def unavailable(*_args, **_kwargs):
|
|
raise OSError("disk full")
|
|
|
|
monkeypatch.setattr("artemis.plugins.runtime.persist_receipt", unavailable)
|
|
outcome = await runtime.call("write", {})
|
|
session.call.assert_awaited_once()
|
|
assert outcome.status == "success"
|
|
assert outcome.execution == "completed"
|
|
assert outcome.side_effect != "none"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_v2_observation_without_task_context_never_starts_recording(monkeypatch):
|
|
runtime = PluginRuntime.__new__(PluginRuntime)
|
|
runtime.ctx = None
|
|
runtime.bindings = {"observe": ("visual-test", observation_tool())}
|
|
runtime.packages = {
|
|
"visual-test": (
|
|
manifest(),
|
|
SimpleNamespace(argument_constraints={}, secrets={}),
|
|
None,
|
|
)
|
|
}
|
|
runtime.sessions = {
|
|
"visual-test": SimpleNamespace(
|
|
call=AsyncMock(
|
|
return_value=CallToolResult(
|
|
content=[ImageContent(type="image", mimeType="image/png", data="aW1hZ2U=")],
|
|
structuredContent={
|
|
"artemis": {
|
|
"version": 1,
|
|
"execution": "completed",
|
|
"source": {"kind": "computer", "target_id": "123"},
|
|
}
|
|
},
|
|
)
|
|
),
|
|
stop_uncertain=False,
|
|
)
|
|
}
|
|
runtime.ensure_resources_ready = lambda: None
|
|
runtime._image_targets = {}
|
|
runtime.last_images = {}
|
|
runtime._observation_calls = {}
|
|
runtime._recordings = None
|
|
|
|
def unexpected_recorder(*_args):
|
|
pytest.fail("Recording requires an owning task trace")
|
|
|
|
monkeypatch.setattr("artemis.plugins.recording.PluginRecordings", unexpected_recorder)
|
|
outcome = await runtime.call("observe", {"app": "fixture"})
|
|
assert outcome.status == "success"
|
|
assert outcome.source.target_id == "123"
|
|
assert outcome.source_id.startswith("plugin:visual-test:")
|
|
assert runtime.last_images["visual-test"] == b"image"
|
|
assert runtime._recordings is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"target,pre_valid,cleared",
|
|
[
|
|
("computer", False, True),
|
|
("computer", True, False),
|
|
("android", False, True),
|
|
("mixed", False, True),
|
|
],
|
|
)
|
|
async def test_pro_cross_window_evidence_does_not_reuse_another_source_before_image(
|
|
tmp_path, monkeypatch, target, pre_valid, cleared
|
|
):
|
|
from artemis.agents.validator import execution_loop
|
|
from artemis.data_engine.engine import DataEngine
|
|
from artemis.data_engine.models import ImageRecord, SessionMetadata, StepRecord
|
|
from artemis.data_engine.storage import StorageManager
|
|
from apps.admin_console.services import report_export
|
|
|
|
storage = StorageManager(tmp_path / "evidence.db", tmp_path)
|
|
session_id, step_id = uuid4(), uuid4()
|
|
storage.create_session(SessionMetadata(session_id=session_id, initial_goal="fixture"))
|
|
storage.create_image(ImageRecord(image_name="window-a"))
|
|
storage.create_image(ImageRecord(image_name="window-b"))
|
|
storage.create_step(
|
|
StepRecord(step_id=step_id, session_id=session_id, step_number=1, pre_image_name="window-a")
|
|
)
|
|
engine = DataEngine.__new__(DataEngine)
|
|
engine.storage = storage
|
|
engine._run_in_background = lambda fn, *args: fn(*args)
|
|
engine.get_step_number = lambda _step_id: 1
|
|
engine._get_generic_tools_for_sse = lambda _step_id: []
|
|
engine._publish = Mock()
|
|
engine.record_trace = Mock()
|
|
engine.get_relative_time = lambda _timestamp: "0s"
|
|
ctx = SimpleNamespace(execution_target=target, data_engine=engine)
|
|
node = SimpleNamespace(
|
|
ctx=ctx,
|
|
_parse_decisions=lambda _: ([{"action": "plugin-click"}], None),
|
|
_get_mcp_session=AsyncMock(),
|
|
_get_initial_screenshot=AsyncMock(return_value=("old-screen", "window-a")),
|
|
)
|
|
state = SimpleNamespace(
|
|
structured_decisions=None, current_step_id=str(step_id), open_incident=None
|
|
)
|
|
outcome = execution_loop._ActionOutcome(
|
|
action_item={
|
|
"action": "plugin-click",
|
|
"plugin_id": "visual-test",
|
|
"source_id": "window-b",
|
|
"post_image_name": "window-b",
|
|
"pre_image_valid": pre_valid,
|
|
},
|
|
success=True,
|
|
post_screenshot_b64="new-screen",
|
|
)
|
|
monkeypatch.setattr(execution_loop, "_process_action", AsyncMock(return_value=outcome))
|
|
monkeypatch.setattr(execution_loop, "_absorb_post_screenshot", lambda *_args: "window-b")
|
|
await execution_loop.run_validation_loop(node, state)
|
|
step = storage.get_steps(session_id)[0].model_dump()
|
|
assert (step["pre_image_name"] is None) is cleared
|
|
if cleared:
|
|
assert step["post_image_name"] == "window-b"
|
|
assert engine._publish.call_args.args[1]["pre_image_name"] is None
|
|
before = tmp_path / "window-a.png"
|
|
after = tmp_path / "window-b.png"
|
|
before.write_bytes(b"a")
|
|
after.write_bytes(b"b")
|
|
monkeypatch.setattr(
|
|
report_export,
|
|
"image_path",
|
|
lambda name: {"window-a": before, "window-b": after}.get(name),
|
|
)
|
|
exported = report_export.session_screenshots({"steps": [step]})
|
|
assert [path for path, _name in exported] == [after]
|
|
assert all("操作前" not in name for _path, name in exported)
|