feat(plugins): add source-aware evidence and recording timelines
This commit is contained in:
chenmingxuan committed
1 parent
72ff342605
commit
ef7af9988e
61 files changed
+3847
-108
No files matched your search
@@ -0,0 +1,396 @@
|
||||
"""Plugin contracts and evidence persistence, without devices or external services."""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
from uuid import uuid4
|
||||
|
||||
from mcp.types import CallToolResult, ImageContent, TextContent, Tool, ToolAnnotations
|
||||
import pytest
|
||||
|
||||
from artemis.plugins.evidence import (
|
||||
decode_receipt,
|
||||
persist_attachments,
|
||||
persist_receipt,
|
||||
source_identity,
|
||||
validate_tools,
|
||||
)
|
||||
from artemis.plugins.models import EvidenceSource, PluginCallResult, PluginManifest
|
||||
from artemis.plugins.runtime import PluginRuntime
|
||||
|
||||
|
||||
def manifest(**updates):
|
||||
return PluginManifest.model_validate(
|
||||
{
|
||||
"schema_version": 2,
|
||||
"id": "visual-test",
|
||||
"name": "Visual fixture",
|
||||
"version": "1.0.0",
|
||||
"skill": "SKILL.md",
|
||||
"observation": {"tool": "observe", "refresh_tools": ["observe"]},
|
||||
"capabilities": {
|
||||
"kind": "visual",
|
||||
"recording": "windows",
|
||||
"tools": {
|
||||
"observe": {
|
||||
"effect": "read",
|
||||
"observation": True,
|
||||
"target_argument": "app",
|
||||
}
|
||||
},
|
||||
"required_tools": ["observe"],
|
||||
},
|
||||
**updates,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def observation_tool(*, readonly=True, properties=None):
|
||||
return Tool(
|
||||
name="observe",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {"app": {}} if properties is None else properties,
|
||||
},
|
||||
annotations=ToolAnnotations(readOnlyHint=readonly),
|
||||
)
|
||||
|
||||
|
||||
def attachment(data=b"verified evidence", name="capture.png"):
|
||||
return {
|
||||
"name": name,
|
||||
"data": base64.b64encode(data).decode(),
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
"media_type": "image/png",
|
||||
}
|
||||
|
||||
|
||||
def test_schema_one_stays_compatible_but_schema_two_requires_contract():
|
||||
legacy = manifest(schema_version=1, capabilities=None)
|
||||
validate_tools(legacy, [observation_tool(readonly=False)])
|
||||
with pytest.raises(ValueError, match="capabilities"):
|
||||
manifest(capabilities=None)
|
||||
with pytest.raises(ValueError, match="schema_version=2"):
|
||||
manifest(schema_version=1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("effect,observation", [("write", True), ("read", False)])
|
||||
def test_auto_observation_requires_readonly_contract(effect, observation):
|
||||
with pytest.raises(ValueError, match="只读"):
|
||||
manifest(
|
||||
capabilities={
|
||||
"kind": "visual",
|
||||
"tools": {"observe": {"effect": effect, "observation": observation}},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_nonvisual_plugin_cannot_request_screen_recording():
|
||||
with pytest.raises(ValueError, match="视觉插件"):
|
||||
manifest(observation=None, capabilities={"kind": "api", "recording": "windows"})
|
||||
|
||||
|
||||
def test_manifest_is_checked_against_actual_mcp_tools():
|
||||
validate_tools(manifest(), [observation_tool()])
|
||||
with pytest.raises(ValueError, match="只读声明"):
|
||||
validate_tools(manifest(), [observation_tool(readonly=False)])
|
||||
with pytest.raises(ValueError, match="缺少目标参数"):
|
||||
validate_tools(manifest(), [observation_tool(properties={})])
|
||||
with pytest.raises(ValueError, match="缺少：observe"):
|
||||
validate_tools(manifest(), [])
|
||||
with pytest.raises(ValueError, match="未声明：click"):
|
||||
validate_tools(manifest(), [observation_tool(), Tool(name="click", inputSchema={})])
|
||||
|
||||
|
||||
def test_source_identity_invalidates_geometry_but_not_title_or_capture_time():
|
||||
source = EvidenceSource(
|
||||
kind="computer",
|
||||
target_id="123",
|
||||
label="before",
|
||||
captured_at=1,
|
||||
width=800,
|
||||
height=600,
|
||||
window={"left": 0, "top": 0, "width": 800, "height": 600},
|
||||
)
|
||||
identity = source_identity("visual-test", source)
|
||||
assert identity == source_identity(
|
||||
"visual-test", source.model_copy(update={"label": "after", "captured_at": 2})
|
||||
)
|
||||
for change in (
|
||||
{"target_id": "456"},
|
||||
{"width": 400},
|
||||
{"window": {"left": 100, "top": 0, "width": 800, "height": 600}},
|
||||
):
|
||||
assert identity != source_identity("visual-test", source.model_copy(update=change))
|
||||
assert identity != source_identity("another-plugin", source)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"execution,retryable,observe",
|
||||
[
|
||||
("unknown", False, True),
|
||||
("partial", False, True),
|
||||
("completed", False, False),
|
||||
("not_started", True, False),
|
||||
],
|
||||
)
|
||||
def test_uncertain_writes_cannot_claim_to_be_retryable(execution, retryable, observe):
|
||||
result = PluginCallResult(
|
||||
status="error",
|
||||
structured_content={
|
||||
"artemis": {
|
||||
"version": 1,
|
||||
"execution": execution,
|
||||
"side_effect": "possible",
|
||||
"retryable": True,
|
||||
"error_code": "remote_failure",
|
||||
}
|
||||
},
|
||||
)
|
||||
decode_receipt(result)
|
||||
assert result.execution == execution
|
||||
assert result.retryable is retryable
|
||||
assert result.requires_observation is observe
|
||||
assert result.error_code == "remote_failure"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", ["invalid", {"version": 99}])
|
||||
def test_rejects_unsupported_receipt_shapes(raw):
|
||||
with pytest.raises(ValueError):
|
||||
decode_receipt(PluginCallResult(status="success", structured_content={"artemis": raw}))
|
||||
|
||||
|
||||
def test_attachments_are_read_back_and_never_overwrite_existing_evidence(tmp_path):
|
||||
receipt = attachment()
|
||||
saved = persist_attachments(tmp_path, "visual-test", [receipt])[0]
|
||||
path = tmp_path / saved["path"]
|
||||
assert path.read_bytes() == b"verified evidence"
|
||||
assert saved["sha256"] == receipt["sha256"]
|
||||
assert saved["bytes"] == len(b"verified evidence")
|
||||
assert (
|
||||
persist_attachments(
|
||||
tmp_path, "visual-test", [{k: v for k, v in receipt.items() if k != "data"}]
|
||||
)[0]
|
||||
== saved
|
||||
)
|
||||
with pytest.raises(ValueError, match="不能覆盖"):
|
||||
persist_attachments(tmp_path, "visual-test", [attachment(b"different")])
|
||||
assert path.read_bytes() == b"verified evidence"
|
||||
|
||||
|
||||
def test_invalid_hash_does_not_publish_attachment(tmp_path):
|
||||
receipt = attachment()
|
||||
receipt["sha256"] = "0" * 64
|
||||
with pytest.raises(ValueError, match="校验失败"):
|
||||
persist_attachments(tmp_path, "visual-test", [receipt])
|
||||
assert not (tmp_path / "artifacts/plugins/visual-test/capture.png").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"../outside.png",
|
||||
"..\\outside.png",
|
||||
"C:\\outside.png",
|
||||
"file.png:stream",
|
||||
"CON.png",
|
||||
"LPT1",
|
||||
"file.",
|
||||
"file ",
|
||||
"bad\nname",
|
||||
],
|
||||
)
|
||||
def test_attachment_names_cannot_escape_or_alias_windows_files(tmp_path, name):
|
||||
with pytest.raises(ValueError, match="普通文件名"):
|
||||
persist_attachments(tmp_path, "visual-test", [attachment(name=name)])
|
||||
|
||||
|
||||
def test_receipt_index_has_references_instead_of_binary_content(tmp_path):
|
||||
result = PluginCallResult(
|
||||
status="success",
|
||||
call_id="call-1",
|
||||
image_names=["image-1"],
|
||||
content=[{"type": "image", "data": "large-binary"}],
|
||||
structured_content={"artemis": {"large": "payload"}},
|
||||
)
|
||||
persist_receipt(
|
||||
SimpleNamespace(data_engine=SimpleNamespace(base_dir=tmp_path)),
|
||||
result,
|
||||
{"app": "fixture"},
|
||||
123,
|
||||
)
|
||||
receipt = json.loads((tmp_path / "evidence/calls.jsonl").read_text(encoding="utf-8"))
|
||||
assert receipt["call_id"] == "call-1"
|
||||
assert receipt["image_names"] == ["image-1"]
|
||||
assert receipt["started_at"] == 123
|
||||
assert "content" not in receipt
|
||||
assert "structured_content" not in receipt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_receipt_disk_failure_preserves_successful_write_result(monkeypatch):
|
||||
runtime = PluginRuntime.__new__(PluginRuntime)
|
||||
runtime.ctx = None
|
||||
runtime.bindings = {
|
||||
"write": (
|
||||
"visual-test",
|
||||
SimpleNamespace(name="write", inputSchema={"type": "object"}, annotations=None),
|
||||
)
|
||||
}
|
||||
runtime.packages = {
|
||||
"visual-test": (
|
||||
SimpleNamespace(capabilities=None, observation=None, version="1.0.0"),
|
||||
SimpleNamespace(argument_constraints={}, secrets={}),
|
||||
None,
|
||||
)
|
||||
}
|
||||
session = SimpleNamespace(
|
||||
call=AsyncMock(
|
||||
return_value=CallToolResult(
|
||||
content=[TextContent(type="text", text="changed")], isError=False
|
||||
)
|
||||
),
|
||||
stop_uncertain=False,
|
||||
)
|
||||
runtime.sessions = {"visual-test": session}
|
||||
runtime.ensure_resources_ready = lambda: None
|
||||
runtime._image_targets = {}
|
||||
runtime.last_images = {}
|
||||
runtime._observation_calls = {}
|
||||
|
||||
def unavailable(*_args, **_kwargs):
|
||||
raise OSError("disk full")
|
||||
|
||||
monkeypatch.setattr("artemis.plugins.runtime.persist_receipt", unavailable)
|
||||
outcome = await runtime.call("write", {})
|
||||
session.call.assert_awaited_once()
|
||||
assert outcome.status == "success"
|
||||
assert outcome.execution == "completed"
|
||||
assert outcome.side_effect != "none"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v2_observation_without_task_context_never_starts_recording(monkeypatch):
|
||||
runtime = PluginRuntime.__new__(PluginRuntime)
|
||||
runtime.ctx = None
|
||||
runtime.bindings = {"observe": ("visual-test", observation_tool())}
|
||||
runtime.packages = {
|
||||
"visual-test": (
|
||||
manifest(),
|
||||
SimpleNamespace(argument_constraints={}, secrets={}),
|
||||
None,
|
||||
)
|
||||
}
|
||||
runtime.sessions = {
|
||||
"visual-test": SimpleNamespace(
|
||||
call=AsyncMock(
|
||||
return_value=CallToolResult(
|
||||
content=[ImageContent(type="image", mimeType="image/png", data="aW1hZ2U=")],
|
||||
structuredContent={
|
||||
"artemis": {
|
||||
"version": 1,
|
||||
"execution": "completed",
|
||||
"source": {"kind": "computer", "target_id": "123"},
|
||||
}
|
||||
},
|
||||
)
|
||||
),
|
||||
stop_uncertain=False,
|
||||
)
|
||||
}
|
||||
runtime.ensure_resources_ready = lambda: None
|
||||
runtime._image_targets = {}
|
||||
runtime.last_images = {}
|
||||
runtime._observation_calls = {}
|
||||
runtime._recordings = None
|
||||
|
||||
def unexpected_recorder(*_args):
|
||||
pytest.fail("Recording requires an owning task trace")
|
||||
|
||||
monkeypatch.setattr("artemis.plugins.recording.PluginRecordings", unexpected_recorder)
|
||||
outcome = await runtime.call("observe", {"app": "fixture"})
|
||||
assert outcome.status == "success"
|
||||
assert outcome.source.target_id == "123"
|
||||
assert outcome.source_id.startswith("plugin:visual-test:")
|
||||
assert runtime.last_images["visual-test"] == b"image"
|
||||
assert runtime._recordings is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"target,pre_valid,cleared",
|
||||
[
|
||||
("computer", False, True),
|
||||
("computer", True, False),
|
||||
("android", False, False),
|
||||
("mixed", False, False),
|
||||
],
|
||||
)
|
||||
async def test_pro_cross_window_evidence_does_not_reuse_another_source_before_image(
|
||||
tmp_path, monkeypatch, target, pre_valid, cleared
|
||||
):
|
||||
from artemis.agents.validator import execution_loop
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
from artemis.data_engine.models import ImageRecord, SessionMetadata, StepRecord
|
||||
from artemis.data_engine.storage import StorageManager
|
||||
from apps.admin_console.services import report_export
|
||||
|
||||
storage = StorageManager(tmp_path / "evidence.db", tmp_path)
|
||||
session_id, step_id = uuid4(), uuid4()
|
||||
storage.create_session(SessionMetadata(session_id=session_id, initial_goal="fixture"))
|
||||
storage.create_image(ImageRecord(image_name="window-a"))
|
||||
storage.create_image(ImageRecord(image_name="window-b"))
|
||||
storage.create_step(
|
||||
StepRecord(step_id=step_id, session_id=session_id, step_number=1, pre_image_name="window-a")
|
||||
)
|
||||
engine = DataEngine.__new__(DataEngine)
|
||||
engine.storage = storage
|
||||
engine._run_in_background = lambda fn, *args: fn(*args)
|
||||
engine.get_step_number = lambda _step_id: 1
|
||||
engine._get_generic_tools_for_sse = lambda _step_id: []
|
||||
engine._publish = Mock()
|
||||
engine.record_trace = Mock()
|
||||
engine.get_relative_time = lambda _timestamp: "0s"
|
||||
ctx = SimpleNamespace(execution_target=target, data_engine=engine)
|
||||
node = SimpleNamespace(
|
||||
ctx=ctx,
|
||||
_parse_decisions=lambda _: ([{"action": "plugin-click"}], None),
|
||||
_get_mcp_session=AsyncMock(),
|
||||
_get_initial_screenshot=AsyncMock(return_value=("old-screen", "window-a")),
|
||||
)
|
||||
state = SimpleNamespace(
|
||||
structured_decisions=None, current_step_id=str(step_id), open_incident=None
|
||||
)
|
||||
outcome = execution_loop._ActionOutcome(
|
||||
action_item={
|
||||
"action": "plugin-click",
|
||||
"plugin_id": "visual-test",
|
||||
"source_id": "window-b",
|
||||
"post_image_name": "window-b",
|
||||
"pre_image_valid": pre_valid,
|
||||
},
|
||||
success=True,
|
||||
post_screenshot_b64="new-screen",
|
||||
)
|
||||
monkeypatch.setattr(execution_loop, "_process_action", AsyncMock(return_value=outcome))
|
||||
monkeypatch.setattr(execution_loop, "_absorb_post_screenshot", lambda *_args: "window-b")
|
||||
await execution_loop.run_validation_loop(node, state)
|
||||
step = storage.get_steps(session_id)[0].model_dump()
|
||||
assert (step["pre_image_name"] is None) is cleared
|
||||
if cleared:
|
||||
assert step["post_image_name"] == "window-b"
|
||||
assert engine._publish.call_args.args[1]["pre_image_name"] is None
|
||||
before = tmp_path / "window-a.png"
|
||||
after = tmp_path / "window-b.png"
|
||||
before.write_bytes(b"a")
|
||||
after.write_bytes(b"b")
|
||||
monkeypatch.setattr(
|
||||
report_export,
|
||||
"image_path",
|
||||
lambda name: {"window-a": before, "window-b": after}.get(name),
|
||||
)
|
||||
exported = report_export.session_screenshots({"steps": [step]})
|
||||
assert [path for path, _name in exported] == [after]
|
||||
assert all("操作前" not in name for _path, name in exported)
|
||||
Reference in new issue
Block a user