chenmingxuan
fac4ccb760
Add dark theme, richer Markdown and readable plugin results. Improve Android input evidence, response integrity and diagnostics. Unify lifecycle, guidance receipts and evidence sources across clients. Keep direct action dispatch without an added AI quota approval layer. Align version metadata at 2.2.0 and bundle Helper 1.2.2. Validation: 3063 backend tests before the version bump; 44 after. Frontend: 317 tests and production build passed.
454 lines
16 KiB
Python
454 lines
16 KiB
Python
"""Plugin contracts and evidence persistence, without devices or external services."""
|
|
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, Mock
|
|
from uuid import uuid4
|
|
|
|
from mcp.types import CallToolResult, ImageContent, TextContent, Tool, ToolAnnotations
|
|
import pytest
|
|
|
|
from artemis.plugins.evidence import (
|
|
decode_receipt,
|
|
persist_attachments,
|
|
persist_receipt,
|
|
source_identity,
|
|
validate_tools,
|
|
)
|
|
from artemis.plugins.models import EvidenceSource, PluginCallResult, PluginManifest
|
|
from artemis.plugins.runtime import PluginRuntime
|
|
|
|
|
|
def manifest(**updates):
|
|
return PluginManifest.model_validate(
|
|
{
|
|
"schema_version": 2,
|
|
"id": "visual-test",
|
|
"name": "Visual fixture",
|
|
"version": "1.0.0",
|
|
"skill": "SKILL.md",
|
|
"observation": {"tool": "observe", "refresh_tools": ["observe"]},
|
|
"capabilities": {
|
|
"kind": "visual",
|
|
"recording": "windows",
|
|
"tools": {
|
|
"observe": {
|
|
"effect": "read",
|
|
"observation": True,
|
|
"target_argument": "app",
|
|
}
|
|
},
|
|
"required_tools": ["observe"],
|
|
},
|
|
**updates,
|
|
}
|
|
)
|
|
|
|
|
|
def observation_tool(*, readonly=True, properties=None):
|
|
return Tool(
|
|
name="observe",
|
|
inputSchema={
|
|
"type": "object",
|
|
"properties": {"app": {}} if properties is None else properties,
|
|
},
|
|
annotations=ToolAnnotations(readOnlyHint=readonly),
|
|
)
|
|
|
|
|
|
def attachment(data=b"verified evidence", name="capture.png"):
|
|
return {
|
|
"name": name,
|
|
"data": base64.b64encode(data).decode(),
|
|
"sha256": hashlib.sha256(data).hexdigest(),
|
|
"media_type": "image/png",
|
|
}
|
|
|
|
|
|
def test_schema_one_stays_compatible_but_schema_two_requires_contract():
|
|
legacy = manifest(schema_version=1, capabilities=None)
|
|
validate_tools(legacy, [observation_tool(readonly=False)])
|
|
with pytest.raises(ValueError, match="capabilities"):
|
|
manifest(capabilities=None)
|
|
with pytest.raises(ValueError, match="schema_version=2"):
|
|
manifest(schema_version=1)
|
|
|
|
|
|
@pytest.mark.parametrize("effect,observation", [("write", True), ("read", False)])
|
|
def test_auto_observation_requires_readonly_contract(effect, observation):
|
|
with pytest.raises(ValueError, match="只读"):
|
|
manifest(
|
|
capabilities={
|
|
"kind": "visual",
|
|
"tools": {"observe": {"effect": effect, "observation": observation}},
|
|
}
|
|
)
|
|
|
|
|
|
def test_nonvisual_plugin_cannot_request_screen_recording():
|
|
with pytest.raises(ValueError, match="视觉插件"):
|
|
manifest(observation=None, capabilities={"kind": "api", "recording": "windows"})
|
|
|
|
|
|
def test_manifest_is_checked_against_actual_mcp_tools():
|
|
validate_tools(manifest(), [observation_tool()])
|
|
with pytest.raises(ValueError, match="只读声明"):
|
|
validate_tools(manifest(), [observation_tool(readonly=False)])
|
|
with pytest.raises(ValueError, match="缺少目标参数"):
|
|
validate_tools(manifest(), [observation_tool(properties={})])
|
|
with pytest.raises(ValueError, match="缺少：observe"):
|
|
validate_tools(manifest(), [])
|
|
with pytest.raises(ValueError, match="未声明：click"):
|
|
validate_tools(manifest(), [observation_tool(), Tool(name="click", inputSchema={})])
|
|
|
|
|
|
def test_source_identity_invalidates_geometry_but_not_title_or_capture_time():
|
|
source = EvidenceSource(
|
|
kind="computer",
|
|
target_id="123",
|
|
label="before",
|
|
captured_at=1,
|
|
width=800,
|
|
height=600,
|
|
window={"left": 0, "top": 0, "width": 800, "height": 600},
|
|
)
|
|
identity = source_identity("visual-test", source)
|
|
assert identity == source_identity(
|
|
"visual-test", source.model_copy(update={"label": "after", "captured_at": 2})
|
|
)
|
|
for change in (
|
|
{"target_id": "456"},
|
|
{"width": 400},
|
|
{"window": {"left": 100, "top": 0, "width": 800, "height": 600}},
|
|
):
|
|
assert identity != source_identity("visual-test", source.model_copy(update=change))
|
|
assert identity != source_identity("another-plugin", source)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"execution,retryable,observe",
|
|
[
|
|
("unknown", False, True),
|
|
("partial", False, True),
|
|
("completed", False, False),
|
|
("not_started", True, False),
|
|
],
|
|
)
|
|
def test_uncertain_writes_cannot_claim_to_be_retryable(execution, retryable, observe):
|
|
result = PluginCallResult(
|
|
status="error",
|
|
structured_content={
|
|
"artemis": {
|
|
"version": 1,
|
|
"execution": execution,
|
|
"side_effect": "possible",
|
|
"retryable": True,
|
|
"error_code": "remote_failure",
|
|
}
|
|
},
|
|
)
|
|
decode_receipt(result)
|
|
assert result.execution == execution
|
|
assert result.retryable is retryable
|
|
assert result.requires_observation is observe
|
|
assert result.error_code == "remote_failure"
|
|
|
|
|
|
@pytest.mark.parametrize("raw", ["invalid", {"version": 99}])
|
|
def test_rejects_unsupported_receipt_shapes(raw):
|
|
with pytest.raises(ValueError):
|
|
decode_receipt(PluginCallResult(status="success", structured_content={"artemis": raw}))
|
|
|
|
|
|
def test_attachments_are_read_back_and_never_overwrite_existing_evidence(tmp_path):
|
|
receipt = attachment()
|
|
saved = persist_attachments(tmp_path, "visual-test", [receipt])[0]
|
|
path = tmp_path / saved["path"]
|
|
assert path.read_bytes() == b"verified evidence"
|
|
assert saved["sha256"] == receipt["sha256"]
|
|
assert saved["bytes"] == len(b"verified evidence")
|
|
assert (
|
|
persist_attachments(
|
|
tmp_path, "visual-test", [{k: v for k, v in receipt.items() if k != "data"}]
|
|
)[0]
|
|
== saved
|
|
)
|
|
with pytest.raises(ValueError, match="不能覆盖"):
|
|
persist_attachments(tmp_path, "visual-test", [attachment(b"different")])
|
|
assert path.read_bytes() == b"verified evidence"
|
|
|
|
|
|
def test_invalid_hash_does_not_publish_attachment(tmp_path):
|
|
receipt = attachment()
|
|
receipt["sha256"] = "0" * 64
|
|
with pytest.raises(ValueError, match="校验失败"):
|
|
persist_attachments(tmp_path, "visual-test", [receipt])
|
|
assert not (tmp_path / "artifacts/plugins/visual-test/capture.png").exists()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"name",
|
|
[
|
|
"../outside.png",
|
|
"..\\outside.png",
|
|
"C:\\outside.png",
|
|
"file.png:stream",
|
|
"CON.png",
|
|
"LPT1",
|
|
"file.",
|
|
"file ",
|
|
"bad\nname",
|
|
],
|
|
)
|
|
def test_attachment_names_cannot_escape_or_alias_windows_files(tmp_path, name):
|
|
with pytest.raises(ValueError, match="普通文件名"):
|
|
persist_attachments(tmp_path, "visual-test", [attachment(name=name)])
|
|
|
|
|
|
def test_receipt_index_has_references_instead_of_binary_content(tmp_path):
|
|
result = PluginCallResult(
|
|
status="success",
|
|
call_id="call-1",
|
|
image_names=["image-1"],
|
|
content=[{"type": "image", "data": "large-binary"}],
|
|
structured_content={"artemis": {"large": "payload"}},
|
|
)
|
|
persist_receipt(
|
|
SimpleNamespace(data_engine=SimpleNamespace(base_dir=tmp_path)),
|
|
result,
|
|
{"app": "fixture"},
|
|
123,
|
|
)
|
|
receipt = json.loads((tmp_path / "evidence/calls.jsonl").read_text(encoding="utf-8"))
|
|
assert receipt["call_id"] == "call-1"
|
|
assert receipt["image_names"] == ["image-1"]
|
|
assert receipt["started_at"] == 123
|
|
assert "content" not in receipt
|
|
assert "structured_content" not in receipt
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_receipt_disk_failure_preserves_successful_write_result(monkeypatch):
|
|
runtime = PluginRuntime.__new__(PluginRuntime)
|
|
runtime.ctx = None
|
|
runtime.bindings = {
|
|
"write": (
|
|
"visual-test",
|
|
SimpleNamespace(name="write", inputSchema={"type": "object"}, annotations=None),
|
|
)
|
|
}
|
|
runtime.packages = {
|
|
"visual-test": (
|
|
SimpleNamespace(capabilities=None, observation=None, version="1.0.0"),
|
|
SimpleNamespace(argument_constraints={}, secrets={}),
|
|
None,
|
|
)
|
|
}
|
|
session = SimpleNamespace(
|
|
call=AsyncMock(
|
|
return_value=CallToolResult(
|
|
content=[TextContent(type="text", text="changed")], isError=False
|
|
)
|
|
),
|
|
stop_uncertain=False,
|
|
)
|
|
runtime.sessions = {"visual-test": session}
|
|
runtime.ensure_resources_ready = lambda: None
|
|
runtime._image_targets = {}
|
|
runtime.last_images = {}
|
|
runtime._observation_calls = {}
|
|
|
|
def unavailable(*_args, **_kwargs):
|
|
raise OSError("disk full")
|
|
|
|
monkeypatch.setattr("artemis.plugins.runtime.persist_receipt", unavailable)
|
|
outcome = await runtime.call("write", {})
|
|
session.call.assert_awaited_once()
|
|
assert outcome.status == "success"
|
|
assert outcome.execution == "completed"
|
|
assert outcome.side_effect != "none"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_write_popup_source_updates_cached_observation_target():
|
|
runtime = PluginRuntime.__new__(PluginRuntime)
|
|
runtime.ctx = None
|
|
runtime.bindings = {
|
|
"write": ("visual-test", Tool(name="write", inputSchema={"type": "object"}))
|
|
}
|
|
runtime.packages = {
|
|
"visual-test": (
|
|
manifest(
|
|
capabilities={
|
|
"kind": "visual",
|
|
"tools": {
|
|
"observe": {
|
|
"effect": "read",
|
|
"observation": True,
|
|
"target_argument": "window_id",
|
|
},
|
|
"write": {"effect": "write", "target_argument": "window_id"},
|
|
},
|
|
}
|
|
),
|
|
SimpleNamespace(argument_constraints={}, secrets={}),
|
|
None,
|
|
)
|
|
}
|
|
runtime.sessions = {
|
|
"visual-test": SimpleNamespace(
|
|
call=AsyncMock(
|
|
return_value=CallToolResult(
|
|
content=[ImageContent(type="image", mimeType="image/png", data="aW1hZ2U=")],
|
|
structuredContent={
|
|
"artemis": {
|
|
"version": 1,
|
|
"execution": "completed",
|
|
"source": {"kind": "computer", "target_id": "123"},
|
|
}
|
|
},
|
|
)
|
|
),
|
|
stop_uncertain=False,
|
|
)
|
|
}
|
|
runtime.ensure_resources_ready = lambda: None
|
|
runtime._image_targets = {}
|
|
runtime.last_images = {}
|
|
runtime._observation_calls = {"visual-test": ("observe", {"pid": 7, "window_id": 10})}
|
|
runtime._recordings = None
|
|
first = await runtime.call("write", {"pid": 7, "window_id": 10})
|
|
assert first.status == "success"
|
|
assert first.pre_image_valid is False
|
|
assert runtime._image_targets["visual-test"][0] == "123"
|
|
assert runtime._observation_calls["visual-test"] == ("observe", {"pid": 7, "window_id": 123})
|
|
second = await runtime.call("write", {"pid": 7, "window_id": 123})
|
|
assert second.pre_image_valid is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_v2_observation_without_task_context_never_starts_recording(monkeypatch):
|
|
runtime = PluginRuntime.__new__(PluginRuntime)
|
|
runtime.ctx = None
|
|
runtime.bindings = {"observe": ("visual-test", observation_tool())}
|
|
runtime.packages = {
|
|
"visual-test": (
|
|
manifest(),
|
|
SimpleNamespace(argument_constraints={}, secrets={}),
|
|
None,
|
|
)
|
|
}
|
|
runtime.sessions = {
|
|
"visual-test": SimpleNamespace(
|
|
call=AsyncMock(
|
|
return_value=CallToolResult(
|
|
content=[ImageContent(type="image", mimeType="image/png", data="aW1hZ2U=")],
|
|
structuredContent={
|
|
"artemis": {
|
|
"version": 1,
|
|
"execution": "completed",
|
|
"source": {"kind": "computer", "target_id": "123"},
|
|
}
|
|
},
|
|
)
|
|
),
|
|
stop_uncertain=False,
|
|
)
|
|
}
|
|
runtime.ensure_resources_ready = lambda: None
|
|
runtime._image_targets = {}
|
|
runtime.last_images = {}
|
|
runtime._observation_calls = {}
|
|
runtime._recordings = None
|
|
|
|
def unexpected_recorder(*_args):
|
|
pytest.fail("Recording requires an owning task trace")
|
|
|
|
monkeypatch.setattr("artemis.plugins.recording.PluginRecordings", unexpected_recorder)
|
|
outcome = await runtime.call("observe", {"app": "fixture"})
|
|
assert outcome.status == "success"
|
|
assert outcome.source.target_id == "123"
|
|
assert outcome.source_id.startswith("plugin:visual-test:")
|
|
assert runtime.last_images["visual-test"] == b"image"
|
|
assert runtime._recordings is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"target,pre_valid,cleared",
|
|
[
|
|
("computer", False, True),
|
|
("computer", True, False),
|
|
("android", False, True),
|
|
("mixed", False, True),
|
|
],
|
|
)
|
|
async def test_pro_cross_window_evidence_does_not_reuse_another_source_before_image(
|
|
tmp_path, monkeypatch, target, pre_valid, cleared
|
|
):
|
|
from artemis.agents.validator import execution_loop
|
|
from artemis.data_engine.engine import DataEngine
|
|
from artemis.data_engine.models import ImageRecord, SessionMetadata, StepRecord
|
|
from artemis.data_engine.storage import StorageManager
|
|
from apps.admin_console.services import report_export
|
|
|
|
storage = StorageManager(tmp_path / "evidence.db", tmp_path)
|
|
session_id, step_id = uuid4(), uuid4()
|
|
storage.create_session(SessionMetadata(session_id=session_id, initial_goal="fixture"))
|
|
storage.create_image(ImageRecord(image_name="window-a"))
|
|
storage.create_image(ImageRecord(image_name="window-b"))
|
|
storage.create_step(
|
|
StepRecord(step_id=step_id, session_id=session_id, step_number=1, pre_image_name="window-a")
|
|
)
|
|
engine = DataEngine.__new__(DataEngine)
|
|
engine.storage = storage
|
|
engine._run_in_background = lambda fn, *args: fn(*args)
|
|
engine.get_step_number = lambda _step_id: 1
|
|
engine._get_generic_tools_for_sse = lambda _step_id: []
|
|
engine._publish = Mock()
|
|
engine.record_trace = Mock()
|
|
engine.get_relative_time = lambda _timestamp: "0s"
|
|
ctx = SimpleNamespace(execution_target=target, data_engine=engine)
|
|
node = SimpleNamespace(
|
|
ctx=ctx,
|
|
_parse_decisions=lambda _: ([{"action": "plugin-click"}], None),
|
|
_get_mcp_session=AsyncMock(),
|
|
_get_initial_screenshot=AsyncMock(return_value=("old-screen", "window-a")),
|
|
)
|
|
state = SimpleNamespace(
|
|
structured_decisions=None, current_step_id=str(step_id), open_incident=None
|
|
)
|
|
outcome = execution_loop._ActionOutcome(
|
|
action_item={
|
|
"action": "plugin-click",
|
|
"plugin_id": "visual-test",
|
|
"source_id": "window-b",
|
|
"post_image_name": "window-b",
|
|
"pre_image_valid": pre_valid,
|
|
},
|
|
success=True,
|
|
post_screenshot_b64="new-screen",
|
|
)
|
|
monkeypatch.setattr(execution_loop, "_process_action", AsyncMock(return_value=outcome))
|
|
monkeypatch.setattr(execution_loop, "_absorb_post_screenshot", lambda *_args: "window-b")
|
|
await execution_loop.run_validation_loop(node, state)
|
|
step = storage.get_steps(session_id)[0].model_dump()
|
|
assert (step["pre_image_name"] is None) is cleared
|
|
if cleared:
|
|
assert step["post_image_name"] == "window-b"
|
|
assert engine._publish.call_args.args[1]["pre_image_name"] is None
|
|
before = tmp_path / "window-a.png"
|
|
after = tmp_path / "window-b.png"
|
|
before.write_bytes(b"a")
|
|
after.write_bytes(b"b")
|
|
monkeypatch.setattr(
|
|
report_export,
|
|
"image_path",
|
|
lambda name: {"window-a": before, "window-b": after}.get(name),
|
|
)
|
|
exported = report_export.session_screenshots({"steps": [step]})
|
|
assert [path for path, _name in exported] == [after]
|
|
assert all("操作前" not in name for _path, name in exported)
|