chenmingxuan
e1ea099592
Install and update artemis-use with task diagnostics and plugin guidance. Preserve cross-device file receipts, review canonical reports, guard stage transitions and reduce small history summaries. Improve exploration lifecycle, request recovery and browser live connections.
275 lines
11 KiB
Python
275 lines
11 KiB
Python
"""Cross-device results must reach the model and history on their own resource."""
|
|
|
|
import base64
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
|
from langchain_openai import ChatOpenAI
|
|
from PIL import Image
|
|
|
|
from artemis.context import ArtemisContext
|
|
from artemis.agents.validator.execution_loop import _process_action
|
|
from artemis.agents.validator.tool_declarations import ToolExecutionResult
|
|
from artemis.data_engine.engine import DataEngine
|
|
from artemis.data_engine.history_reader import OfflineHistoryReader
|
|
from artemis.data_engine.models import ImageRecord, SessionMetadata, StepRecord
|
|
from artemis.data_engine.storage import StorageManager
|
|
from artemis.memory.transcript import TranscriptLedger
|
|
from artemis.plugins.results import plugin_result_messages
|
|
from artemis.plugins.models import PluginCallResult
|
|
from artemis.plugins.runtime import PluginRuntime
|
|
from artemis.tools.history.screenshot import load_step_screenshot
|
|
|
|
|
|
def test_large_tree_deduplicates_without_mutating_raw_evidence():
|
|
tree = "- Button abc\n" * 3000
|
|
result = PluginCallResult(
|
|
status="success",
|
|
plugin_id="test",
|
|
content=[{"type": "text", "text": "Screenshot\n" + tree}],
|
|
structured_content={
|
|
"tree_markdown": tree,
|
|
"capture_id": "capture-current",
|
|
"window_bounds": {"width": 2560},
|
|
},
|
|
)
|
|
text = result.text()
|
|
assert text.count(tree) == 1
|
|
assert result.structured_content["tree_markdown"] == tree
|
|
assert "capture-current" in text and "2560" in text
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ["flash", "pro"])
|
|
def test_verified_receipts_survive_observation_truncation_for_both_modes(mode):
|
|
receipt = {
|
|
"path": "artifacts/plugins/test/screen.png",
|
|
"sha256": "a" * 64,
|
|
"bytes": 123,
|
|
"media_type": "image/png",
|
|
}
|
|
result = PluginCallResult(
|
|
status="success",
|
|
plugin_id="test",
|
|
content=[{"type": "text", "text": "TREE " * 10000}],
|
|
attachments=[receipt],
|
|
)
|
|
if mode == "flash":
|
|
blocks = PluginRuntime.content_blocks(result)
|
|
else:
|
|
action = _action("call", attachments=[receipt])
|
|
action["result"] = result.text()
|
|
blocks = plugin_result_messages({"execution": [action]}, None)["call"][0].content
|
|
text = str(blocks)
|
|
assert receipt["path"] in text and receipt["sha256"] in text
|
|
assert "verified file receipts" in text
|
|
assert "观察文本已节选" in text
|
|
assert len(text) < 14000
|
|
|
|
|
|
def test_plugin_claim_does_not_become_runtime_receipt():
|
|
result = PluginCallResult(
|
|
status="success",
|
|
plugin_id="test",
|
|
structured_content={"attachments": [{"path": "pretend.png", "verified": True}]},
|
|
)
|
|
assert "verified file receipts" not in str(PluginRuntime.content_blocks(result))
|
|
|
|
|
|
def _action(call_id, name="plug_test_get_app_state", **kwargs):
|
|
return dict(
|
|
action=name,
|
|
tool_call_id=call_id,
|
|
plugin_id="test",
|
|
plugin_status="success",
|
|
result=f"来源：电脑插件 / {call_id}\nAddress bar",
|
|
**kwargs,
|
|
)
|
|
|
|
|
|
def _stage(ledger, actions):
|
|
ledger.stage_turn(
|
|
[
|
|
AIMessage(
|
|
content="",
|
|
tool_calls=[
|
|
{"name": a["action"], "id": a["tool_call_id"], "args": {}} for a in actions
|
|
],
|
|
),
|
|
*[
|
|
ToolMessage(content="Action Recorded", tool_call_id=a["tool_call_id"])
|
|
for a in actions
|
|
],
|
|
]
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("image_format", ["JPEG", "PNG", "WEBP"])
|
|
def test_burst_results_reach_responses_once_with_images_before_compression(tmp_path, image_format):
|
|
image = tmp_path / "pc.jpg"
|
|
Image.new("RGB", (200, 100), "blue").save(image, format=image_format)
|
|
engine = SimpleNamespace(get_image_path=lambda _: image)
|
|
actions = [_action(f"call_{i}", post_image_name="pc") for i in range(4)]
|
|
# A long accessibility tree and the tail must both survive delivery.
|
|
actions[0]["result"] += " UI node" * 12000 + " END_OF_TREE"
|
|
actions[1]["plugin_status"] = "error"
|
|
actions[1]["result"] = "窗口已关闭，请重新观察。"
|
|
actions[2].pop("plugin_status")
|
|
actions[2].pop("result")
|
|
actions[2].pop("post_image_name")
|
|
actions[2]["attempts"] = ["Skipped"]
|
|
report = {"status": "failed", "execution": actions}
|
|
ledger = TranscriptLedger(
|
|
image_scrub_depth=1, pending_grace_steps=0, context_budget_tokens=300_000
|
|
)
|
|
_stage(ledger, actions)
|
|
ledger.commit_staged(
|
|
step_key="step1",
|
|
validator_result=report,
|
|
tool_results=plugin_result_messages(report, engine),
|
|
)
|
|
messages = ledger.render([])
|
|
outputs = [m for m in messages if isinstance(m, ToolMessage)]
|
|
assert [m.tool_call_id for m in outputs] == [f"call_{i}" for i in range(4)]
|
|
assert [m.status for m in outputs] == ["success", "error", "error", "success"]
|
|
assert "END_OF_TREE" in str(outputs[0].content)
|
|
assert "未完成或未执行" in str(outputs[2].content)
|
|
assert "Action Recorded" not in str(messages)
|
|
wire = ChatOpenAI(model="fixture", api_key="fixture", use_responses_api=True, store=False)
|
|
payload = wire._get_request_payload(messages)["input"]
|
|
assert len([item for item in payload if item.get("type") == "function_call_output"]) == 4
|
|
assert str(payload).count(base64.b64encode(image.read_bytes()).decode()) == 3
|
|
assert f"data:{Image.MIME[image_format]};base64," in str(payload)
|
|
# No duplicate outputs or rewritten images on the following ordinary turn.
|
|
ledger.commit_staged(
|
|
validator_result=report, tool_results=plugin_result_messages(report, engine)
|
|
)
|
|
later = ledger.render([])
|
|
assert len([m for m in later if isinstance(m, ToolMessage)]) == 4
|
|
assert str(later).count(base64.b64encode(image.read_bytes()).decode()) == 3
|
|
ledger.compact_active()
|
|
later = ledger.render([])
|
|
assert str(later).count(base64.b64encode(image.read_bytes()).decode()) < 3
|
|
assert str(messages).count(base64.b64encode(image.read_bytes()).decode()) == 3
|
|
|
|
|
|
def test_missing_plugin_screenshot_keeps_real_result_and_reports_missing_evidence(tmp_path):
|
|
action = _action("call", post_image_name="missing")
|
|
messages = plugin_result_messages(
|
|
{"execution": [action]}, SimpleNamespace(get_image_path=lambda _: tmp_path / "missing.jpg")
|
|
)["call"]
|
|
assert "Address bar" in str(messages)
|
|
assert "截图无法读取" in str(messages)
|
|
assert not any(isinstance(m, HumanMessage) for m in messages)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_validator_preserves_model_call_id_and_actual_plugin_result(monkeypatch):
|
|
name = "plug_test_get_app_state"
|
|
plugins = SimpleNamespace(
|
|
bindings={name: None},
|
|
execute=AsyncMock(
|
|
return_value=ToolExecutionResult(
|
|
tool_name=name,
|
|
tool_call_id="original",
|
|
status="success",
|
|
text_summary="PC tree",
|
|
screenshot_bytes=b"pc",
|
|
metadata={"plugin_id": "test", "plugin_status": "success"},
|
|
)
|
|
),
|
|
)
|
|
monkeypatch.setattr("artemis.plugins.runtime.get_plugin_runtime", lambda _: plugins)
|
|
ctx = ArtemisContext.model_construct()
|
|
outcome = await _process_action(
|
|
SimpleNamespace(ctx=ctx),
|
|
None,
|
|
None,
|
|
{"action": name, "args": {"app": "browser"}, "tool_call_id": "original"},
|
|
name,
|
|
"phone",
|
|
burst=False,
|
|
)
|
|
plugins.execute.assert_awaited_once_with(name, {"app": "browser"}, "original")
|
|
assert outcome.action_item["tool_call_id"] == "original"
|
|
assert outcome.action_item["result"] == "PC tree"
|
|
assert base64.b64decode(outcome.post_screenshot_b64) == b"pc"
|
|
|
|
|
|
@pytest.mark.parametrize("live", [False, True])
|
|
def test_old_mixed_step_history_returns_pc_post_and_never_phone_pre(tmp_path, live):
|
|
(tmp_path / "images").mkdir()
|
|
for name, size in [("phone", (100, 200)), ("pc", (200, 100))]:
|
|
Image.new("RGB", size).save(tmp_path / "images" / f"{name}.jpg")
|
|
storage = StorageManager(tmp_path / "data_engine.db", tmp_path)
|
|
session_id = uuid4()
|
|
storage.create_session(SessionMetadata(session_id=session_id, initial_goal="fixture"))
|
|
for name in ("phone", "pc"):
|
|
storage.create_image(ImageRecord(image_name=name))
|
|
storage.create_step(
|
|
StepRecord(
|
|
step_id=uuid4(),
|
|
session_id=session_id,
|
|
step_number=24,
|
|
pre_image_name="phone",
|
|
last_execution_result={"execution": [_action("original", post_image_name="pc")]},
|
|
)
|
|
)
|
|
reader = OfflineHistoryReader(tmp_path / "data_engine.db", tmp_path, session_id)
|
|
if live:
|
|
engine = DataEngine.__new__(DataEngine)
|
|
engine.get_step_record = reader.get_step_record
|
|
engine.get_image_path = reader.get_image_path
|
|
reader = engine
|
|
assert load_step_screenshot(reader, 24, "pre").image_bytes is None
|
|
assert (
|
|
load_step_screenshot(reader, 24, "post").image_bytes
|
|
== (tmp_path / "images/pc.jpg").read_bytes()
|
|
)
|
|
|
|
|
|
def test_cross_resource_burst_has_no_single_step_image():
|
|
from artemis.data_engine.step_evidence import step_image_name
|
|
|
|
record = {
|
|
"pre_image_name": "phone",
|
|
"post_image_name": "pc",
|
|
"last_execution_result": {
|
|
"execution": [{"action": "tap"}, _action("pc", post_image_name="pc")]
|
|
},
|
|
}
|
|
assert step_image_name(record, "pre") is None
|
|
assert step_image_name(record, "post") is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_interrupted_plugin_call_is_not_reported_as_executed(monkeypatch):
|
|
from artemis.runtime.task_control import TaskControl
|
|
|
|
name = "plug_test_click"
|
|
plugins = SimpleNamespace(bindings={name: None}, execute=AsyncMock())
|
|
monkeypatch.setattr("artemis.plugins.runtime.get_plugin_runtime", lambda _: plugins)
|
|
control = TaskControl.__new__(TaskControl)
|
|
|
|
async def interrupt(operation, **_kwargs):
|
|
operation.close()
|
|
return True, None
|
|
|
|
control.run_interruptibly = interrupt
|
|
outcome = await _process_action(
|
|
SimpleNamespace(ctx=SimpleNamespace(task_control=control)),
|
|
None,
|
|
None,
|
|
{"action": name, "args": {}, "tool_call_id": "interrupted"},
|
|
name,
|
|
"phone",
|
|
burst=False,
|
|
)
|
|
assert not outcome.success
|
|
messages = plugin_result_messages({"execution": [outcome.action_item]}, None)["interrupted"]
|
|
assert messages[0].status == "error"
|
|
assert "打断" in str(messages[0].content)
|
|
plugins.execute.assert_not_awaited()
|