219 lines
8.6 KiB
Python
219 lines
8.6 KiB
Python
"""Cross-device results must reach the model and history on their own resource."""
|
|
|
|
import base64
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
|
from langchain_openai import ChatOpenAI
|
|
from PIL import Image
|
|
|
|
from artemis.context import ArtemisContext
|
|
from artemis.agents.validator.execution_loop import _process_action
|
|
from artemis.agents.validator.tool_declarations import ToolExecutionResult
|
|
from artemis.data_engine.engine import DataEngine
|
|
from artemis.data_engine.history_reader import OfflineHistoryReader
|
|
from artemis.data_engine.models import ImageRecord, SessionMetadata, StepRecord
|
|
from artemis.data_engine.storage import StorageManager
|
|
from artemis.memory.transcript import TranscriptLedger
|
|
from artemis.plugins.results import plugin_result_messages
|
|
from artemis.tools.history.screenshot import load_step_screenshot
|
|
|
|
|
|
def _action(call_id, name="plug_test_get_app_state", **kwargs):
|
|
return dict(
|
|
action=name,
|
|
tool_call_id=call_id,
|
|
plugin_id="test",
|
|
plugin_status="success",
|
|
result=f"来源：电脑插件 / {call_id}\nAddress bar",
|
|
**kwargs,
|
|
)
|
|
|
|
|
|
def _stage(ledger, actions):
|
|
ledger.stage_turn(
|
|
[
|
|
AIMessage(
|
|
content="",
|
|
tool_calls=[
|
|
{"name": a["action"], "id": a["tool_call_id"], "args": {}} for a in actions
|
|
],
|
|
),
|
|
*[
|
|
ToolMessage(content="Action Recorded", tool_call_id=a["tool_call_id"])
|
|
for a in actions
|
|
],
|
|
]
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("image_format", ["JPEG", "PNG", "WEBP"])
|
|
def test_burst_results_reach_responses_once_with_images_before_compression(tmp_path, image_format):
|
|
image = tmp_path / "pc.jpg"
|
|
Image.new("RGB", (200, 100), "blue").save(image, format=image_format)
|
|
engine = SimpleNamespace(get_image_path=lambda _: image)
|
|
actions = [_action(f"call_{i}", post_image_name="pc") for i in range(4)]
|
|
# A long accessibility tree and the tail must both survive delivery.
|
|
actions[0]["result"] += " UI node" * 12000 + " END_OF_TREE"
|
|
actions[1]["plugin_status"] = "error"
|
|
actions[1]["result"] = "窗口已关闭，请重新观察。"
|
|
actions[2].pop("plugin_status")
|
|
actions[2].pop("result")
|
|
actions[2].pop("post_image_name")
|
|
actions[2]["attempts"] = ["Skipped"]
|
|
report = {"status": "failed", "execution": actions}
|
|
ledger = TranscriptLedger(
|
|
image_scrub_depth=1, pending_grace_steps=0, context_budget_tokens=300_000
|
|
)
|
|
_stage(ledger, actions)
|
|
ledger.commit_staged(
|
|
step_key="step1",
|
|
validator_result=report,
|
|
tool_results=plugin_result_messages(report, engine),
|
|
)
|
|
messages = ledger.render([])
|
|
outputs = [m for m in messages if isinstance(m, ToolMessage)]
|
|
assert [m.tool_call_id for m in outputs] == [f"call_{i}" for i in range(4)]
|
|
assert [m.status for m in outputs] == ["success", "error", "error", "success"]
|
|
assert "END_OF_TREE" in str(outputs[0].content)
|
|
assert "未完成或未执行" in str(outputs[2].content)
|
|
assert "Action Recorded" not in str(messages)
|
|
wire = ChatOpenAI(model="fixture", api_key="fixture", use_responses_api=True, store=False)
|
|
payload = wire._get_request_payload(messages)["input"]
|
|
assert len([item for item in payload if item.get("type") == "function_call_output"]) == 4
|
|
assert str(payload).count(base64.b64encode(image.read_bytes()).decode()) == 3
|
|
assert f"data:{Image.MIME[image_format]};base64," in str(payload)
|
|
# No duplicate outputs or rewritten images on the following ordinary turn.
|
|
ledger.commit_staged(
|
|
validator_result=report, tool_results=plugin_result_messages(report, engine)
|
|
)
|
|
later = ledger.render([])
|
|
assert len([m for m in later if isinstance(m, ToolMessage)]) == 4
|
|
assert str(later).count(base64.b64encode(image.read_bytes()).decode()) == 3
|
|
ledger.compact_active()
|
|
later = ledger.render([])
|
|
assert str(later).count(base64.b64encode(image.read_bytes()).decode()) < 3
|
|
assert str(messages).count(base64.b64encode(image.read_bytes()).decode()) == 3
|
|
|
|
|
|
def test_missing_plugin_screenshot_keeps_real_result_and_reports_missing_evidence(tmp_path):
|
|
action = _action("call", post_image_name="missing")
|
|
messages = plugin_result_messages(
|
|
{"execution": [action]}, SimpleNamespace(get_image_path=lambda _: tmp_path / "missing.jpg")
|
|
)["call"]
|
|
assert "Address bar" in str(messages)
|
|
assert "截图无法读取" in str(messages)
|
|
assert not any(isinstance(m, HumanMessage) for m in messages)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_validator_preserves_model_call_id_and_actual_plugin_result(monkeypatch):
|
|
name = "plug_test_get_app_state"
|
|
plugins = SimpleNamespace(
|
|
bindings={name: None},
|
|
execute=AsyncMock(
|
|
return_value=ToolExecutionResult(
|
|
tool_name=name,
|
|
tool_call_id="original",
|
|
status="success",
|
|
text_summary="PC tree",
|
|
screenshot_bytes=b"pc",
|
|
metadata={"plugin_id": "test", "plugin_status": "success"},
|
|
)
|
|
),
|
|
)
|
|
monkeypatch.setattr("artemis.plugins.runtime.get_plugin_runtime", lambda _: plugins)
|
|
ctx = ArtemisContext.model_construct()
|
|
outcome = await _process_action(
|
|
SimpleNamespace(ctx=ctx),
|
|
None,
|
|
None,
|
|
{"action": name, "args": {"app": "browser"}, "tool_call_id": "original"},
|
|
name,
|
|
"phone",
|
|
burst=False,
|
|
)
|
|
plugins.execute.assert_awaited_once_with(name, {"app": "browser"}, "original")
|
|
assert outcome.action_item["tool_call_id"] == "original"
|
|
assert outcome.action_item["result"] == "PC tree"
|
|
assert base64.b64decode(outcome.post_screenshot_b64) == b"pc"
|
|
|
|
|
|
@pytest.mark.parametrize("live", [False, True])
|
|
def test_old_mixed_step_history_returns_pc_post_and_never_phone_pre(tmp_path, live):
|
|
(tmp_path / "images").mkdir()
|
|
for name, size in [("phone", (100, 200)), ("pc", (200, 100))]:
|
|
Image.new("RGB", size).save(tmp_path / "images" / f"{name}.jpg")
|
|
storage = StorageManager(tmp_path / "data_engine.db", tmp_path)
|
|
session_id = uuid4()
|
|
storage.create_session(SessionMetadata(session_id=session_id, initial_goal="fixture"))
|
|
for name in ("phone", "pc"):
|
|
storage.create_image(ImageRecord(image_name=name))
|
|
storage.create_step(
|
|
StepRecord(
|
|
step_id=uuid4(),
|
|
session_id=session_id,
|
|
step_number=24,
|
|
pre_image_name="phone",
|
|
last_execution_result={"execution": [_action("original", post_image_name="pc")]},
|
|
)
|
|
)
|
|
reader = OfflineHistoryReader(tmp_path / "data_engine.db", tmp_path, session_id)
|
|
if live:
|
|
engine = DataEngine.__new__(DataEngine)
|
|
engine.get_step_record = reader.get_step_record
|
|
engine.get_image_path = reader.get_image_path
|
|
reader = engine
|
|
assert load_step_screenshot(reader, 24, "pre").image_bytes is None
|
|
assert (
|
|
load_step_screenshot(reader, 24, "post").image_bytes
|
|
== (tmp_path / "images/pc.jpg").read_bytes()
|
|
)
|
|
|
|
|
|
def test_cross_resource_burst_has_no_single_step_image():
|
|
from artemis.data_engine.step_evidence import step_image_name
|
|
|
|
record = {
|
|
"pre_image_name": "phone",
|
|
"post_image_name": "pc",
|
|
"last_execution_result": {
|
|
"execution": [{"action": "tap"}, _action("pc", post_image_name="pc")]
|
|
},
|
|
}
|
|
assert step_image_name(record, "pre") is None
|
|
assert step_image_name(record, "post") is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_interrupted_plugin_call_is_not_reported_as_executed(monkeypatch):
|
|
from artemis.runtime.task_control import TaskControl
|
|
|
|
name = "plug_test_click"
|
|
plugins = SimpleNamespace(bindings={name: None}, execute=AsyncMock())
|
|
monkeypatch.setattr("artemis.plugins.runtime.get_plugin_runtime", lambda _: plugins)
|
|
control = TaskControl.__new__(TaskControl)
|
|
|
|
async def interrupt(operation, **_kwargs):
|
|
operation.close()
|
|
return True, None
|
|
|
|
control.run_interruptibly = interrupt
|
|
outcome = await _process_action(
|
|
SimpleNamespace(ctx=SimpleNamespace(task_control=control)),
|
|
None,
|
|
None,
|
|
{"action": name, "args": {}, "tool_call_id": "interrupted"},
|
|
name,
|
|
"phone",
|
|
burst=False,
|
|
)
|
|
assert not outcome.success
|
|
messages = plugin_result_messages({"execution": [outcome.action_item]}, None)["interrupted"]
|
|
assert messages[0].status == "error"
|
|
assert "打断" in str(messages[0].content)
|
|
plugins.execute.assert_not_awaited()
|