"""Cross-device results must reach the model and history on their own resource.""" import base64 from types import SimpleNamespace from unittest.mock import AsyncMock from uuid import uuid4 import pytest from langchain_core.messages import AIMessage, HumanMessage, ToolMessage from langchain_openai import ChatOpenAI from PIL import Image from artemis.context import ArtemisContext from artemis.agents.validator.execution_loop import _process_action from artemis.agents.validator.tool_declarations import ToolExecutionResult from artemis.data_engine.engine import DataEngine from artemis.data_engine.history_reader import OfflineHistoryReader from artemis.data_engine.models import ImageRecord, SessionMetadata, StepRecord from artemis.data_engine.storage import StorageManager from artemis.memory.transcript import TranscriptLedger from artemis.plugins.results import plugin_result_messages from artemis.tools.history.screenshot import load_step_screenshot def _action(call_id, name="plug_test_get_app_state", **kwargs): return dict( action=name, tool_call_id=call_id, plugin_id="test", plugin_status="success", result=f"来源：电脑插件 / {call_id}\nAddress bar", **kwargs, ) def _stage(ledger, actions): ledger.stage_turn( [ AIMessage( content="", tool_calls=[ {"name": a["action"], "id": a["tool_call_id"], "args": {}} for a in actions ], ), *[ ToolMessage(content="Action Recorded", tool_call_id=a["tool_call_id"]) for a in actions ], ] ) @pytest.mark.parametrize("image_format", ["JPEG", "PNG", "WEBP"]) def test_burst_results_reach_responses_once_with_images_before_compression(tmp_path, image_format): image = tmp_path / "pc.jpg" Image.new("RGB", (200, 100), "blue").save(image, format=image_format) engine = SimpleNamespace(get_image_path=lambda _: image) actions = [_action(f"call_{i}", post_image_name="pc") for i in range(4)] # A long accessibility tree and the tail must both survive delivery. actions[0]["result"] += " UI node" * 12000 + " END_OF_TREE" actions[1]["plugin_status"] = "error" actions[1]["result"] = "窗口已关闭，请重新观察。" actions[2].pop("plugin_status") actions[2].pop("result") actions[2].pop("post_image_name") actions[2]["attempts"] = ["Skipped"] report = {"status": "failed", "execution": actions} ledger = TranscriptLedger( image_scrub_depth=1, pending_grace_steps=0, context_budget_tokens=300_000 ) _stage(ledger, actions) ledger.commit_staged( step_key="step1", validator_result=report, tool_results=plugin_result_messages(report, engine), ) messages = ledger.render([]) outputs = [m for m in messages if isinstance(m, ToolMessage)] assert [m.tool_call_id for m in outputs] == [f"call_{i}" for i in range(4)] assert [m.status for m in outputs] == ["success", "error", "error", "success"] assert "END_OF_TREE" in str(outputs[0].content) assert "未完成或未执行" in str(outputs[2].content) assert "Action Recorded" not in str(messages) wire = ChatOpenAI(model="fixture", api_key="fixture", use_responses_api=True, store=False) payload = wire._get_request_payload(messages)["input"] assert len([item for item in payload if item.get("type") == "function_call_output"]) == 4 assert str(payload).count(base64.b64encode(image.read_bytes()).decode()) == 3 assert f"data:{Image.MIME[image_format]};base64," in str(payload) # No duplicate outputs or rewritten images on the following ordinary turn. ledger.commit_staged( validator_result=report, tool_results=plugin_result_messages(report, engine) ) later = ledger.render([]) assert len([m for m in later if isinstance(m, ToolMessage)]) == 4 assert str(later).count(base64.b64encode(image.read_bytes()).decode()) == 3 ledger.compact_active() later = ledger.render([]) assert str(later).count(base64.b64encode(image.read_bytes()).decode()) < 3 assert str(messages).count(base64.b64encode(image.read_bytes()).decode()) == 3 def test_missing_plugin_screenshot_keeps_real_result_and_reports_missing_evidence(tmp_path): action = _action("call", post_image_name="missing") messages = plugin_result_messages( {"execution": [action]}, SimpleNamespace(get_image_path=lambda _: tmp_path / "missing.jpg") )["call"] assert "Address bar" in str(messages) assert "截图无法读取" in str(messages) assert not any(isinstance(m, HumanMessage) for m in messages) @pytest.mark.asyncio async def test_validator_preserves_model_call_id_and_actual_plugin_result(monkeypatch): name = "plug_test_get_app_state" plugins = SimpleNamespace( bindings={name: None}, execute=AsyncMock( return_value=ToolExecutionResult( tool_name=name, tool_call_id="original", status="success", text_summary="PC tree", screenshot_bytes=b"pc", metadata={"plugin_id": "test", "plugin_status": "success"}, ) ), ) monkeypatch.setattr("artemis.plugins.runtime.get_plugin_runtime", lambda _: plugins) ctx = ArtemisContext.model_construct() outcome = await _process_action( SimpleNamespace(ctx=ctx), None, None, {"action": name, "args": {"app": "browser"}, "tool_call_id": "original"}, name, "phone", burst=False, ) plugins.execute.assert_awaited_once_with(name, {"app": "browser"}, "original") assert outcome.action_item["tool_call_id"] == "original" assert outcome.action_item["result"] == "PC tree" assert base64.b64decode(outcome.post_screenshot_b64) == b"pc" @pytest.mark.parametrize("live", [False, True]) def test_old_mixed_step_history_returns_pc_post_and_never_phone_pre(tmp_path, live): (tmp_path / "images").mkdir() for name, size in [("phone", (100, 200)), ("pc", (200, 100))]: Image.new("RGB", size).save(tmp_path / "images" / f"{name}.jpg") storage = StorageManager(tmp_path / "data_engine.db", tmp_path) session_id = uuid4() storage.create_session(SessionMetadata(session_id=session_id, initial_goal="fixture")) for name in ("phone", "pc"): storage.create_image(ImageRecord(image_name=name)) storage.create_step( StepRecord( step_id=uuid4(), session_id=session_id, step_number=24, pre_image_name="phone", last_execution_result={"execution": [_action("original", post_image_name="pc")]}, ) ) reader = OfflineHistoryReader(tmp_path / "data_engine.db", tmp_path, session_id) if live: engine = DataEngine.__new__(DataEngine) engine.get_step_record = reader.get_step_record engine.get_image_path = reader.get_image_path reader = engine assert load_step_screenshot(reader, 24, "pre").image_bytes is None assert ( load_step_screenshot(reader, 24, "post").image_bytes == (tmp_path / "images/pc.jpg").read_bytes() ) def test_cross_resource_burst_has_no_single_step_image(): from artemis.data_engine.step_evidence import step_image_name record = { "pre_image_name": "phone", "post_image_name": "pc", "last_execution_result": { "execution": [{"action": "tap"}, _action("pc", post_image_name="pc")] }, } assert step_image_name(record, "pre") is None assert step_image_name(record, "post") is None @pytest.mark.asyncio async def test_interrupted_plugin_call_is_not_reported_as_executed(monkeypatch): from artemis.runtime.task_control import TaskControl name = "plug_test_click" plugins = SimpleNamespace(bindings={name: None}, execute=AsyncMock()) monkeypatch.setattr("artemis.plugins.runtime.get_plugin_runtime", lambda _: plugins) control = TaskControl.__new__(TaskControl) async def interrupt(operation, **_kwargs): operation.close() return True, None control.run_interruptibly = interrupt outcome = await _process_action( SimpleNamespace(ctx=SimpleNamespace(task_control=control)), None, None, {"action": name, "args": {}, "tool_call_id": "interrupted"}, name, "phone", burst=False, ) assert not outcome.success messages = plugin_result_messages({"execution": [outcome.action_item]}, None)["interrupted"] assert messages[0].status == "error" assert "打断" in str(messages[0].content) plugins.execute.assert_not_awaited()