114 lines
4.0 KiB
Python
114 lines
4.0 KiB
Python
"""Mixed-task acceptance must see each resource's actual recorded captures."""
|
|
|
|
import hashlib
|
|
from types import SimpleNamespace
|
|
from unittest.mock import Mock
|
|
|
|
from artemis.agents.checker.evidence import final_evidence
|
|
|
|
|
|
def engine(tmp_path, steps):
|
|
return SimpleNamespace(
|
|
storage=SimpleNamespace(get_steps=lambda _: steps),
|
|
current_session_id="task",
|
|
get_image_path=lambda name: tmp_path / f"{name}.jpg",
|
|
)
|
|
|
|
|
|
def step(n, pre=None, post=None, action=None, execution=None):
|
|
return SimpleNamespace(
|
|
step_number=n,
|
|
timestamp=float(n),
|
|
pre_image_name=pre,
|
|
post_image_name=post,
|
|
action_taken=action or {"action": "swipe"},
|
|
last_execution_result={"execution": execution} if execution else {},
|
|
)
|
|
|
|
|
|
def image(tmp_path, value):
|
|
name = hashlib.sha256(value).hexdigest()
|
|
(tmp_path / f"{name}.jpg").write_bytes(value)
|
|
return name
|
|
|
|
|
|
def pc(post, pre=None, target="browser", width=1280):
|
|
return {
|
|
"action": "plug_scroll",
|
|
"plugin_id": "cua-driver",
|
|
"source": {
|
|
"kind": "computer",
|
|
"target_id": target,
|
|
"width": width,
|
|
"captured_at": 7,
|
|
},
|
|
"pre_image_name": pre,
|
|
"post_image_name": post,
|
|
"pre_image_valid": bool(pre),
|
|
}
|
|
|
|
|
|
def test_phone_and_unchanged_pc_captures_are_both_sent_with_origin(tmp_path):
|
|
phone = image(tmp_path, b"phone")
|
|
browser = image(tmp_path, b"pc")
|
|
# Old records deduplicated the step's post ref; per-call receipt still has it.
|
|
steps = [
|
|
step(15, post=phone),
|
|
step(27, action=pc(browser, browser)),
|
|
step(28, action=pc(browser, browser)),
|
|
]
|
|
blocks = final_evidence(engine(tmp_path, steps))
|
|
assert sum(b["type"] == "image_url" for b in blocks) == 2
|
|
text = str(blocks)
|
|
assert "android" in text and "computer" in text and "cua-driver" in text
|
|
assert "Step 27" in text and "Step 28" in text
|
|
assert "identical to pre-capture" in text
|
|
assert "not proof of success" in text
|
|
|
|
|
|
def test_native_resolution_change_does_not_hide_previous_same_window_capture(tmp_path):
|
|
a = image(tmp_path, b"native")
|
|
b = image(tmp_path, b"scaled")
|
|
blocks = final_evidence(
|
|
engine(tmp_path, [step(1, action=pc(a, width=2560)), step(2, action=pc(b))]), max_sources=1
|
|
)
|
|
assert sum(x["type"] == "image_url" for x in blocks) == 2
|
|
assert "Step 1" in str(blocks) and "Step 2" in str(blocks)
|
|
|
|
|
|
def test_burst_uses_per_call_sources_and_never_phone_step_fallback(tmp_path):
|
|
a = image(tmp_path, b"A")
|
|
b = image(tmp_path, b"B")
|
|
blocks = final_evidence(
|
|
engine(tmp_path, [step(4, post="phone", execution=[pc(a, target="A"), pc(b, target="B")])])
|
|
)
|
|
assert sum(x["type"] == "image_url" for x in blocks) == 2
|
|
assert "android" not in str(blocks)
|
|
|
|
|
|
def test_missing_images_and_untrusted_paths_are_not_fabricated(tmp_path):
|
|
e = engine(
|
|
tmp_path, [step(1, action=pc("a" * 64)), step(2, action=pc("../private", target="B"))]
|
|
)
|
|
e.get_image_path = Mock(side_effect=e.get_image_path)
|
|
blocks = final_evidence(e)
|
|
assert "unavailable" in str(blocks)
|
|
assert not any(x["type"] == "image_url" for x in blocks)
|
|
e.get_image_path.assert_called_once_with("a" * 64)
|
|
|
|
|
|
def test_last_sources_are_selected_by_execution_order_not_string_step_number(tmp_path):
|
|
name = image(tmp_path, b"frame")
|
|
steps = [step(9, action=pc(name, target="old")), step(10, action=pc(name, target="new"))]
|
|
blocks = final_evidence(engine(tmp_path, steps), max_sources=1)
|
|
assert "Step 10" in str(blocks) and "Step 9" not in str(blocks)
|
|
assert "Additional resources omitted" in str(blocks)
|
|
|
|
|
|
def test_later_call_without_observation_marks_previous_picture_historical(tmp_path):
|
|
name = image(tmp_path, b"old")
|
|
blocks = final_evidence(engine(tmp_path, [step(1, action=pc(name)), step(2, action=pc(None))]))
|
|
assert "Step 2" in str(blocks) and "no post-action image" in str(blocks)
|
|
assert "Earlier captures do not prove" in str(blocks)
|
|
assert sum(x["type"] == "image_url" for x in blocks) == 1
|