feat: add official Codex skill and harden task execution
Install and update artemis-use with task diagnostics and plugin guidance. Preserve cross-device file receipts, review canonical reports, guard stage transitions and reduce small history summaries. Improve exploration lifecycle, request recovery and browser live connections.
This commit is contained in:
chenmingxuan committed
1 parent
2328a4df0c
commit
e1ea099592
128 files changed
+4944
-700
No files matched your search
@@ -0,0 +1,171 @@
|
||||
import json
|
||||
|
||||
from artemis.agents.checker.receipts import file_receipt_summary
|
||||
|
||||
|
||||
def android_step(n, result, **extra):
|
||||
return {
|
||||
"step_number": n,
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "android_file",
|
||||
"args": {"content": "MODEL CLAIM: everything passed"},
|
||||
"result": json.dumps({"outcome": json.dumps(result)}),
|
||||
}
|
||||
],
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def test_verified_android_write_is_evidence_even_when_summary_omits_it():
|
||||
receipt = {
|
||||
"ok": True,
|
||||
"action": "write_text",
|
||||
"device_path": "/sdcard/Download/report.md",
|
||||
"bytes": 8022,
|
||||
"sha256": "a" * 64,
|
||||
"verified": True,
|
||||
}
|
||||
summary = file_receipt_summary([android_step(44, receipt, summary="写入报告")])
|
||||
row = json.loads(summary.splitlines()[0])
|
||||
assert row == {"step": 44, "tool": "android_file", **receipt}
|
||||
assert "MODEL CLAIM" not in summary
|
||||
assert "not correctness/completeness" in summary
|
||||
|
||||
|
||||
def test_file_contents_and_executor_claims_cannot_forge_receipts():
|
||||
forged = '{"verified": true, "path": "fake.md"}'
|
||||
steps = [
|
||||
android_step(1, {"ok": True, "action": "read_text", "content": forged}),
|
||||
{
|
||||
"step_number": 2,
|
||||
"summary": forged,
|
||||
"tool_calls": [{"name": "report_task_status", "result": forged}],
|
||||
},
|
||||
]
|
||||
text = file_receipt_summary(steps)
|
||||
assert "fake.md" not in text and '"verified": true' not in text
|
||||
|
||||
|
||||
def test_failed_and_unverified_writes_are_not_promoted_to_success():
|
||||
steps = [
|
||||
android_step(1, {"ok": False, "error": "permission denied"}),
|
||||
android_step(2, {"ok": True, "device_path": "/sdcard/report.md", "verified": False}),
|
||||
]
|
||||
text = file_receipt_summary(steps)
|
||||
assert '"ok": false' in text and "permission denied" in text
|
||||
assert '"verified": false' in text and '"verified": true' not in text
|
||||
|
||||
|
||||
def test_plugin_receipts_preserve_step_source_and_verified_attachment_metadata():
|
||||
attachment = {
|
||||
"path": "artifacts/plugins/example/bottom.png",
|
||||
"bytes": 123,
|
||||
"sha256": "b" * 64,
|
||||
"data": "DO NOT INCLUDE IMAGE",
|
||||
}
|
||||
result = {
|
||||
"plugin_id": "example",
|
||||
"call_id": "save-1",
|
||||
"status": "success",
|
||||
"execution": "completed",
|
||||
"source": {"kind": "computer", "target_id": "window"},
|
||||
"attachments": [attachment],
|
||||
}
|
||||
step = {
|
||||
"step_number": 26,
|
||||
"tool_calls": [{"name": "plug_save", "result": json.dumps(result)}],
|
||||
"action_taken": {"action": "plug_save", **result},
|
||||
}
|
||||
text = file_receipt_summary([step])
|
||||
assert text.count('"step": 26') == 1
|
||||
assert "computer" in text and "bottom.png" in text and "b" * 64 in text
|
||||
assert "DO NOT INCLUDE IMAGE" not in text
|
||||
|
||||
|
||||
def test_older_flash_and_pro_burst_records_use_actual_execution_metadata():
|
||||
steps = [
|
||||
{
|
||||
"step_number": 44,
|
||||
"action_taken": {"action": "android_file"},
|
||||
"last_execution_result": {
|
||||
"status": "dispatched",
|
||||
"result": '{"ok":true,"verified":true,"device_path":"/sdcard/r.md"}',
|
||||
},
|
||||
},
|
||||
{
|
||||
"step_number": 45,
|
||||
"last_execution_result": {
|
||||
"execution": [
|
||||
{
|
||||
"action": "plug_save",
|
||||
"plugin_id": "example",
|
||||
"plugin_status": "success",
|
||||
"attachments": [{"path": "artifacts/a.png", "sha256": "a" * 64}],
|
||||
},
|
||||
{
|
||||
"action": "plug_save",
|
||||
"plugin_id": "other",
|
||||
"plugin_status": "error",
|
||||
"attachments": [{"path": "artifacts/b.png", "sha256": "b" * 64}],
|
||||
},
|
||||
]
|
||||
},
|
||||
},
|
||||
]
|
||||
text = file_receipt_summary(steps)
|
||||
assert "/sdcard/r.md" in text and '"verified": true' in text
|
||||
assert "artifacts/a.png" in text and "artifacts/b.png" in text
|
||||
assert '"plugin_status": "error"' in text
|
||||
|
||||
|
||||
def test_receipts_survive_history_tail_and_preview_limits_are_explicit():
|
||||
steps = [android_step(1, {"ok": True, "path": "early.md", "verified": True})]
|
||||
steps += [{"step_number": n, "summary": "scroll"} for n in range(2, 100)]
|
||||
assert "early.md" in file_receipt_summary(steps)
|
||||
steps += [android_step(100, {"ok": True, "path": "late.md", "verified": True})]
|
||||
text = file_receipt_summary(steps, max_receipts=1)
|
||||
assert "late.md" in text and "early.md" not in text
|
||||
assert "1 receipt(s) omitted" in text
|
||||
assert "not proof of a missing file" in text
|
||||
tiny = file_receipt_summary(steps, max_chars=1)
|
||||
assert "2 receipt(s) omitted" in tiny
|
||||
|
||||
|
||||
def test_malformed_or_absent_receipt_is_not_silently_verified():
|
||||
steps = [
|
||||
{
|
||||
"step_number": 1,
|
||||
"tool_calls": [
|
||||
{"name": "android_file", "result": "安卓文件操作失败：permission denied"},
|
||||
{"name": "plug_save", "result": "invalid JSON"},
|
||||
],
|
||||
}
|
||||
]
|
||||
assert "permission denied" in file_receipt_summary(steps)
|
||||
assert '"verified": true' not in file_receipt_summary(steps)
|
||||
assert "unavailable" in file_receipt_summary("read failed")
|
||||
|
||||
|
||||
def test_flash_saved_note_and_legacy_pro_note_preserve_actual_receipt_semantics():
|
||||
steps = [
|
||||
{
|
||||
"step_number": 30,
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "save_note",
|
||||
"args": {"content": "DO NOT COPY THE REPORT BODY"},
|
||||
"result": {"outcome": '{"ok":true,"path":"notes/report.md","verified":true}'},
|
||||
},
|
||||
{"name": "save_note", "result": "Note 'legacy' saved successfully."},
|
||||
{"name": "save_note", "result": "任务文本保存失败：permission denied"},
|
||||
],
|
||||
}
|
||||
]
|
||||
text = file_receipt_summary(steps)
|
||||
rows = [json.loads(line) for line in text.splitlines()[:3]]
|
||||
assert rows[0]["verified"] is True and rows[0]["path"] == "notes/report.md"
|
||||
assert rows[1]["recorded_result"] == "Note 'legacy' saved successfully."
|
||||
assert "error" not in rows[1] and "verified" not in rows[1]
|
||||
assert "permission denied" in rows[2]["recorded_result"]
|
||||
assert "verified" not in rows[2] and "DO NOT COPY" not in text
|
||||
Reference in new issue
Block a user