731 lines
31 KiB
Python
731 lines
31 KiB
Python
"""Automation acceptance checks with isolated persistence and no real device side effects."""
|
||
|
||
from io import BytesIO
|
||
import json
|
||
import sqlite3
|
||
import threading
|
||
import time
|
||
from unittest.mock import AsyncMock, Mock
|
||
from uuid import UUID
|
||
import zipfile
|
||
|
||
from docx import Document
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
from openpyxl import load_workbook
|
||
import pytest
|
||
|
||
from apps.admin_console.routers import automation as api
|
||
from apps.admin_console.services import automation_service as service
|
||
from apps.admin_console.services import report_export as reports
|
||
from apps.admin_console.services import task_import
|
||
from apps.admin_console.services.automation_store import AutomationStore
|
||
from artemis.data_engine.storage import StorageManager
|
||
|
||
|
||
@pytest.fixture
|
||
def isolated(tmp_path, monkeypatch):
|
||
db = tmp_path / "data.db"
|
||
StorageManager(db_path=db, base_trace_dir=tmp_path)
|
||
store = AutomationStore(db)
|
||
for module in (api, service, reports):
|
||
monkeypatch.setattr(module, "store", store)
|
||
monkeypatch.setattr(reports, "TRACES_PATH", tmp_path / "traces")
|
||
monkeypatch.setattr(reports, "EXPORT_ROOT", tmp_path / "exports")
|
||
from apps.admin_console.database.repositories.session_repository import SessionRepository
|
||
from apps.admin_console.database.repositories.step_repository import StepRepository
|
||
|
||
monkeypatch.setattr(reports, "SessionRepository", lambda *a: SessionRepository(db))
|
||
monkeypatch.setattr(reports, "StepRepository", lambda *a: StepRepository(db))
|
||
return store
|
||
|
||
|
||
@pytest.fixture
|
||
def client(isolated):
|
||
app = FastAPI()
|
||
app.include_router(api.router)
|
||
return TestClient(app)
|
||
|
||
|
||
def payload(**extra):
|
||
return {
|
||
"name": "批次",
|
||
"cases": [{"name": "用例甲", "goal": "观察设置"}, {"name": "用例乙", "goal": "观察首页"}],
|
||
"device_serial": "test-device",
|
||
**extra,
|
||
}
|
||
|
||
|
||
def workbook_bytes(rows):
|
||
wb = load_workbook(BytesIO(task_import.template()))
|
||
for row in rows:
|
||
wb["任务"].append(row)
|
||
output = BytesIO()
|
||
wb.save(output)
|
||
return output.getvalue()
|
||
|
||
|
||
def test_template_preview_rejects_formula_and_bad_json():
|
||
data = workbook_bytes(
|
||
[
|
||
["正常", "设置", "冒烟", "打开设置", "", "观察设置", '{"profile":"flash"}'],
|
||
["公式", "", "", "", "", "=SUM(1,2)"],
|
||
["参数错误", "", "", "", "", "测试", "{"],
|
||
]
|
||
)
|
||
result = task_import.parse_workbook(data)
|
||
assert len(result["tasks"]) == 1
|
||
assert [e["row"] for e in result["errors"]] == [3, 4]
|
||
assert not result["valid"]
|
||
|
||
|
||
def test_prompt_only_and_per_row_config_validation():
|
||
goal = "先查看屏幕，再按实际画面判断。"
|
||
data = workbook_bytes(
|
||
[
|
||
[
|
||
"CASE-7",
|
||
"检查项目",
|
||
"子项",
|
||
"用例名称",
|
||
"仅作索引的约束",
|
||
goal,
|
||
'{"profile":"pro","model":"mimo-v2.6-flash","record_video":true}',
|
||
],
|
||
["CASE-8", "", "", "", "", "观察屏幕", '{"OPENAI_API_KEY":"never"}'],
|
||
]
|
||
)
|
||
preview = task_import.parse_workbook(data)
|
||
assert [e["row"] for e in preview["errors"]] == [3]
|
||
case = preview["tasks"][0]
|
||
assert task_import.case_goal(case) == goal
|
||
assert case["preconditions"] == "仅作索引的约束"
|
||
assert case["task_config"]["record_video"] is True
|
||
|
||
|
||
def test_invalid_archive_and_empty_template():
|
||
with pytest.raises(ValueError):
|
||
task_import.parse_workbook(b"not a zip")
|
||
assert not task_import.parse_workbook(task_import.template())["valid"]
|
||
|
||
|
||
def test_batch_archive_preparation_reports_progress_and_reuses_ready_file(
|
||
client, isolated, monkeypatch
|
||
):
|
||
batch = service.create_batch(payload())
|
||
batch_id = batch["id"]
|
||
started = threading.Event()
|
||
release = threading.Event()
|
||
calls = []
|
||
|
||
def slow_export(identifier, progress=None):
|
||
calls.append(identifier)
|
||
progress(0, 2, "正在打包任务 1/2")
|
||
started.set()
|
||
assert release.wait(5)
|
||
progress(2, 2, "正在完成压缩包")
|
||
path = reports.EXPORT_ROOT / "batches" / identifier / "批次报告.zip"
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_bytes(b"prepared archive")
|
||
return path
|
||
|
||
monkeypatch.setattr(reports, "export_batch", slow_export)
|
||
base = f"/api/automation/batches/{batch_id}/archive"
|
||
try:
|
||
response = client.post(base + "/prepare")
|
||
assert response.status_code == 200
|
||
assert response.json()["status"] == "preparing"
|
||
assert started.wait(2)
|
||
progress = client.get(base + "/status").json()
|
||
assert progress["total"] == 2
|
||
assert progress["stage"] == "正在打包任务 1/2"
|
||
assert client.get(base).status_code == 409
|
||
finally:
|
||
release.set()
|
||
for _ in range(50):
|
||
if client.get(base + "/status").json()["status"] == "ready":
|
||
break
|
||
time.sleep(0.02)
|
||
assert client.get(base + "/status").json()["status"] == "ready"
|
||
assert client.get(base).content == b"prepared archive"
|
||
assert calls == [batch_id]
|
||
|
||
|
||
def test_completed_batch_reuses_existing_archive(client, isolated, monkeypatch):
|
||
batch = service.create_batch(payload())
|
||
batch_id = batch["id"]
|
||
isolated.put("batch", batch_id, {**batch, "status": "completed", "report_ready": True})
|
||
path = reports.EXPORT_ROOT / "batches" / batch_id / "批次报告.zip"
|
||
path.parent.mkdir(parents=True)
|
||
path.write_bytes(b"existing archive")
|
||
exporter = Mock(side_effect=AssertionError("archive must not be rebuilt"))
|
||
monkeypatch.setattr(reports, "export_batch", exporter)
|
||
base = f"/api/automation/batches/{batch_id}/archive"
|
||
assert client.post(base + "/prepare").json()["status"] == "ready"
|
||
assert client.get(base).content == b"existing archive"
|
||
exporter.assert_not_called()
|
||
|
||
|
||
def test_preview_does_not_enqueue(client, isolated):
|
||
response = client.post(
|
||
"/api/automation/import/preview",
|
||
files={
|
||
"file": ("tasks.xlsx", workbook_bytes([["案例", "", "", "观察页面", "", "观察页面"]]))
|
||
},
|
||
)
|
||
assert response.status_code == 200
|
||
assert response.json()["valid"]
|
||
assert not isolated.jobs()
|
||
|
||
|
||
def test_preferences_and_required_device(client):
|
||
assert client.get("/api/automation/preferences").json()["formats"] == ["docx", "xlsx"]
|
||
assert client.put("/api/automation/preferences", json={"formats": ["xlsx"]}).status_code == 200
|
||
assert client.put("/api/automation/preferences", json={"formats": []}).status_code == 422
|
||
assert client.post("/api/automation/batches", json=payload(device_serial="")).status_code == 422
|
||
|
||
|
||
def test_batch_delete_removes_tasks_and_files_and_prevents_request_replay(client, isolated):
|
||
created = client.post(
|
||
"/api/automation/batches", json=payload(request_id="delete-example")
|
||
).json()
|
||
first, second = isolated.jobs(created["id"])
|
||
isolated.save_job({**first, "status": "completed"})
|
||
trace_dir = reports.TRACES_PATH / first["session_id"]
|
||
trace_dir.mkdir(parents=True)
|
||
(trace_dir / "result.json").write_text("{}", encoding="utf-8")
|
||
report_dir = reports.EXPORT_ROOT / "sessions" / first["session_id"]
|
||
report_dir.mkdir(parents=True)
|
||
(report_dir / "任务报告.docx").write_bytes(b"report")
|
||
batch_dir = reports.EXPORT_ROOT / "batches" / created["id"]
|
||
batch_dir.mkdir(parents=True)
|
||
(batch_dir / "批次报告.zip").write_bytes(b"zip")
|
||
mcp_task_dir = reports.EXPORT_ROOT.parent / "mcp_exports" / "sessions" / first["session_id"]
|
||
mcp_task_dir.mkdir(parents=True)
|
||
(mcp_task_dir / "报告.docx").write_bytes(b"report")
|
||
mcp_batch_dir = reports.EXPORT_ROOT.parent / "mcp_exports" / "batches" / created["id"]
|
||
mcp_batch_dir.mkdir(parents=True)
|
||
(mcp_batch_dir / "批次.zip").write_bytes(b"zip")
|
||
unrelated = reports.TRACES_PATH / "unrelated-task"
|
||
unrelated.mkdir()
|
||
(unrelated / "keep.txt").write_text("keep", encoding="utf-8")
|
||
|
||
response = client.delete(f"/api/automation/batches/{created['id']}")
|
||
assert response.status_code == 200
|
||
assert client.get(f"/api/automation/batches/{created['id']}").status_code == 404
|
||
assert isolated.job(first["session_id"]) is None
|
||
assert isolated.job(second["session_id"]) is None
|
||
assert not trace_dir.exists()
|
||
assert not report_dir.exists()
|
||
assert not batch_dir.exists()
|
||
assert not mcp_task_dir.exists()
|
||
assert not mcp_batch_dir.exists()
|
||
assert (unrelated / "keep.txt").read_text(encoding="utf-8") == "keep"
|
||
with pytest.raises(ValueError, match="不存在"):
|
||
reports.session_report(first["session_id"])
|
||
assert (
|
||
client.post(
|
||
"/api/automation/batches", json=payload(request_id="delete-example")
|
||
).status_code
|
||
== 409
|
||
)
|
||
|
||
|
||
@pytest.mark.parametrize("status", ["queued", "running", "task_paused", "resuming", "finalizing"])
|
||
def test_batch_delete_rejects_active_jobs(client, isolated, status):
|
||
batch = service.create_batch(payload())
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": status})
|
||
|
||
response = client.delete(f"/api/automation/batches/{batch['id']}")
|
||
assert response.status_code == 409
|
||
assert isolated.get("batch", batch["id"]) is not None
|
||
assert len(isolated.jobs(batch["id"])) == 2
|
||
|
||
|
||
def test_batch_deletion_reservation_blocks_new_dispatch(isolated):
|
||
batch = service.create_batch(payload())
|
||
job = isolated.jobs(batch["id"])[0]
|
||
|
||
isolated.begin_delete_batch(batch["id"])
|
||
|
||
with pytest.raises(ValueError, match="正在删除"):
|
||
isolated.save_job({**job, "status": "queued"})
|
||
with pytest.raises(ValueError, match="正在删除"):
|
||
isolated.put("batch", batch["id"], {**batch, "status": "running"})
|
||
assert isolated.jobs_for_sessions([job["session_id"]])[job["session_id"]]["status"] == "pending"
|
||
|
||
|
||
def test_batch_deletion_preserves_shared_screenshot_until_last_task(client, isolated):
|
||
first_batch = service.create_batch(payload())
|
||
second_batch = service.create_batch(payload(name="另一批次"))
|
||
first = isolated.jobs(first_batch["id"])[0]
|
||
other = isolated.jobs(second_batch["id"])[0]
|
||
shared_hash = "a" * 64
|
||
storage = StorageManager(isolated.path, reports.TRACES_PATH)
|
||
image_file = reports.TRACES_PATH / "images" / f"{shared_hash}.jpg"
|
||
image_file.parent.mkdir(parents=True)
|
||
image_file.write_bytes(b"shared image")
|
||
with sqlite3.connect(isolated.path) as conn:
|
||
conn.execute("INSERT INTO images(image_name) VALUES (?)", (shared_hash,))
|
||
for job in (first, other):
|
||
conn.execute(
|
||
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info) "
|
||
"VALUES (?,?,?,?,?)",
|
||
(job["session_id"], "观察设置", 1.0, "completed", "{}"),
|
||
)
|
||
conn.execute(
|
||
"INSERT INTO steps(step_id,session_id,pre_image_name) VALUES (?,?,?)",
|
||
(job["session_id"], job["session_id"], shared_hash),
|
||
)
|
||
|
||
assert client.delete(f"/api/automation/batches/{first_batch['id']}").status_code == 200
|
||
assert image_file.read_bytes() == b"shared image"
|
||
assert storage.get_session(UUID(other["session_id"])) is not None
|
||
assert client.delete(f"/api/automation/batches/{second_batch['id']}").status_code == 200
|
||
assert not image_file.exists()
|
||
|
||
|
||
def test_restart_never_repeats_started_work(isolated):
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**job, "status": "queued"})
|
||
service.recover_batches()
|
||
assert isolated.get("batch", batch["id"])["status"] == "paused"
|
||
assert [j["status"] for j in isolated.jobs(batch["id"])] == ["interrupted", "pending"]
|
||
retry = service.batch_action(batch["id"], "retry_failed")
|
||
assert retry["status"] == "draft"
|
||
assert len(isolated.jobs(retry["id"])) == 1
|
||
assert isolated.jobs(retry["id"])[0]["session_id"] != job["session_id"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_batch_admission_idempotent_and_bound_device(isolated, monkeypatch):
|
||
from apps.admin_console import server
|
||
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
||
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
||
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.validate_explicit_serial_async", AsyncMock(return_value=None)
|
||
)
|
||
from artemis.runtime.device_pool import DeviceStatus
|
||
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async",
|
||
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
||
)
|
||
await service.tick_batches()
|
||
await service.tick_batches()
|
||
assert enqueue.await_count == 1
|
||
assert enqueue.call_args.kwargs["device_serial"] == "test-device"
|
||
assert isolated.jobs(batch["id"])[1]["status"] == "pending"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_paused_case_yields_batch_slot_to_next_case(isolated, monkeypatch):
|
||
from apps.admin_console import server
|
||
from artemis.runtime.device_pool import DeviceStatus
|
||
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": "task_paused", "started_at": time.time()})
|
||
monkeypatch.setattr(
|
||
service, "state", type("State", (), {"is_paused": False, "active_runs": {}})()
|
||
)
|
||
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
||
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async",
|
||
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
||
)
|
||
await service.tick_batches()
|
||
assert enqueue.await_count == 1
|
||
assert enqueue.call_args.args[0] == ["观察首页"]
|
||
assert [job["status"] for job in isolated.jobs(batch["id"])] == ["task_paused", "queued"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_device_outage_pauses_without_consuming_pending_case(isolated, monkeypatch):
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[])
|
||
)
|
||
await service.tick_batches()
|
||
assert isolated.get("batch", batch["id"])["status"] == "paused"
|
||
assert all(job["status"] == "pending" for job in isolated.jobs(batch["id"]))
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fail_fast_includes_skipped_reports(isolated, monkeypatch):
|
||
batch = service.create_batch(payload(stop_on_failure=True))
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": "failed"})
|
||
service.batch_action(batch["id"], "resume")
|
||
await service.tick_batches()
|
||
assert [j["status"] for j in isolated.jobs(batch["id"])] == ["failed", "skipped"]
|
||
archive = reports.export_batch(batch["id"])
|
||
with zipfile.ZipFile(archive) as z:
|
||
assert len(z.namelist()) == 6 # two summary files + two files per task
|
||
assert sum(n.endswith(".docx") for n in z.namelist()) == 3
|
||
|
||
|
||
def test_batch_archive_contains_each_task_media_when_present(isolated, monkeypatch, tmp_path):
|
||
batch = service.create_batch(payload())
|
||
first, second = isolated.jobs(batch["id"])
|
||
image1 = tmp_path / "first.png"
|
||
image2 = tmp_path / "second.png"
|
||
video = tmp_path / "recording.mp4"
|
||
image1.write_bytes(b"first screenshot")
|
||
image2.write_bytes(b"second screenshot")
|
||
video.write_bytes(b"saved recording")
|
||
images = {first["session_id"]: image1, second["session_id"]: image2}
|
||
monkeypatch.setattr(
|
||
reports,
|
||
"session_screenshots",
|
||
lambda report: [(images[report["id"]], "0001_步骤0001_操作前.png")],
|
||
)
|
||
monkeypatch.setattr(
|
||
reports,
|
||
"session_videos",
|
||
lambda report: (
|
||
[(video, "0001_recording.mp4")] if report["id"] == first["session_id"] else []
|
||
),
|
||
)
|
||
|
||
archive = reports.export_batch(batch["id"])
|
||
with zipfile.ZipFile(archive) as z:
|
||
names = z.namelist()
|
||
assert len([name for name in names if "/步骤截图/" in name]) == 2
|
||
assert len([name for name in names if "/任务录屏/" in name]) == 1
|
||
assert all(name.startswith("任务报告/") for name in names if "步骤截图" in name)
|
||
assert z.read(next(name for name in names if "/任务录屏/" in name)) == b"saved recording"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_report_failure_does_not_strand_job(isolated, monkeypatch):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
monkeypatch.setattr(
|
||
reports, "export_session", lambda *a: (_ for _ in ()).throw(OSError("disk full"))
|
||
)
|
||
await service.on_terminal({"session_id": job["session_id"]}, "completed")
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "completed"
|
||
assert "disk full" in saved["report_error"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_batch_timeout_stays_visible_in_job_report_and_history(client, isolated):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
isolated.save_job(
|
||
{**job, "status": "running", "timed_out": True, "error": "任务执行超时（30 秒）"}
|
||
)
|
||
await service.on_terminal({"session_id": job["session_id"]}, "cancelled")
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "failed"
|
||
assert "超时" in saved["error"]
|
||
assert "超时" in reports.session_report(job["session_id"])["error"]
|
||
history = client.get("/api/automation/history").json()["items"]
|
||
assert any(
|
||
item["session_id"] == job["session_id"] and "超时" in item["error"] for item in history
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
@pytest.mark.parametrize("stop_on_failure", [False, True])
|
||
async def test_timed_out_case_releases_next_batch_case(isolated, monkeypatch, stop_on_failure):
|
||
from apps.admin_console import server
|
||
from artemis.runtime.device_pool import DeviceStatus
|
||
|
||
batch = service.create_batch(payload(timeout_seconds=1, stop_on_failure=stop_on_failure))
|
||
service.batch_action(batch["id"], "resume")
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": "running", "started_at": time.time() - 2})
|
||
monkeypatch.setattr(
|
||
service, "state", type("State", (), {"is_paused": False, "active_runs": {}})()
|
||
)
|
||
stop = Mock(return_value=True)
|
||
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
||
monkeypatch.setattr(server.task_queue_service, "stop_tasks", stop)
|
||
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async",
|
||
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
||
)
|
||
monkeypatch.setattr(reports, "export_batch", Mock())
|
||
|
||
await service.tick_batches()
|
||
assert isolated.get("batch", batch["id"])["status"] == "running"
|
||
assert "超时" in isolated.job(first["session_id"])["error"]
|
||
stop.assert_called_once_with(session_id=first["session_id"])
|
||
await service.tick_batches()
|
||
stop.assert_called_once()
|
||
|
||
await service.on_terminal({"session_id": first["session_id"]}, "cancelled")
|
||
await service.tick_batches()
|
||
statuses = [job["status"] for job in isolated.jobs(batch["id"])]
|
||
assert statuses == (["failed", "skipped"] if stop_on_failure else ["failed", "queued"])
|
||
assert enqueue.await_count == (0 if stop_on_failure else 1)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_queued_case_does_not_consume_execution_timeout(isolated, monkeypatch):
|
||
from apps.admin_console import server
|
||
|
||
batch = service.create_batch(payload(timeout_seconds=1))
|
||
service.batch_action(batch["id"], "resume")
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": "queued", "started_at": time.time() - 30})
|
||
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
||
stop = Mock()
|
||
monkeypatch.setattr(server.task_queue_service, "stop_tasks", stop)
|
||
|
||
await service.tick_batches()
|
||
stop.assert_not_called()
|
||
assert isolated.job(first["session_id"])["status"] == "queued"
|
||
assert isolated.get("batch", batch["id"])["status"] == "running"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_model_timeout_from_runner_log_reaches_failed_task_and_report(isolated):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
folder = reports.TRACES_PATH / job["session_id"]
|
||
folder.mkdir(parents=True)
|
||
(folder / "stderr.log").write_text(
|
||
"TimeoutError: model request timed out after 300 seconds\n", encoding="utf-8"
|
||
)
|
||
|
||
await service.on_terminal({"session_id": job["session_id"]}, "failed")
|
||
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "failed"
|
||
assert "超时" in saved["error"]
|
||
assert reports.session_report(job["session_id"])["status"] == "failed"
|
||
assert "超时" in reports.session_report(job["session_id"])["error"]
|
||
|
||
|
||
def test_optional_timeout_warnings_are_not_reported_as_task_failure(tmp_path):
|
||
(tmp_path / "stdout.log").write_text(
|
||
"⚠ Could not clean unowned screen wake locks on emulator-5554: "
|
||
"Timed out acquiring the Artemis awake-lease mutex.\n"
|
||
"⚠ VisualStepSummarizer: Error generating summary for step 3: \n"
|
||
"ℹ StepMemoryService: Retrying Step 3 (attempt 4) after 1.0s\n",
|
||
encoding="utf-8",
|
||
)
|
||
assert reports.failure_reason(tmp_path, "failed", None) == (
|
||
"任务未完成，运行日志中没有可提取的具体异常；请查看执行步骤。"
|
||
)
|
||
|
||
(tmp_path / "stderr.log").write_text(
|
||
"TimeoutError: model request timed out after 300 seconds\n", encoding="utf-8"
|
||
)
|
||
reason = reports.failure_reason(tmp_path, "failed", None)
|
||
assert "model request timed out" in reason
|
||
assert "awake-lease" not in reason
|
||
|
||
(tmp_path / "stderr.log").write_text(
|
||
"DeviceBusyError: device queue reservation was cancelled\n", encoding="utf-8"
|
||
)
|
||
assert reports.failure_reason(tmp_path, "failed", None) == (
|
||
"DeviceBusyError: device queue reservation was cancelled"
|
||
)
|
||
|
||
|
||
def test_reports_do_not_execute_formulas_or_invent_pass(isolated):
|
||
batch = service.create_batch(payload())
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job(
|
||
{**job, "case": {"name": "=1+1", "goal": '=HYPERLINK("x")'}, "status": "completed"}
|
||
)
|
||
report = reports.session_report(job["session_id"])
|
||
assert report["verdict"].startswith("未验证")
|
||
assert reports.verdict({"tests": {"passed": 2, "failed": 1}}) == "验证失败"
|
||
assert reports.verdict({"tests": {"passed": 2, "unchecked": 1}}) == "验证不完整"
|
||
paths = reports.export_session(job["session_id"])
|
||
wb = load_workbook(next(p for p in paths if p.suffix == ".xlsx"))
|
||
assert wb["任务结果"]["B3"].data_type == "s"
|
||
with zipfile.ZipFile(next(p for p in paths if p.suffix == ".docx")) as doc:
|
||
assert "未验证".encode() in doc.read("word/document.xml")
|
||
|
||
|
||
def test_failed_report_exposes_logged_reason_and_formats_chinese_markdown(isolated):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
isolated.save_job({**job, "status": "failed"})
|
||
folder = reports.TRACES_PATH / job["session_id"]
|
||
(folder / "notes").mkdir(parents=True)
|
||
(folder / "stdout.log").write_text(
|
||
"❌ [run] Error running automation: 模型连续 3 次未提供可执行的工具调用\n",
|
||
encoding="utf-8",
|
||
)
|
||
(folder / "notes" / "output.md").write_text(
|
||
"## 结果\n- **网络**正常\n\n| 分类 | 状态 |\n| --- | --- |\n| 系统 | 未确认 |",
|
||
encoding="utf-8",
|
||
)
|
||
report = reports.session_report(job["session_id"])
|
||
assert "连续 3 次" in report["error"]
|
||
assert report["verdict"] == "未验证（任务异常终止）"
|
||
assert "abc123secret" not in reports.failure_reason(
|
||
folder, "failed", "认证失败 OPENAI_API_KEY=abc123secret Authorization: Bearer abc123secret"
|
||
)
|
||
assert "超时" not in reports.failure_reason(folder, "failed", "LLMExhaustedError: 额度不足")
|
||
paths = reports.export_session(job["session_id"])
|
||
doc = Document(next(path for path in paths if path.suffix == ".docx"))
|
||
assert doc.styles["Normal"].font.name == "Microsoft YaHei"
|
||
assert any("异常原因" in paragraph.text for paragraph in doc.paragraphs)
|
||
assert any(
|
||
run.bold and "网络" in run.text for paragraph in doc.paragraphs for run in paragraph.runs
|
||
)
|
||
assert any(
|
||
cell.text == "分类" for table in doc.tables for row in table.rows for cell in row.cells
|
||
)
|
||
wb = load_workbook(next(path for path in paths if path.suffix == ".xlsx"))
|
||
assert "连续 3 次" in wb["任务结果"]["B15"].value
|
||
assert wb["结论"]["B2"].value == "结果"
|
||
assert wb["结果表1"]["A2"].value == "系统"
|
||
assert wb["执行证据"]["A1"].font.name == "微软雅黑"
|
||
assert wb["执行证据"].page_setup.orientation == "landscape"
|
||
assert reports.execution_result_label({"status": "dispatched"}) == "动作已发出（需核对截图）"
|
||
assert reports.conclusion_rows("## 结果\n- **通过**检查\n```txt\n| 不是 | 表格 |\n```") == [
|
||
("类型", "结论与发现"),
|
||
("小节", "结果"),
|
||
("要点", "通过检查"),
|
||
("代码", "| 不是 | 表格 |"),
|
||
]
|
||
|
||
|
||
def test_task_material_zip_contains_ordered_evidence_and_reports(
|
||
client, isolated, tmp_path, monkeypatch
|
||
):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
isolated.save_job({**job, "case": {**job["case"], "case_id": "1.1"}})
|
||
evidence = tmp_path / "evidence"
|
||
evidence.mkdir()
|
||
first = evidence / "first.png"
|
||
second = evidence / "second.png"
|
||
video = evidence / "recording.mp4"
|
||
for path in (first, second, video):
|
||
path.write_bytes(path.name.encode())
|
||
monkeypatch.setattr(
|
||
reports,
|
||
"session_screenshots",
|
||
lambda report: [(first, "0001_步骤0001_操作前.png"), (second, "0002_步骤0002_操作后.png")],
|
||
)
|
||
monkeypatch.setattr(reports, "session_videos", lambda report: [(video, "0001_recording.mp4")])
|
||
|
||
response = client.get(f"/api/automation/reports/{job['session_id']}/all")
|
||
assert response.status_code == 200
|
||
with zipfile.ZipFile(BytesIO(response.content)) as archive:
|
||
files = archive.namelist()
|
||
assert all(name.startswith("1.1/") for name in files)
|
||
assert any(name.endswith(".docx") for name in files)
|
||
assert any(name.endswith(".xlsx") for name in files)
|
||
assert files.index("1.1/步骤截图/0001_步骤0001_操作前.png") < files.index(
|
||
"1.1/步骤截图/0002_步骤0002_操作后.png"
|
||
)
|
||
assert archive.read("1.1/任务录屏/0001_recording.mp4") == b"recording.mp4"
|
||
manifest = json.loads(archive.read("1.1/文件清单.json"))
|
||
assert manifest["session_id"] == job["session_id"]
|
||
assert "1.1-all.zip" in response.headers["content-disposition"]
|
||
|
||
|
||
def test_video_export_accepts_finalized_worker_trace_and_rejects_external_path(
|
||
isolated, tmp_path, monkeypatch
|
||
):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
session_id = job["session_id"]
|
||
recording_dir = reports.TRACES_PATH / f"web_123_{session_id[:8]}_PASS_2026-09-28"
|
||
recording_dir.mkdir(parents=True)
|
||
first = recording_dir / "recording.mp4"
|
||
first.write_bytes(b"first segment")
|
||
resumed = recording_dir / "resume_0001"
|
||
resumed.mkdir()
|
||
second = resumed / "recording.mp4"
|
||
second.write_bytes(b"second segment")
|
||
|
||
repository = Mock()
|
||
repository.get_video_recording_for_session.return_value = {"local_video_path": str(first)}
|
||
repository.get_session_by_id.return_value = {"video_filepath": str(first)}
|
||
monkeypatch.setattr(reports, "SessionRepository", lambda: repository)
|
||
|
||
archive_path = reports.export_session_media(session_id, "video")
|
||
with zipfile.ZipFile(archive_path) as archive:
|
||
names = archive.namelist()
|
||
assert names == [
|
||
"任务录屏/0001_web_123_" + session_id[:8] + "_PASS_2026-09-28_recording.mp4",
|
||
"任务录屏/0002_resume_0001_recording.mp4",
|
||
"文件清单.json",
|
||
]
|
||
assert archive.read(names[0]) == b"first segment"
|
||
assert archive.read(names[1]) == b"second segment"
|
||
|
||
external = tmp_path / "outside" / "recording.mp4"
|
||
external.parent.mkdir()
|
||
external.write_bytes(b"unrelated")
|
||
repository.get_video_recording_for_session.return_value = {"local_video_path": str(external)}
|
||
assert reports.session_videos({"id": session_id}) == []
|
||
|
||
|
||
def test_history_filters_all_persisted_rows_and_escapes_like(client, isolated):
|
||
batch = service.create_batch(payload())
|
||
first, second = isolated.jobs(batch["id"])
|
||
isolated.save_job(
|
||
{
|
||
**first,
|
||
"status": "completed",
|
||
"created_at": 100,
|
||
"case": {"name": "搜索_%", "goal": "abc", "tags": "回归"},
|
||
}
|
||
)
|
||
isolated.save_job({**second, "status": "failed", "created_at": 200})
|
||
assert client.get("/api/automation/history", params={"q": "_%"}).json()["total"] == 1
|
||
assert (
|
||
client.get("/api/automation/history", params={"q": "回归"}).json()["items"][0]["session_id"]
|
||
== first["session_id"]
|
||
)
|
||
assert (
|
||
client.get(
|
||
"/api/automation/history", params={"start": 150, "end": 250, "status": "failed"}
|
||
).json()["total"]
|
||
== 1
|
||
)
|
||
assert (
|
||
client.get("/api/automation/history", params={"size": 1, "page": 2}).json()["items"][0][
|
||
"session_id"
|
||
]
|
||
== first["session_id"]
|
||
)
|
||
|
||
|
||
def test_batch_submission_is_atomic_and_idempotent(client, isolated):
|
||
value = payload(request_id="same-request")
|
||
first = client.post("/api/automation/batches", json=value)
|
||
second = client.post("/api/automation/batches", json=value)
|
||
assert first.status_code == second.status_code == 200
|
||
assert first.json()["id"] == second.json()["id"]
|
||
assert len(isolated.jobs()) == 2
|
||
assert len(isolated.records("batch")) == 1
|
||
conflict = client.post("/api/automation/batches", json={**value, "name": "different"})
|
||
assert conflict.status_code == 409
|
||
lookup = client.get("/api/automation/batches/by-request/same-request").json()
|
||
assert lookup["id"] == first.json()["id"]
|
||
assert len(lookup["jobs"]) == 2
|
||
|
||
|
||
def test_removed_replay_is_inert_but_history_survives(client, isolated):
|
||
batch = service.create_batch(payload())
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**job, "execution_mode": "strict", "workflow_id": "legacy"})
|
||
service.recover_batches()
|
||
assert isolated.get("batch", batch["id"])["status"] == "cancelled"
|
||
assert all(j["status"] == "cancelled" for j in isolated.jobs(batch["id"]))
|
||
assert client.post(f"/api/automation/batches/{batch['id']}/resume").status_code == 409
|
||
assert client.post(f"/api/automation/batches/{batch['id']}/retry_failed").status_code == 409
|
||
assert client.get("/api/automation/workflows").status_code == 404
|
||
assert client.get("/api/automation/history").json()["total"] == 2
|