413 lines
17 KiB
Python
413 lines
17 KiB
Python
"""Automation acceptance checks with isolated persistence and no real device side effects."""
|
|
|
|
from io import BytesIO
|
|
import json
|
|
import time
|
|
from unittest.mock import AsyncMock
|
|
import zipfile
|
|
|
|
from docx import Document
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
from openpyxl import load_workbook
|
|
import pytest
|
|
|
|
from apps.admin_console.routers import automation as api
|
|
from apps.admin_console.services import automation_service as service
|
|
from apps.admin_console.services import report_export as reports
|
|
from apps.admin_console.services import task_import
|
|
from apps.admin_console.services.automation_store import AutomationStore
|
|
from artemis.data_engine.storage import StorageManager
|
|
|
|
|
|
@pytest.fixture
|
|
def isolated(tmp_path, monkeypatch):
|
|
db = tmp_path / "data.db"
|
|
StorageManager(db_path=db, base_trace_dir=tmp_path)
|
|
store = AutomationStore(db)
|
|
for module in (api, service, reports):
|
|
monkeypatch.setattr(module, "store", store)
|
|
monkeypatch.setattr(reports, "TRACES_PATH", tmp_path / "traces")
|
|
monkeypatch.setattr(reports, "EXPORT_ROOT", tmp_path / "exports")
|
|
from apps.admin_console.database.repositories.session_repository import SessionRepository
|
|
from apps.admin_console.database.repositories.step_repository import StepRepository
|
|
|
|
monkeypatch.setattr(reports, "SessionRepository", lambda *a: SessionRepository(db))
|
|
monkeypatch.setattr(reports, "StepRepository", lambda *a: StepRepository(db))
|
|
return store
|
|
|
|
|
|
@pytest.fixture
|
|
def client(isolated):
|
|
app = FastAPI()
|
|
app.include_router(api.router)
|
|
return TestClient(app)
|
|
|
|
|
|
def payload(**extra):
|
|
return {
|
|
"name": "批次",
|
|
"cases": [{"name": "用例甲", "goal": "观察设置"}, {"name": "用例乙", "goal": "观察首页"}],
|
|
"device_serial": "test-device",
|
|
**extra,
|
|
}
|
|
|
|
|
|
def workbook_bytes(rows):
|
|
wb = load_workbook(BytesIO(task_import.template()))
|
|
for row in rows:
|
|
wb["任务"].append(row)
|
|
output = BytesIO()
|
|
wb.save(output)
|
|
return output.getvalue()
|
|
|
|
|
|
def test_template_preview_rejects_formula_and_bad_json():
|
|
data = workbook_bytes(
|
|
[
|
|
["正常", "设置", "冒烟", "打开设置", "", "观察设置", '{"profile":"flash"}'],
|
|
["公式", "", "", "", "", "=SUM(1,2)"],
|
|
["参数错误", "", "", "", "", "测试", "{"],
|
|
]
|
|
)
|
|
result = task_import.parse_workbook(data)
|
|
assert len(result["tasks"]) == 1
|
|
assert [e["row"] for e in result["errors"]] == [3, 4]
|
|
assert not result["valid"]
|
|
|
|
|
|
def test_prompt_only_and_per_row_config_validation():
|
|
goal = "先查看屏幕，再按实际画面判断。"
|
|
data = workbook_bytes(
|
|
[
|
|
[
|
|
"CASE-7",
|
|
"检查项目",
|
|
"子项",
|
|
"用例名称",
|
|
"仅作索引的约束",
|
|
goal,
|
|
'{"profile":"pro","model":"mimo-v2.6-flash","record_video":true}',
|
|
],
|
|
["CASE-8", "", "", "", "", "观察屏幕", '{"OPENAI_API_KEY":"never"}'],
|
|
]
|
|
)
|
|
preview = task_import.parse_workbook(data)
|
|
assert [e["row"] for e in preview["errors"]] == [3]
|
|
case = preview["tasks"][0]
|
|
assert task_import.case_goal(case) == goal
|
|
assert case["preconditions"] == "仅作索引的约束"
|
|
assert case["task_config"]["record_video"] is True
|
|
|
|
|
|
def test_invalid_archive_and_empty_template():
|
|
with pytest.raises(ValueError):
|
|
task_import.parse_workbook(b"not a zip")
|
|
assert not task_import.parse_workbook(task_import.template())["valid"]
|
|
|
|
|
|
def test_preview_does_not_enqueue(client, isolated):
|
|
response = client.post(
|
|
"/api/automation/import/preview",
|
|
files={
|
|
"file": ("tasks.xlsx", workbook_bytes([["案例", "", "", "观察页面", "", "观察页面"]]))
|
|
},
|
|
)
|
|
assert response.status_code == 200
|
|
assert response.json()["valid"]
|
|
assert not isolated.jobs()
|
|
|
|
|
|
def test_preferences_and_required_device(client):
|
|
assert client.get("/api/automation/preferences").json()["formats"] == ["docx", "xlsx"]
|
|
assert client.put("/api/automation/preferences", json={"formats": ["xlsx"]}).status_code == 200
|
|
assert client.put("/api/automation/preferences", json={"formats": []}).status_code == 422
|
|
assert client.post("/api/automation/batches", json=payload(device_serial="")).status_code == 422
|
|
|
|
|
|
def test_batch_delete_retains_executed_task_and_prevents_request_replay(client, isolated):
|
|
created = client.post(
|
|
"/api/automation/batches", json=payload(request_id="delete-example")
|
|
).json()
|
|
first, second = isolated.jobs(created["id"])
|
|
isolated.save_job({**first, "status": "completed"})
|
|
|
|
response = client.delete(f"/api/automation/batches/{created['id']}")
|
|
assert response.status_code == 200
|
|
assert client.get(f"/api/automation/batches/{created['id']}").status_code == 404
|
|
assert isolated.job(first["session_id"])["case"]["name"] == "用例甲"
|
|
assert isolated.job(second["session_id"]) is None
|
|
assert reports.session_report(first["session_id"])["name"] == "用例甲"
|
|
assert (
|
|
client.post(
|
|
"/api/automation/batches", json=payload(request_id="delete-example")
|
|
).status_code
|
|
== 409
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("status", ["queued", "running", "task_paused", "resuming", "finalizing"])
|
|
def test_batch_delete_rejects_active_jobs(client, isolated, status):
|
|
batch = service.create_batch(payload())
|
|
first = isolated.jobs(batch["id"])[0]
|
|
isolated.save_job({**first, "status": status})
|
|
|
|
response = client.delete(f"/api/automation/batches/{batch['id']}")
|
|
assert response.status_code == 409
|
|
assert isolated.get("batch", batch["id"]) is not None
|
|
assert len(isolated.jobs(batch["id"])) == 2
|
|
|
|
|
|
def test_restart_never_repeats_started_work(isolated):
|
|
batch = service.create_batch(payload())
|
|
service.batch_action(batch["id"], "resume")
|
|
job = isolated.jobs(batch["id"])[0]
|
|
isolated.save_job({**job, "status": "queued"})
|
|
service.recover_batches()
|
|
assert isolated.get("batch", batch["id"])["status"] == "paused"
|
|
assert [j["status"] for j in isolated.jobs(batch["id"])] == ["interrupted", "pending"]
|
|
retry = service.batch_action(batch["id"], "retry_failed")
|
|
assert retry["status"] == "draft"
|
|
assert len(isolated.jobs(retry["id"])) == 1
|
|
assert isolated.jobs(retry["id"])[0]["session_id"] != job["session_id"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_batch_admission_idempotent_and_bound_device(isolated, monkeypatch):
|
|
from apps.admin_console import server
|
|
|
|
batch = service.create_batch(payload())
|
|
service.batch_action(batch["id"], "resume")
|
|
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
|
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
|
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
|
monkeypatch.setattr(
|
|
"artemis.runtime.device_pool.validate_explicit_serial_async", AsyncMock(return_value=None)
|
|
)
|
|
from artemis.runtime.device_pool import DeviceStatus
|
|
|
|
monkeypatch.setattr(
|
|
"artemis.runtime.device_pool.try_list_devices_async",
|
|
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
|
)
|
|
await service.tick_batches()
|
|
await service.tick_batches()
|
|
assert enqueue.await_count == 1
|
|
assert enqueue.call_args.kwargs["device_serial"] == "test-device"
|
|
assert isolated.jobs(batch["id"])[1]["status"] == "pending"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_paused_case_yields_batch_slot_to_next_case(isolated, monkeypatch):
|
|
from apps.admin_console import server
|
|
from artemis.runtime.device_pool import DeviceStatus
|
|
|
|
batch = service.create_batch(payload())
|
|
service.batch_action(batch["id"], "resume")
|
|
first = isolated.jobs(batch["id"])[0]
|
|
isolated.save_job({**first, "status": "task_paused", "started_at": time.time()})
|
|
monkeypatch.setattr(
|
|
service, "state", type("State", (), {"is_paused": False, "active_runs": {}})()
|
|
)
|
|
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
|
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
|
monkeypatch.setattr(
|
|
"artemis.runtime.device_pool.try_list_devices_async",
|
|
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
|
)
|
|
await service.tick_batches()
|
|
assert enqueue.await_count == 1
|
|
assert enqueue.call_args.args[0] == ["观察首页"]
|
|
assert [job["status"] for job in isolated.jobs(batch["id"])] == ["task_paused", "queued"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_device_outage_pauses_without_consuming_pending_case(isolated, monkeypatch):
|
|
batch = service.create_batch(payload())
|
|
service.batch_action(batch["id"], "resume")
|
|
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
|
monkeypatch.setattr(
|
|
"artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[])
|
|
)
|
|
await service.tick_batches()
|
|
assert isolated.get("batch", batch["id"])["status"] == "paused"
|
|
assert all(job["status"] == "pending" for job in isolated.jobs(batch["id"]))
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fail_fast_includes_skipped_reports(isolated, monkeypatch):
|
|
batch = service.create_batch(payload(stop_on_failure=True))
|
|
first = isolated.jobs(batch["id"])[0]
|
|
isolated.save_job({**first, "status": "failed"})
|
|
service.batch_action(batch["id"], "resume")
|
|
await service.tick_batches()
|
|
assert [j["status"] for j in isolated.jobs(batch["id"])] == ["failed", "skipped"]
|
|
archive = reports.export_batch(batch["id"])
|
|
with zipfile.ZipFile(archive) as z:
|
|
assert len(z.namelist()) == 6 # two summary files + two files per task
|
|
assert sum(n.endswith(".docx") for n in z.namelist()) == 3
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_report_failure_does_not_strand_job(isolated, monkeypatch):
|
|
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
|
monkeypatch.setattr(
|
|
reports, "export_session", lambda *a: (_ for _ in ()).throw(OSError("disk full"))
|
|
)
|
|
await service.on_terminal({"session_id": job["session_id"]}, "completed")
|
|
saved = isolated.job(job["session_id"])
|
|
assert saved["status"] == "completed"
|
|
assert "disk full" in saved["report_error"]
|
|
|
|
|
|
def test_reports_do_not_execute_formulas_or_invent_pass(isolated):
|
|
batch = service.create_batch(payload())
|
|
job = isolated.jobs(batch["id"])[0]
|
|
isolated.save_job(
|
|
{**job, "case": {"name": "=1+1", "goal": '=HYPERLINK("x")'}, "status": "completed"}
|
|
)
|
|
report = reports.session_report(job["session_id"])
|
|
assert report["verdict"].startswith("未验证")
|
|
assert reports.verdict({"tests": {"passed": 2, "failed": 1}}) == "验证失败"
|
|
assert reports.verdict({"tests": {"passed": 2, "unchecked": 1}}) == "验证不完整"
|
|
paths = reports.export_session(job["session_id"])
|
|
wb = load_workbook(next(p for p in paths if p.suffix == ".xlsx"))
|
|
assert wb["任务结果"]["B3"].data_type == "s"
|
|
with zipfile.ZipFile(next(p for p in paths if p.suffix == ".docx")) as doc:
|
|
assert "未验证".encode() in doc.read("word/document.xml")
|
|
|
|
|
|
def test_failed_report_exposes_logged_reason_and_formats_chinese_markdown(isolated):
|
|
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
|
isolated.save_job({**job, "status": "failed"})
|
|
folder = reports.TRACES_PATH / job["session_id"]
|
|
(folder / "notes").mkdir(parents=True)
|
|
(folder / "stdout.log").write_text(
|
|
"❌ [run] Error running automation: 模型连续 3 次未提供可执行的工具调用\n",
|
|
encoding="utf-8",
|
|
)
|
|
(folder / "notes" / "output.md").write_text(
|
|
"## 结果\n- **网络**正常\n\n| 分类 | 状态 |\n| --- | --- |\n| 系统 | 未确认 |",
|
|
encoding="utf-8",
|
|
)
|
|
report = reports.session_report(job["session_id"])
|
|
assert "连续 3 次" in report["error"]
|
|
assert report["verdict"] == "未验证（任务异常终止）"
|
|
assert "abc123secret" not in reports.failure_reason(
|
|
folder, "failed", "认证失败 OPENAI_API_KEY=abc123secret Authorization: Bearer abc123secret"
|
|
)
|
|
paths = reports.export_session(job["session_id"])
|
|
doc = Document(next(path for path in paths if path.suffix == ".docx"))
|
|
assert doc.styles["Normal"].font.name == "Microsoft YaHei"
|
|
assert any("异常原因" in paragraph.text for paragraph in doc.paragraphs)
|
|
assert any(
|
|
run.bold and "网络" in run.text for paragraph in doc.paragraphs for run in paragraph.runs
|
|
)
|
|
assert any(
|
|
cell.text == "分类" for table in doc.tables for row in table.rows for cell in row.cells
|
|
)
|
|
wb = load_workbook(next(path for path in paths if path.suffix == ".xlsx"))
|
|
assert "连续 3 次" in wb["任务结果"]["B15"].value
|
|
assert wb["结论"]["B2"].value == "结果"
|
|
assert wb["结果表1"]["A2"].value == "系统"
|
|
assert wb["执行证据"]["A1"].font.name == "微软雅黑"
|
|
assert wb["执行证据"].page_setup.orientation == "landscape"
|
|
assert reports.execution_result_label({"status": "dispatched"}) == "动作已发出（需核对截图）"
|
|
assert reports.conclusion_rows("## 结果\n- **通过**检查\n```txt\n| 不是 | 表格 |\n```") == [
|
|
("类型", "结论与发现"),
|
|
("小节", "结果"),
|
|
("要点", "通过检查"),
|
|
("代码", "| 不是 | 表格 |"),
|
|
]
|
|
|
|
|
|
def test_task_material_zip_contains_ordered_evidence_and_reports(
|
|
client, isolated, tmp_path, monkeypatch
|
|
):
|
|
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
|
evidence = tmp_path / "evidence"
|
|
evidence.mkdir()
|
|
first = evidence / "first.png"
|
|
second = evidence / "second.png"
|
|
video = evidence / "recording.mp4"
|
|
for path in (first, second, video):
|
|
path.write_bytes(path.name.encode())
|
|
monkeypatch.setattr(
|
|
reports,
|
|
"session_screenshots",
|
|
lambda report: [(first, "0001_步骤0001_操作前.png"), (second, "0002_步骤0002_操作后.png")],
|
|
)
|
|
monkeypatch.setattr(reports, "session_videos", lambda report: [(video, "0001_recording.mp4")])
|
|
|
|
response = client.get(f"/api/automation/reports/{job['session_id']}/all")
|
|
assert response.status_code == 200
|
|
with zipfile.ZipFile(BytesIO(response.content)) as archive:
|
|
files = archive.namelist()
|
|
assert any(name.endswith(".docx") for name in files)
|
|
assert any(name.endswith(".xlsx") for name in files)
|
|
assert files.index("步骤截图/0001_步骤0001_操作前.png") < files.index(
|
|
"步骤截图/0002_步骤0002_操作后.png"
|
|
)
|
|
assert archive.read("任务录屏/0001_recording.mp4") == b"recording.mp4"
|
|
manifest = json.loads(archive.read("文件清单.json"))
|
|
assert manifest["session_id"] == job["session_id"]
|
|
|
|
|
|
def test_history_filters_all_persisted_rows_and_escapes_like(client, isolated):
|
|
batch = service.create_batch(payload())
|
|
first, second = isolated.jobs(batch["id"])
|
|
isolated.save_job(
|
|
{
|
|
**first,
|
|
"status": "completed",
|
|
"created_at": 100,
|
|
"case": {"name": "搜索_%", "goal": "abc", "tags": "回归"},
|
|
}
|
|
)
|
|
isolated.save_job({**second, "status": "failed", "created_at": 200})
|
|
assert client.get("/api/automation/history", params={"q": "_%"}).json()["total"] == 1
|
|
assert (
|
|
client.get("/api/automation/history", params={"q": "回归"}).json()["items"][0]["session_id"]
|
|
== first["session_id"]
|
|
)
|
|
assert (
|
|
client.get(
|
|
"/api/automation/history", params={"start": 150, "end": 250, "status": "failed"}
|
|
).json()["total"]
|
|
== 1
|
|
)
|
|
assert (
|
|
client.get("/api/automation/history", params={"size": 1, "page": 2}).json()["items"][0][
|
|
"session_id"
|
|
]
|
|
== first["session_id"]
|
|
)
|
|
|
|
|
|
def test_batch_submission_is_atomic_and_idempotent(client, isolated):
|
|
value = payload(request_id="same-request")
|
|
first = client.post("/api/automation/batches", json=value)
|
|
second = client.post("/api/automation/batches", json=value)
|
|
assert first.status_code == second.status_code == 200
|
|
assert first.json()["id"] == second.json()["id"]
|
|
assert len(isolated.jobs()) == 2
|
|
assert len(isolated.records("batch")) == 1
|
|
conflict = client.post("/api/automation/batches", json={**value, "name": "different"})
|
|
assert conflict.status_code == 409
|
|
lookup = client.get("/api/automation/batches/by-request/same-request").json()
|
|
assert lookup["id"] == first.json()["id"]
|
|
assert len(lookup["jobs"]) == 2
|
|
|
|
|
|
def test_removed_replay_is_inert_but_history_survives(client, isolated):
|
|
batch = service.create_batch(payload())
|
|
job = isolated.jobs(batch["id"])[0]
|
|
isolated.save_job({**job, "execution_mode": "strict", "workflow_id": "legacy"})
|
|
service.recover_batches()
|
|
assert isolated.get("batch", batch["id"])["status"] == "cancelled"
|
|
assert all(j["status"] == "cancelled" for j in isolated.jobs(batch["id"]))
|
|
assert client.post(f"/api/automation/batches/{batch['id']}/resume").status_code == 409
|
|
assert client.post(f"/api/automation/batches/{batch['id']}/retry_failed").status_code == 409
|
|
assert client.get("/api/automation/workflows").status_code == 404
|
|
assert client.get("/api/automation/history").json()["total"] == 2
|