"""Automation acceptance checks with isolated persistence and no real device side effects.""" from io import BytesIO import json import time from unittest.mock import AsyncMock import zipfile from fastapi import FastAPI from fastapi.testclient import TestClient from openpyxl import load_workbook import pytest from apps.admin_console.routers import automation as api from apps.admin_console.services import automation_service as service from apps.admin_console.services import report_export as reports from apps.admin_console.services import task_import from apps.admin_console.services.automation_store import AutomationStore from artemis.data_engine.storage import StorageManager @pytest.fixture def isolated(tmp_path, monkeypatch): db = tmp_path / "data.db" StorageManager(db_path=db, base_trace_dir=tmp_path) store = AutomationStore(db) for module in (api, service, reports): monkeypatch.setattr(module, "store", store) monkeypatch.setattr(reports, "TRACES_PATH", tmp_path / "traces") monkeypatch.setattr(reports, "EXPORT_ROOT", tmp_path / "exports") from apps.admin_console.database.repositories.session_repository import SessionRepository from apps.admin_console.database.repositories.step_repository import StepRepository monkeypatch.setattr(reports, "SessionRepository", lambda *a: SessionRepository(db)) monkeypatch.setattr(reports, "StepRepository", lambda *a: StepRepository(db)) return store @pytest.fixture def client(isolated): app = FastAPI() app.include_router(api.router) return TestClient(app) def payload(**extra): return { "name": "批次", "cases": [{"name": "用例甲", "goal": "观察设置"}, {"name": "用例乙", "goal": "观察首页"}], "device_serial": "test-device", **extra, } def workbook_bytes(rows): wb = load_workbook(BytesIO(task_import.template())) for row in rows: wb["任务"].append(row) output = BytesIO() wb.save(output) return output.getvalue() def test_template_preview_rejects_formula_and_bad_json(): data = workbook_bytes( [ ["正常", "设置", "冒烟", "打开设置", "", "观察设置", '{"profile":"flash"}'], ["公式", "", "", "", "", "=SUM(1,2)"], ["参数错误", "", "", "", "", "测试", "{"], ] ) result = task_import.parse_workbook(data) assert len(result["tasks"]) == 1 assert [e["row"] for e in result["errors"]] == [3, 4] assert not result["valid"] def test_prompt_only_and_per_row_config_validation(): goal = "先查看屏幕,再按实际画面判断。" data = workbook_bytes( [ [ "CASE-7", "检查项目", "子项", "用例名称", "仅作索引的约束", goal, '{"profile":"pro","model":"mimo-v2.6-flash","record_video":true}', ], ["CASE-8", "", "", "", "", "观察屏幕", '{"OPENAI_API_KEY":"never"}'], ] ) preview = task_import.parse_workbook(data) assert [e["row"] for e in preview["errors"]] == [3] case = preview["tasks"][0] assert task_import.case_goal(case) == goal assert case["preconditions"] == "仅作索引的约束" assert case["task_config"]["record_video"] is True def test_invalid_archive_and_empty_template(): with pytest.raises(ValueError): task_import.parse_workbook(b"not a zip") assert not task_import.parse_workbook(task_import.template())["valid"] def test_preview_does_not_enqueue(client, isolated): response = client.post( "/api/automation/import/preview", files={ "file": ("tasks.xlsx", workbook_bytes([["案例", "", "", "观察页面", "", "观察页面"]])) }, ) assert response.status_code == 200 assert response.json()["valid"] assert not isolated.jobs() def test_preferences_and_required_device(client): assert client.get("/api/automation/preferences").json()["formats"] == ["docx", "xlsx"] assert client.put("/api/automation/preferences", json={"formats": ["xlsx"]}).status_code == 200 assert client.put("/api/automation/preferences", json={"formats": []}).status_code == 422 assert client.post("/api/automation/batches", json=payload(device_serial="")).status_code == 422 def test_restart_never_repeats_started_work(isolated): batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") job = isolated.jobs(batch["id"])[0] isolated.save_job({**job, "status": "queued"}) service.recover_batches() assert isolated.get("batch", batch["id"])["status"] == "paused" assert [j["status"] for j in isolated.jobs(batch["id"])] == ["interrupted", "pending"] retry = service.batch_action(batch["id"], "retry_failed") assert retry["status"] == "draft" assert len(isolated.jobs(retry["id"])) == 1 assert isolated.jobs(retry["id"])[0]["session_id"] != job["session_id"] @pytest.mark.asyncio async def test_batch_admission_idempotent_and_bound_device(isolated, monkeypatch): from apps.admin_console import server batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})()) enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]}) monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue) monkeypatch.setattr( "artemis.runtime.device_pool.validate_explicit_serial_async", AsyncMock(return_value=None) ) from artemis.runtime.device_pool import DeviceStatus monkeypatch.setattr( "artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]), ) await service.tick_batches() await service.tick_batches() assert enqueue.await_count == 1 assert enqueue.call_args.kwargs["device_serial"] == "test-device" assert isolated.jobs(batch["id"])[1]["status"] == "pending" @pytest.mark.asyncio async def test_paused_case_yields_batch_slot_to_next_case(isolated, monkeypatch): from apps.admin_console import server from artemis.runtime.device_pool import DeviceStatus batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") first = isolated.jobs(batch["id"])[0] isolated.save_job({**first, "status": "task_paused", "started_at": time.time()}) monkeypatch.setattr( service, "state", type("State", (), {"is_paused": False, "active_runs": {}})() ) enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]}) monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue) monkeypatch.setattr( "artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]), ) await service.tick_batches() assert enqueue.await_count == 1 assert enqueue.call_args.args[0] == ["观察首页"] assert [job["status"] for job in isolated.jobs(batch["id"])] == ["task_paused", "queued"] @pytest.mark.asyncio async def test_device_outage_pauses_without_consuming_pending_case(isolated, monkeypatch): batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})()) monkeypatch.setattr( "artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[]) ) await service.tick_batches() assert isolated.get("batch", batch["id"])["status"] == "paused" assert all(job["status"] == "pending" for job in isolated.jobs(batch["id"])) @pytest.mark.asyncio async def test_fail_fast_includes_skipped_reports(isolated, monkeypatch): batch = service.create_batch(payload(stop_on_failure=True)) first = isolated.jobs(batch["id"])[0] isolated.save_job({**first, "status": "failed"}) service.batch_action(batch["id"], "resume") await service.tick_batches() assert [j["status"] for j in isolated.jobs(batch["id"])] == ["failed", "skipped"] archive = reports.export_batch(batch["id"]) with zipfile.ZipFile(archive) as z: assert len(z.namelist()) == 6 # two summary files + two files per task assert sum(n.endswith(".docx") for n in z.namelist()) == 3 @pytest.mark.asyncio async def test_report_failure_does_not_strand_job(isolated, monkeypatch): job = isolated.jobs(service.create_batch(payload())["id"])[0] monkeypatch.setattr( reports, "export_session", lambda *a: (_ for _ in ()).throw(OSError("disk full")) ) await service.on_terminal({"session_id": job["session_id"]}, "completed") saved = isolated.job(job["session_id"]) assert saved["status"] == "completed" assert "disk full" in saved["report_error"] def test_reports_do_not_execute_formulas_or_invent_pass(isolated): batch = service.create_batch(payload()) job = isolated.jobs(batch["id"])[0] isolated.save_job( {**job, "case": {"name": "=1+1", "goal": '=HYPERLINK("x")'}, "status": "completed"} ) report = reports.session_report(job["session_id"]) assert report["verdict"].startswith("未验证") assert reports.verdict({"tests": {"passed": 2, "failed": 1}}) == "验证失败" assert reports.verdict({"tests": {"passed": 2, "unchecked": 1}}) == "验证不完整" paths = reports.export_session(job["session_id"]) wb = load_workbook(next(p for p in paths if p.suffix == ".xlsx")) assert wb["任务结果"]["B3"].data_type == "s" with zipfile.ZipFile(next(p for p in paths if p.suffix == ".docx")) as doc: assert "未验证".encode() in doc.read("word/document.xml") def test_task_material_zip_contains_ordered_evidence_and_reports( client, isolated, tmp_path, monkeypatch ): job = isolated.jobs(service.create_batch(payload())["id"])[0] evidence = tmp_path / "evidence" evidence.mkdir() first = evidence / "first.png" second = evidence / "second.png" video = evidence / "recording.mp4" for path in (first, second, video): path.write_bytes(path.name.encode()) monkeypatch.setattr( reports, "session_screenshots", lambda report: [(first, "0001_步骤0001_操作前.png"), (second, "0002_步骤0002_操作后.png")], ) monkeypatch.setattr(reports, "session_videos", lambda report: [(video, "0001_recording.mp4")]) response = client.get(f"/api/automation/reports/{job['session_id']}/all") assert response.status_code == 200 with zipfile.ZipFile(BytesIO(response.content)) as archive: files = archive.namelist() assert any(name.endswith(".docx") for name in files) assert any(name.endswith(".xlsx") for name in files) assert files.index("步骤截图/0001_步骤0001_操作前.png") < files.index( "步骤截图/0002_步骤0002_操作后.png" ) assert archive.read("任务录屏/0001_recording.mp4") == b"recording.mp4" manifest = json.loads(archive.read("文件清单.json")) assert manifest["session_id"] == job["session_id"] def test_history_filters_all_persisted_rows_and_escapes_like(client, isolated): batch = service.create_batch(payload()) first, second = isolated.jobs(batch["id"]) isolated.save_job( { **first, "status": "completed", "created_at": 100, "case": {"name": "搜索_%", "goal": "abc", "tags": "回归"}, } ) isolated.save_job({**second, "status": "failed", "created_at": 200}) assert client.get("/api/automation/history", params={"q": "_%"}).json()["total"] == 1 assert ( client.get("/api/automation/history", params={"q": "回归"}).json()["items"][0]["session_id"] == first["session_id"] ) assert ( client.get( "/api/automation/history", params={"start": 150, "end": 250, "status": "failed"} ).json()["total"] == 1 ) assert ( client.get("/api/automation/history", params={"size": 1, "page": 2}).json()["items"][0][ "session_id" ] == first["session_id"] ) def test_batch_submission_is_atomic_and_idempotent(client, isolated): value = payload(request_id="same-request") first = client.post("/api/automation/batches", json=value) second = client.post("/api/automation/batches", json=value) assert first.status_code == second.status_code == 200 assert first.json()["id"] == second.json()["id"] assert len(isolated.jobs()) == 2 assert len(isolated.records("batch")) == 1 conflict = client.post("/api/automation/batches", json={**value, "name": "different"}) assert conflict.status_code == 409 lookup = client.get("/api/automation/batches/by-request/same-request").json() assert lookup["id"] == first.json()["id"] assert len(lookup["jobs"]) == 2 def test_removed_replay_is_inert_but_history_survives(client, isolated): batch = service.create_batch(payload()) job = isolated.jobs(batch["id"])[0] isolated.save_job({**job, "execution_mode": "strict", "workflow_id": "legacy"}) service.recover_batches() assert isolated.get("batch", batch["id"])["status"] == "cancelled" assert all(j["status"] == "cancelled" for j in isolated.jobs(batch["id"])) assert client.post(f"/api/automation/batches/{batch['id']}/resume").status_code == 409 assert client.post(f"/api/automation/batches/{batch['id']}/retry_failed").status_code == 409 assert client.get("/api/automation/workflows").status_code == 404 assert client.get("/api/automation/history").json()["total"] == 2