"""Automation acceptance checks with isolated persistence and no real device side effects.""" from io import BytesIO import json import sqlite3 import time from unittest.mock import AsyncMock, Mock from uuid import UUID import zipfile from docx import Document from fastapi import FastAPI from fastapi.testclient import TestClient from openpyxl import load_workbook import pytest from apps.admin_console.routers import automation as api from apps.admin_console.services import automation_service as service from apps.admin_console.services import report_export as reports from apps.admin_console.services import task_import from apps.admin_console.services.automation_store import AutomationStore from artemis.data_engine.storage import StorageManager @pytest.fixture def isolated(tmp_path, monkeypatch): db = tmp_path / "data.db" StorageManager(db_path=db, base_trace_dir=tmp_path) store = AutomationStore(db) for module in (api, service, reports): monkeypatch.setattr(module, "store", store) monkeypatch.setattr(reports, "TRACES_PATH", tmp_path / "traces") monkeypatch.setattr(reports, "EXPORT_ROOT", tmp_path / "exports") from apps.admin_console.database.repositories.session_repository import SessionRepository from apps.admin_console.database.repositories.step_repository import StepRepository monkeypatch.setattr(reports, "SessionRepository", lambda *a: SessionRepository(db)) monkeypatch.setattr(reports, "StepRepository", lambda *a: StepRepository(db)) return store @pytest.fixture def client(isolated): app = FastAPI() app.include_router(api.router) return TestClient(app) def payload(**extra): return { "name": "批次", "cases": [{"name": "用例甲", "goal": "观察设置"}, {"name": "用例乙", "goal": "观察首页"}], "device_serial": "test-device", **extra, } def workbook_bytes(rows): wb = load_workbook(BytesIO(task_import.template())) for row in rows: wb["任务"].append(row) output = BytesIO() wb.save(output) return output.getvalue() def test_template_preview_rejects_formula_and_bad_json(): data = workbook_bytes( [ ["正常", "设置", "冒烟", "打开设置", "", "观察设置", '{"profile":"flash"}'], ["公式", "", "", "", "", "=SUM(1,2)"], ["参数错误", "", "", "", "", "测试", "{"], ] ) result = task_import.parse_workbook(data) assert len(result["tasks"]) == 1 assert [e["row"] for e in result["errors"]] == [3, 4] assert not result["valid"] def test_prompt_only_and_per_row_config_validation(): goal = "先查看屏幕,再按实际画面判断。" data = workbook_bytes( [ [ "CASE-7", "检查项目", "子项", "用例名称", "仅作索引的约束", goal, '{"profile":"pro","model":"mimo-v2.6-flash","record_video":true}', ], ["CASE-8", "", "", "", "", "观察屏幕", '{"OPENAI_API_KEY":"never"}'], ] ) preview = task_import.parse_workbook(data) assert [e["row"] for e in preview["errors"]] == [3] case = preview["tasks"][0] assert task_import.case_goal(case) == goal assert case["preconditions"] == "仅作索引的约束" assert case["task_config"]["record_video"] is True def test_invalid_archive_and_empty_template(): with pytest.raises(ValueError): task_import.parse_workbook(b"not a zip") assert not task_import.parse_workbook(task_import.template())["valid"] def test_preview_does_not_enqueue(client, isolated): response = client.post( "/api/automation/import/preview", files={ "file": ("tasks.xlsx", workbook_bytes([["案例", "", "", "观察页面", "", "观察页面"]])) }, ) assert response.status_code == 200 assert response.json()["valid"] assert not isolated.jobs() def test_preferences_and_required_device(client): assert client.get("/api/automation/preferences").json()["formats"] == ["docx", "xlsx"] assert client.put("/api/automation/preferences", json={"formats": ["xlsx"]}).status_code == 200 assert client.put("/api/automation/preferences", json={"formats": []}).status_code == 422 assert client.post("/api/automation/batches", json=payload(device_serial="")).status_code == 422 def test_batch_delete_removes_tasks_and_files_and_prevents_request_replay(client, isolated): created = client.post( "/api/automation/batches", json=payload(request_id="delete-example") ).json() first, second = isolated.jobs(created["id"]) isolated.save_job({**first, "status": "completed"}) trace_dir = reports.TRACES_PATH / first["session_id"] trace_dir.mkdir(parents=True) (trace_dir / "result.json").write_text("{}", encoding="utf-8") report_dir = reports.EXPORT_ROOT / "sessions" / first["session_id"] report_dir.mkdir(parents=True) (report_dir / "任务报告.docx").write_bytes(b"report") batch_dir = reports.EXPORT_ROOT / "batches" / created["id"] batch_dir.mkdir(parents=True) (batch_dir / "批次报告.zip").write_bytes(b"zip") mcp_task_dir = reports.EXPORT_ROOT.parent / "mcp_exports" / "sessions" / first["session_id"] mcp_task_dir.mkdir(parents=True) (mcp_task_dir / "报告.docx").write_bytes(b"report") mcp_batch_dir = reports.EXPORT_ROOT.parent / "mcp_exports" / "batches" / created["id"] mcp_batch_dir.mkdir(parents=True) (mcp_batch_dir / "批次.zip").write_bytes(b"zip") unrelated = reports.TRACES_PATH / "unrelated-task" unrelated.mkdir() (unrelated / "keep.txt").write_text("keep", encoding="utf-8") response = client.delete(f"/api/automation/batches/{created['id']}") assert response.status_code == 200 assert client.get(f"/api/automation/batches/{created['id']}").status_code == 404 assert isolated.job(first["session_id"]) is None assert isolated.job(second["session_id"]) is None assert not trace_dir.exists() assert not report_dir.exists() assert not batch_dir.exists() assert not mcp_task_dir.exists() assert not mcp_batch_dir.exists() assert (unrelated / "keep.txt").read_text(encoding="utf-8") == "keep" with pytest.raises(ValueError, match="不存在"): reports.session_report(first["session_id"]) assert ( client.post( "/api/automation/batches", json=payload(request_id="delete-example") ).status_code == 409 ) @pytest.mark.parametrize("status", ["queued", "running", "task_paused", "resuming", "finalizing"]) def test_batch_delete_rejects_active_jobs(client, isolated, status): batch = service.create_batch(payload()) first = isolated.jobs(batch["id"])[0] isolated.save_job({**first, "status": status}) response = client.delete(f"/api/automation/batches/{batch['id']}") assert response.status_code == 409 assert isolated.get("batch", batch["id"]) is not None assert len(isolated.jobs(batch["id"])) == 2 def test_batch_deletion_reservation_blocks_new_dispatch(isolated): batch = service.create_batch(payload()) job = isolated.jobs(batch["id"])[0] isolated.begin_delete_batch(batch["id"]) with pytest.raises(ValueError, match="正在删除"): isolated.save_job({**job, "status": "queued"}) with pytest.raises(ValueError, match="正在删除"): isolated.put("batch", batch["id"], {**batch, "status": "running"}) assert isolated.jobs_for_sessions([job["session_id"]])[job["session_id"]]["status"] == "pending" def test_batch_deletion_preserves_shared_screenshot_until_last_task(client, isolated): first_batch = service.create_batch(payload()) second_batch = service.create_batch(payload(name="另一批次")) first = isolated.jobs(first_batch["id"])[0] other = isolated.jobs(second_batch["id"])[0] shared_hash = "a" * 64 storage = StorageManager(isolated.path, reports.TRACES_PATH) image_file = reports.TRACES_PATH / "images" / f"{shared_hash}.jpg" image_file.parent.mkdir(parents=True) image_file.write_bytes(b"shared image") with sqlite3.connect(isolated.path) as conn: conn.execute("INSERT INTO images(image_name) VALUES (?)", (shared_hash,)) for job in (first, other): conn.execute( "INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info) " "VALUES (?,?,?,?,?)", (job["session_id"], "观察设置", 1.0, "completed", "{}"), ) conn.execute( "INSERT INTO steps(step_id,session_id,pre_image_name) VALUES (?,?,?)", (job["session_id"], job["session_id"], shared_hash), ) assert client.delete(f"/api/automation/batches/{first_batch['id']}").status_code == 200 assert image_file.read_bytes() == b"shared image" assert storage.get_session(UUID(other["session_id"])) is not None assert client.delete(f"/api/automation/batches/{second_batch['id']}").status_code == 200 assert not image_file.exists() def test_restart_never_repeats_started_work(isolated): batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") job = isolated.jobs(batch["id"])[0] isolated.save_job({**job, "status": "queued"}) service.recover_batches() assert isolated.get("batch", batch["id"])["status"] == "paused" assert [j["status"] for j in isolated.jobs(batch["id"])] == ["interrupted", "pending"] retry = service.batch_action(batch["id"], "retry_failed") assert retry["status"] == "draft" assert len(isolated.jobs(retry["id"])) == 1 assert isolated.jobs(retry["id"])[0]["session_id"] != job["session_id"] @pytest.mark.asyncio async def test_batch_admission_idempotent_and_bound_device(isolated, monkeypatch): from apps.admin_console import server batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})()) enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]}) monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue) monkeypatch.setattr( "artemis.runtime.device_pool.validate_explicit_serial_async", AsyncMock(return_value=None) ) from artemis.runtime.device_pool import DeviceStatus monkeypatch.setattr( "artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]), ) await service.tick_batches() await service.tick_batches() assert enqueue.await_count == 1 assert enqueue.call_args.kwargs["device_serial"] == "test-device" assert isolated.jobs(batch["id"])[1]["status"] == "pending" @pytest.mark.asyncio async def test_paused_case_yields_batch_slot_to_next_case(isolated, monkeypatch): from apps.admin_console import server from artemis.runtime.device_pool import DeviceStatus batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") first = isolated.jobs(batch["id"])[0] isolated.save_job({**first, "status": "task_paused", "started_at": time.time()}) monkeypatch.setattr( service, "state", type("State", (), {"is_paused": False, "active_runs": {}})() ) enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]}) monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue) monkeypatch.setattr( "artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]), ) await service.tick_batches() assert enqueue.await_count == 1 assert enqueue.call_args.args[0] == ["观察首页"] assert [job["status"] for job in isolated.jobs(batch["id"])] == ["task_paused", "queued"] @pytest.mark.asyncio async def test_device_outage_pauses_without_consuming_pending_case(isolated, monkeypatch): batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})()) monkeypatch.setattr( "artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[]) ) await service.tick_batches() assert isolated.get("batch", batch["id"])["status"] == "paused" assert all(job["status"] == "pending" for job in isolated.jobs(batch["id"])) @pytest.mark.asyncio async def test_fail_fast_includes_skipped_reports(isolated, monkeypatch): batch = service.create_batch(payload(stop_on_failure=True)) first = isolated.jobs(batch["id"])[0] isolated.save_job({**first, "status": "failed"}) service.batch_action(batch["id"], "resume") await service.tick_batches() assert [j["status"] for j in isolated.jobs(batch["id"])] == ["failed", "skipped"] archive = reports.export_batch(batch["id"]) with zipfile.ZipFile(archive) as z: assert len(z.namelist()) == 6 # two summary files + two files per task assert sum(n.endswith(".docx") for n in z.namelist()) == 3 @pytest.mark.asyncio async def test_report_failure_does_not_strand_job(isolated, monkeypatch): job = isolated.jobs(service.create_batch(payload())["id"])[0] monkeypatch.setattr( reports, "export_session", lambda *a: (_ for _ in ()).throw(OSError("disk full")) ) await service.on_terminal({"session_id": job["session_id"]}, "completed") saved = isolated.job(job["session_id"]) assert saved["status"] == "completed" assert "disk full" in saved["report_error"] @pytest.mark.asyncio async def test_batch_timeout_stays_visible_in_job_report_and_history(client, isolated): job = isolated.jobs(service.create_batch(payload())["id"])[0] isolated.save_job( {**job, "status": "running", "timed_out": True, "error": "任务执行超时(30 秒)"} ) await service.on_terminal({"session_id": job["session_id"]}, "cancelled") saved = isolated.job(job["session_id"]) assert saved["status"] == "failed" assert "超时" in saved["error"] assert "超时" in reports.session_report(job["session_id"])["error"] history = client.get("/api/automation/history").json()["items"] assert any( item["session_id"] == job["session_id"] and "超时" in item["error"] for item in history ) @pytest.mark.asyncio @pytest.mark.parametrize("stop_on_failure", [False, True]) async def test_timed_out_case_releases_next_batch_case(isolated, monkeypatch, stop_on_failure): from apps.admin_console import server from artemis.runtime.device_pool import DeviceStatus batch = service.create_batch(payload(timeout_seconds=1, stop_on_failure=stop_on_failure)) service.batch_action(batch["id"], "resume") first = isolated.jobs(batch["id"])[0] isolated.save_job({**first, "status": "running", "started_at": time.time() - 2}) monkeypatch.setattr( service, "state", type("State", (), {"is_paused": False, "active_runs": {}})() ) stop = Mock(return_value=True) enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]}) monkeypatch.setattr(server.task_queue_service, "stop_tasks", stop) monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue) monkeypatch.setattr( "artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]), ) monkeypatch.setattr(reports, "export_batch", Mock()) await service.tick_batches() assert isolated.get("batch", batch["id"])["status"] == "running" assert "超时" in isolated.job(first["session_id"])["error"] stop.assert_called_once_with(session_id=first["session_id"]) await service.tick_batches() stop.assert_called_once() await service.on_terminal({"session_id": first["session_id"]}, "cancelled") await service.tick_batches() statuses = [job["status"] for job in isolated.jobs(batch["id"])] assert statuses == (["failed", "skipped"] if stop_on_failure else ["failed", "queued"]) assert enqueue.await_count == (0 if stop_on_failure else 1) @pytest.mark.asyncio async def test_queued_case_does_not_consume_execution_timeout(isolated, monkeypatch): from apps.admin_console import server batch = service.create_batch(payload(timeout_seconds=1)) service.batch_action(batch["id"], "resume") first = isolated.jobs(batch["id"])[0] isolated.save_job({**first, "status": "queued", "started_at": time.time() - 30}) monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})()) stop = Mock() monkeypatch.setattr(server.task_queue_service, "stop_tasks", stop) await service.tick_batches() stop.assert_not_called() assert isolated.job(first["session_id"])["status"] == "queued" assert isolated.get("batch", batch["id"])["status"] == "running" @pytest.mark.asyncio async def test_model_timeout_from_runner_log_reaches_failed_task_and_report(isolated): job = isolated.jobs(service.create_batch(payload())["id"])[0] folder = reports.TRACES_PATH / job["session_id"] folder.mkdir(parents=True) (folder / "stderr.log").write_text( "TimeoutError: model request timed out after 300 seconds\n", encoding="utf-8" ) await service.on_terminal({"session_id": job["session_id"]}, "failed") saved = isolated.job(job["session_id"]) assert saved["status"] == "failed" assert "超时" in saved["error"] assert reports.session_report(job["session_id"])["status"] == "failed" assert "超时" in reports.session_report(job["session_id"])["error"] def test_reports_do_not_execute_formulas_or_invent_pass(isolated): batch = service.create_batch(payload()) job = isolated.jobs(batch["id"])[0] isolated.save_job( {**job, "case": {"name": "=1+1", "goal": '=HYPERLINK("x")'}, "status": "completed"} ) report = reports.session_report(job["session_id"]) assert report["verdict"].startswith("未验证") assert reports.verdict({"tests": {"passed": 2, "failed": 1}}) == "验证失败" assert reports.verdict({"tests": {"passed": 2, "unchecked": 1}}) == "验证不完整" paths = reports.export_session(job["session_id"]) wb = load_workbook(next(p for p in paths if p.suffix == ".xlsx")) assert wb["任务结果"]["B3"].data_type == "s" with zipfile.ZipFile(next(p for p in paths if p.suffix == ".docx")) as doc: assert "未验证".encode() in doc.read("word/document.xml") def test_failed_report_exposes_logged_reason_and_formats_chinese_markdown(isolated): job = isolated.jobs(service.create_batch(payload())["id"])[0] isolated.save_job({**job, "status": "failed"}) folder = reports.TRACES_PATH / job["session_id"] (folder / "notes").mkdir(parents=True) (folder / "stdout.log").write_text( "❌ [run] Error running automation: 模型连续 3 次未提供可执行的工具调用\n", encoding="utf-8", ) (folder / "notes" / "output.md").write_text( "## 结果\n- **网络**正常\n\n| 分类 | 状态 |\n| --- | --- |\n| 系统 | 未确认 |", encoding="utf-8", ) report = reports.session_report(job["session_id"]) assert "连续 3 次" in report["error"] assert report["verdict"] == "未验证(任务异常终止)" assert "abc123secret" not in reports.failure_reason( folder, "failed", "认证失败 OPENAI_API_KEY=abc123secret Authorization: Bearer abc123secret" ) assert "超时" not in reports.failure_reason(folder, "failed", "LLMExhaustedError: 额度不足") paths = reports.export_session(job["session_id"]) doc = Document(next(path for path in paths if path.suffix == ".docx")) assert doc.styles["Normal"].font.name == "Microsoft YaHei" assert any("异常原因" in paragraph.text for paragraph in doc.paragraphs) assert any( run.bold and "网络" in run.text for paragraph in doc.paragraphs for run in paragraph.runs ) assert any( cell.text == "分类" for table in doc.tables for row in table.rows for cell in row.cells ) wb = load_workbook(next(path for path in paths if path.suffix == ".xlsx")) assert "连续 3 次" in wb["任务结果"]["B15"].value assert wb["结论"]["B2"].value == "结果" assert wb["结果表1"]["A2"].value == "系统" assert wb["执行证据"]["A1"].font.name == "微软雅黑" assert wb["执行证据"].page_setup.orientation == "landscape" assert reports.execution_result_label({"status": "dispatched"}) == "动作已发出(需核对截图)" assert reports.conclusion_rows("## 结果\n- **通过**检查\n```txt\n| 不是 | 表格 |\n```") == [ ("类型", "结论与发现"), ("小节", "结果"), ("要点", "通过检查"), ("代码", "| 不是 | 表格 |"), ] def test_task_material_zip_contains_ordered_evidence_and_reports( client, isolated, tmp_path, monkeypatch ): job = isolated.jobs(service.create_batch(payload())["id"])[0] isolated.save_job({**job, "case": {**job["case"], "case_id": "1.1"}}) evidence = tmp_path / "evidence" evidence.mkdir() first = evidence / "first.png" second = evidence / "second.png" video = evidence / "recording.mp4" for path in (first, second, video): path.write_bytes(path.name.encode()) monkeypatch.setattr( reports, "session_screenshots", lambda report: [(first, "0001_步骤0001_操作前.png"), (second, "0002_步骤0002_操作后.png")], ) monkeypatch.setattr(reports, "session_videos", lambda report: [(video, "0001_recording.mp4")]) response = client.get(f"/api/automation/reports/{job['session_id']}/all") assert response.status_code == 200 with zipfile.ZipFile(BytesIO(response.content)) as archive: files = archive.namelist() assert all(name.startswith("1.1/") for name in files) assert any(name.endswith(".docx") for name in files) assert any(name.endswith(".xlsx") for name in files) assert files.index("1.1/步骤截图/0001_步骤0001_操作前.png") < files.index( "1.1/步骤截图/0002_步骤0002_操作后.png" ) assert archive.read("1.1/任务录屏/0001_recording.mp4") == b"recording.mp4" manifest = json.loads(archive.read("1.1/文件清单.json")) assert manifest["session_id"] == job["session_id"] assert "1.1-all.zip" in response.headers["content-disposition"] def test_history_filters_all_persisted_rows_and_escapes_like(client, isolated): batch = service.create_batch(payload()) first, second = isolated.jobs(batch["id"]) isolated.save_job( { **first, "status": "completed", "created_at": 100, "case": {"name": "搜索_%", "goal": "abc", "tags": "回归"}, } ) isolated.save_job({**second, "status": "failed", "created_at": 200}) assert client.get("/api/automation/history", params={"q": "_%"}).json()["total"] == 1 assert ( client.get("/api/automation/history", params={"q": "回归"}).json()["items"][0]["session_id"] == first["session_id"] ) assert ( client.get( "/api/automation/history", params={"start": 150, "end": 250, "status": "failed"} ).json()["total"] == 1 ) assert ( client.get("/api/automation/history", params={"size": 1, "page": 2}).json()["items"][0][ "session_id" ] == first["session_id"] ) def test_batch_submission_is_atomic_and_idempotent(client, isolated): value = payload(request_id="same-request") first = client.post("/api/automation/batches", json=value) second = client.post("/api/automation/batches", json=value) assert first.status_code == second.status_code == 200 assert first.json()["id"] == second.json()["id"] assert len(isolated.jobs()) == 2 assert len(isolated.records("batch")) == 1 conflict = client.post("/api/automation/batches", json={**value, "name": "different"}) assert conflict.status_code == 409 lookup = client.get("/api/automation/batches/by-request/same-request").json() assert lookup["id"] == first.json()["id"] assert len(lookup["jobs"]) == 2 def test_removed_replay_is_inert_but_history_survives(client, isolated): batch = service.create_batch(payload()) job = isolated.jobs(batch["id"])[0] isolated.save_job({**job, "execution_mode": "strict", "workflow_id": "legacy"}) service.recover_batches() assert isolated.get("batch", batch["id"])["status"] == "cancelled" assert all(j["status"] == "cancelled" for j in isolated.jobs(batch["id"])) assert client.post(f"/api/automation/batches/{batch['id']}/resume").status_code == 409 assert client.post(f"/api/automation/batches/{batch['id']}/retry_failed").status_code == 409 assert client.get("/api/automation/workflows").status_code == 404 assert client.get("/api/automation/history").json()["total"] == 2