"""Automation acceptance checks with isolated persistence and no real device side effects.""" from io import BytesIO import json import sqlite3 import threading import time from unittest.mock import AsyncMock, Mock from uuid import UUID import zipfile from docx import Document from fastapi import FastAPI from fastapi.testclient import TestClient from openpyxl import load_workbook import pytest from apps.admin_console.routers import automation as api from apps.admin_console.services import automation_service as service from apps.admin_console.services import report_export as reports from apps.admin_console.services import task_import from apps.admin_console.services.automation_store import AutomationStore from artemis.data_engine.storage import StorageManager @pytest.fixture def isolated(tmp_path, monkeypatch): db = tmp_path / "data.db" StorageManager(db_path=db, base_trace_dir=tmp_path) store = AutomationStore(db) for module in (api, service, reports): monkeypatch.setattr(module, "store", store) monkeypatch.setattr(reports, "TRACES_PATH", tmp_path / "traces") monkeypatch.setattr(reports, "EXPORT_ROOT", tmp_path / "exports") from apps.admin_console.database.repositories.session_repository import SessionRepository from apps.admin_console.database.repositories.step_repository import StepRepository monkeypatch.setattr(reports, "SessionRepository", lambda *a: SessionRepository(db)) monkeypatch.setattr(reports, "StepRepository", lambda *a: StepRepository(db)) return store @pytest.fixture def client(isolated): app = FastAPI() app.include_router(api.router) return TestClient(app) def payload(**extra): return { "name": "批次", "cases": [{"name": "用例甲", "goal": "观察设置"}, {"name": "用例乙", "goal": "观察首页"}], "device_serial": "test-device", **extra, } def workbook_bytes(rows): wb = load_workbook(BytesIO(task_import.template())) for row in rows: wb["任务"].append(row) output = BytesIO() wb.save(output) return output.getvalue() def test_template_preview_rejects_formula_and_bad_json(): data = workbook_bytes( [ ["正常", "设置", "冒烟", "打开设置", "", "观察设置", '{"profile":"flash"}'], ["公式", "", "", "", "", "=SUM(1,2)"], ["参数错误", "", "", "", "", "测试", "{"], ] ) result = task_import.parse_workbook(data) assert len(result["tasks"]) == 1 assert [e["row"] for e in result["errors"]] == [3, 4] assert not result["valid"] def test_prompt_only_and_per_row_config_validation(): goal = "先查看屏幕,再按实际画面判断。" data = workbook_bytes( [ [ "CASE-7", "检查项目", "子项", "用例名称", "仅作索引的约束", goal, '{"profile":"pro","model":"mimo-v2.6-flash","record_video":true}', ], ["CASE-8", "", "", "", "", "观察屏幕", '{"OPENAI_API_KEY":"never"}'], ] ) preview = task_import.parse_workbook(data) assert [e["row"] for e in preview["errors"]] == [3] case = preview["tasks"][0] assert task_import.case_goal(case) == goal assert case["preconditions"] == "仅作索引的约束" assert case["task_config"]["record_video"] is True def test_invalid_archive_and_empty_template(): with pytest.raises(ValueError): task_import.parse_workbook(b"not a zip") assert not task_import.parse_workbook(task_import.template())["valid"] def test_pc_json_import_does_not_require_a_device_and_rejects_unknown_targets(): data = workbook_bytes( [ ["PC-1", "电脑", "", "浏览器", "", "检查桌面", '{"execution_target":"computer"}'], ["BAD-2", "", "", "", "", "检查页面", '{"execution_target":"pc"}'], ] ) result = task_import.parse_workbook(data) assert [item["row"] for item in result["errors"]] == [3] options = result["tasks"][0]["task_config"] assert options["execution_target"] == "computer" assert options["device_serial"] is None def test_batch_archive_preparation_reports_progress_and_reuses_ready_file( client, isolated, monkeypatch ): batch = service.create_batch(payload()) batch_id = batch["id"] started = threading.Event() release = threading.Event() calls = [] def slow_export(identifier, progress=None): calls.append(identifier) progress(0, 2, "正在打包任务 1/2") started.set() assert release.wait(5) progress(2, 2, "正在完成压缩包") path = reports.EXPORT_ROOT / "batches" / identifier / "批次报告.zip" path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(b"prepared archive") return path monkeypatch.setattr(reports, "export_batch", slow_export) base = f"/api/automation/batches/{batch_id}/archive" try: response = client.post(base + "/prepare") assert response.status_code == 200 assert response.json()["status"] == "preparing" assert started.wait(2) progress = client.get(base + "/status").json() assert progress["total"] == 2 assert progress["stage"] == "正在打包任务 1/2" assert client.get(base).status_code == 409 finally: release.set() for _ in range(50): if client.get(base + "/status").json()["status"] == "ready": break time.sleep(0.02) assert client.get(base + "/status").json()["status"] == "ready" assert client.get(base).content == b"prepared archive" assert calls == [batch_id] def test_completed_batch_reuses_existing_archive(client, isolated, monkeypatch): batch = service.create_batch(payload()) batch_id = batch["id"] isolated.put("batch", batch_id, {**batch, "status": "completed", "report_ready": True}) path = reports.EXPORT_ROOT / "batches" / batch_id / "批次报告.zip" path.parent.mkdir(parents=True) path.write_bytes(b"existing archive") reports.remember(path, reports.BATCH_ARCHIVE_SIGNATURE) exporter = Mock(side_effect=AssertionError("archive must not be rebuilt")) monkeypatch.setattr(reports, "export_batch", exporter) base = f"/api/automation/batches/{batch_id}/archive" assert client.post(base + "/prepare").json()["status"] == "ready" assert client.get(base).content == b"existing archive" exporter.assert_not_called() def test_legacy_batch_archive_is_rebuilt_with_current_recording_layout(client, isolated): batch = service.create_batch(payload()) batch_id = batch["id"] isolated.put("batch", batch_id, {**batch, "status": "completed", "report_ready": True}) path = reports.EXPORT_ROOT / "batches" / batch_id / "批次报告.zip" path.parent.mkdir(parents=True) path.write_bytes(b"legacy archive") base = f"/api/automation/batches/{batch_id}/archive" assert client.get(base + "/status").json()["status"] == "idle" response = client.get(base) assert response.status_code == 200 assert zipfile.is_zipfile(BytesIO(response.content)) assert client.get(base + "/status").json()["status"] == "ready" def test_preview_does_not_enqueue(client, isolated): response = client.post( "/api/automation/import/preview", files={ "file": ("tasks.xlsx", workbook_bytes([["案例", "", "", "观察页面", "", "观察页面"]])) }, ) assert response.status_code == 200 assert response.json()["valid"] assert not isolated.jobs() def test_preferences_and_required_device(client): assert client.get("/api/automation/preferences").json()["formats"] == ["docx", "xlsx"] assert client.put("/api/automation/preferences", json={"formats": ["xlsx"]}).status_code == 200 assert client.put("/api/automation/preferences", json={"formats": []}).status_code == 422 assert client.post("/api/automation/batches", json=payload(device_serial="")).status_code == 422 def test_batch_repetitions_are_ordered_independent_and_idempotent(client, isolated): request = payload(repeat_count=3, request_id="repeat-test") created = client.post("/api/automation/batches", json=request) assert created.status_code == 200 batch = created.json() assert (batch["case_count"], batch["repeat_count"], batch["total_executions"]) == (2, 3, 6) jobs = isolated.jobs(batch["id"]) assert len({j["session_id"] for j in jobs}) == 6 assert [j["position"] for j in jobs] == list(range(1, 7)) assert [(j["iteration"], j["case_position"]) for j in jobs] == [ (1, 1), (1, 2), (2, 1), (2, 2), (3, 1), (3, 2), ] assert [j["case"]["goal"] for j in jobs] == ["观察设置", "观察首页"] * 3 assert all(j["status"] == "pending" for j in jobs) assert client.post("/api/automation/batches", json=request).json()["id"] == batch["id"] assert len(isolated.jobs(batch["id"])) == 6 assert ( client.post("/api/automation/batches", json={**request, "repeat_count": 2}).status_code == 409 ) # Each selected failure is copied once, including failures from different rounds. for job in (jobs[0], jobs[4]): isolated.save_job({**job, "status": "failed"}) retry = service.batch_action(batch["id"], "retry_failed") assert retry["repeat_count"] == 1 assert len(isolated.jobs(retry["id"])) == 2 report = reports.session_report(jobs[4]["session_id"]) assert "第 3 / 3 轮" in report["name"] archive = reports.export_batch(batch["id"]) with zipfile.ZipFile(archive) as z: assert len([n for n in z.namelist() if n.endswith(".xlsx")]) == 7 assert len(set(z.namelist())) == len(z.namelist()) @pytest.mark.parametrize("count", [0, -1, 1.5, True, "3", None]) def test_batch_rejects_invalid_repetitions(client, isolated, count): assert ( client.post("/api/automation/batches", json=payload(repeat_count=count)).status_code == 422 ) assert isolated.records("batch") == [] def test_batch_limit_is_checked_before_creating_records(client, isolated, monkeypatch): monkeypatch.setattr(service.settings, "ARTEMIS_BATCH_MAX_EXECUTIONS", 5) assert client.get("/api/automation/preferences").json()["batch_max_executions"] == 5 response = client.post("/api/automation/batches", json=payload(repeat_count=3)) assert response.status_code == 409 assert "5" in response.json()["detail"] assert not isolated.records("batch") assert not isolated.jobs() @pytest.mark.asyncio async def test_loop_dispatch_continues_after_failure_and_resume_without_repeating( isolated, monkeypatch ): from apps.admin_console import server from artemis.runtime.device_pool import DeviceStatus batch = service.create_batch(payload(repeat_count=2)) service.batch_action(batch["id"], "resume") monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})()) enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]}) monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue) monkeypatch.setattr( "artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]), ) for index in range(4): await service.tick_batches() await service.tick_batches() # Polling must not enqueue the same execution twice. assert enqueue.await_count == index + 1 job = isolated.jobs(batch["id"])[index] assert enqueue.call_args.kwargs["session_id"] == job["session_id"] assert f"第 {job['iteration']} / 2 轮" in enqueue.call_args.kwargs["display_name"] isolated.save_job({**job, "status": "failed" if index == 0 else "completed"}) if index == 1: service.batch_action(batch["id"], "pause") await service.tick_batches() assert enqueue.await_count == 2 service.batch_action(batch["id"], "resume") await service.tick_batches() assert isolated.get("batch", batch["id"])["status"] == "completed" assert enqueue.await_count == 4 @pytest.mark.asyncio async def test_repeat_fail_fast_skips_future_rounds(isolated): batch = service.create_batch(payload(repeat_count=3, stop_on_failure=True)) first = isolated.jobs(batch["id"])[0] isolated.save_job({**first, "status": "failed"}) service.batch_action(batch["id"], "resume") await service.tick_batches() assert [j["status"] for j in isolated.jobs(batch["id"])] == ["failed"] + ["skipped"] * 5 def test_batch_delete_removes_tasks_and_files_and_prevents_request_replay(client, isolated): created = client.post( "/api/automation/batches", json=payload(request_id="delete-example") ).json() first, second = isolated.jobs(created["id"]) isolated.save_job({**first, "status": "completed"}) trace_dir = reports.TRACES_PATH / first["session_id"] trace_dir.mkdir(parents=True) (trace_dir / "result.json").write_text("{}", encoding="utf-8") report_dir = reports.EXPORT_ROOT / "sessions" / first["session_id"] report_dir.mkdir(parents=True) (report_dir / "任务报告.docx").write_bytes(b"report") batch_dir = reports.EXPORT_ROOT / "batches" / created["id"] batch_dir.mkdir(parents=True) (batch_dir / "批次报告.zip").write_bytes(b"zip") mcp_task_dir = reports.EXPORT_ROOT.parent / "mcp_exports" / "sessions" / first["session_id"] mcp_task_dir.mkdir(parents=True) (mcp_task_dir / "报告.docx").write_bytes(b"report") mcp_batch_dir = reports.EXPORT_ROOT.parent / "mcp_exports" / "batches" / created["id"] mcp_batch_dir.mkdir(parents=True) (mcp_batch_dir / "批次.zip").write_bytes(b"zip") unrelated = reports.TRACES_PATH / "unrelated-task" unrelated.mkdir() (unrelated / "keep.txt").write_text("keep", encoding="utf-8") response = client.delete(f"/api/automation/batches/{created['id']}") assert response.status_code == 200 assert client.get(f"/api/automation/batches/{created['id']}").status_code == 404 assert isolated.job(first["session_id"]) is None assert isolated.job(second["session_id"]) is None assert not trace_dir.exists() assert not report_dir.exists() assert not batch_dir.exists() assert not mcp_task_dir.exists() assert not mcp_batch_dir.exists() assert (unrelated / "keep.txt").read_text(encoding="utf-8") == "keep" with pytest.raises(ValueError, match="不存在"): reports.session_report(first["session_id"]) assert ( client.post( "/api/automation/batches", json=payload(request_id="delete-example") ).status_code == 409 ) @pytest.mark.parametrize("status", ["queued", "running", "task_paused", "resuming", "finalizing"]) def test_batch_delete_rejects_active_jobs(client, isolated, status): batch = service.create_batch(payload()) first = isolated.jobs(batch["id"])[0] isolated.save_job({**first, "status": status}) response = client.delete(f"/api/automation/batches/{batch['id']}") assert response.status_code == 409 assert isolated.get("batch", batch["id"]) is not None assert len(isolated.jobs(batch["id"])) == 2 def test_batch_deletion_reservation_blocks_new_dispatch(isolated): batch = service.create_batch(payload()) job = isolated.jobs(batch["id"])[0] isolated.begin_delete_batch(batch["id"]) with pytest.raises(ValueError, match="正在删除"): isolated.save_job({**job, "status": "queued"}) with pytest.raises(ValueError, match="正在删除"): isolated.put("batch", batch["id"], {**batch, "status": "running"}) assert isolated.jobs_for_sessions([job["session_id"]])[job["session_id"]]["status"] == "pending" def test_batch_deletion_preserves_shared_screenshot_until_last_task(client, isolated): first_batch = service.create_batch(payload()) second_batch = service.create_batch(payload(name="另一批次")) first = isolated.jobs(first_batch["id"])[0] other = isolated.jobs(second_batch["id"])[0] shared_hash = "a" * 64 storage = StorageManager(isolated.path, reports.TRACES_PATH) image_file = reports.TRACES_PATH / "images" / f"{shared_hash}.jpg" image_file.parent.mkdir(parents=True) image_file.write_bytes(b"shared image") with sqlite3.connect(isolated.path) as conn: conn.execute("INSERT INTO images(image_name) VALUES (?)", (shared_hash,)) for job in (first, other): conn.execute( "INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info) " "VALUES (?,?,?,?,?)", (job["session_id"], "观察设置", 1.0, "completed", "{}"), ) conn.execute( "INSERT INTO steps(step_id,session_id,pre_image_name) VALUES (?,?,?)", (job["session_id"], job["session_id"], shared_hash), ) assert client.delete(f"/api/automation/batches/{first_batch['id']}").status_code == 200 assert image_file.read_bytes() == b"shared image" assert storage.get_session(UUID(other["session_id"])) is not None assert client.delete(f"/api/automation/batches/{second_batch['id']}").status_code == 200 assert not image_file.exists() @pytest.mark.parametrize("model_paused", [False, True]) def test_restart_never_repeats_started_work(isolated, model_paused): batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") if model_paused: isolated.put( "batch", batch["id"], {**isolated.get("batch", batch["id"]), "status": "paused", "pause_source": "model"}, ) job = isolated.jobs(batch["id"])[0] isolated.save_job({**job, "status": "queued"}) service.recover_batches() assert isolated.get("batch", batch["id"])["status"] == "paused" assert isolated.get("batch", batch["id"])["pause_source"] == "recovery" assert [j["status"] for j in isolated.jobs(batch["id"])] == ["interrupted", "pending"] retry = service.batch_action(batch["id"], "retry_failed") assert retry["status"] == "draft" assert len(isolated.jobs(retry["id"])) == 1 assert isolated.jobs(retry["id"])[0]["session_id"] != job["session_id"] @pytest.mark.parametrize("liveness_source", ["pid", "device_lock"]) def test_recovery_preserves_live_runner_and_batch(isolated, monkeypatch, liveness_source): import os from types import SimpleNamespace from artemis.runtime import DeviceExecutionLock batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") job = isolated.jobs(batch["id"])[0] isolated.save_job({**job, "status": "running"}) if liveness_source == "pid": with sqlite3.connect(isolated.path) as conn: conn.execute( "INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) " "VALUES (?,?,?,?,?,?)", (job["session_id"], "goal", 1, "running", "{}", os.getpid()), ) monkeypatch.setattr( DeviceExecutionLock, "get_active_owners", lambda: ( {"device": SimpleNamespace(session_id=job["session_id"])} if liveness_source == "device_lock" else {} ), ) service.recover_batches() assert isolated.job(job["session_id"])["status"] == "running" assert not isolated.job(job["session_id"]).get("error") assert isolated.get("batch", batch["id"])["status"] == "running" @pytest.mark.asyncio async def test_natural_completion_resolves_old_recovery_error_in_job_report_and_history( client, isolated ): job = isolated.jobs(service.create_batch(payload())["id"])[0] message = "服务中断;未完成任务需手动重跑" isolated.save_job({**job, "status": "interrupted", "error": message}) await service.on_terminal({"session_id": job["session_id"]}, "completed") saved = isolated.job(job["session_id"]) assert saved["status"] == "completed" assert not saved.get("error") assert saved["recovery_events"][0]["message"] == message assert reports.session_report(job["session_id"])["error"] == "" assert not client.get("/api/automation/history").json()["items"][0].get("error") def test_recovery_repairs_completed_legacy_warning_but_preserves_real_failure(isolated): batch = service.create_batch(payload()) first, second = isolated.jobs(batch["id"]) isolated.save_job({**first, "status": "completed", "error": "服务中断,需手动重跑"}) isolated.save_job({**second, "status": "failed", "error": "任务执行超时(30 秒)"}) assert reports.session_report(first["session_id"])["error"] == "" service.recover_batches() assert not isolated.job(first["session_id"]).get("error") assert "服务中断" in isolated.job(first["session_id"])["recovery_events"][0]["message"] assert "超时" in isolated.job(second["session_id"])["error"] @pytest.mark.asyncio async def test_surviving_worker_is_finalized_when_it_exits_without_resubmission( isolated, monkeypatch ): import os from artemis.runtime import DeviceExecutionLock batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") job = isolated.jobs(batch["id"])[0] isolated.save_job({**job, "status": "running"}) monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {}) with sqlite3.connect(isolated.path) as conn: conn.execute( "INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) " "VALUES (?,?,?,?,?,?)", (job["session_id"], "goal", time.time(), "running", "{}", os.getpid()), ) service.recover_batches() await service.reconcile_recovered_jobs() assert isolated.job(job["session_id"])["status"] == "running" with sqlite3.connect(isolated.path) as conn: conn.execute( "UPDATE sessions SET pid=NULL,status='completed' WHERE session_id=?", (job["session_id"],), ) await service.reconcile_recovered_jobs() saved = isolated.job(job["session_id"]) assert saved["status"] == "completed" assert saved["report_ready"] assert not saved.get("recovery_waiting") assert not saved.get("error") assert len(isolated.jobs(batch["id"])) == 2 assert isolated.jobs(batch["id"])[1]["status"] == "pending" @pytest.mark.asyncio async def test_surviving_worker_that_dies_without_result_is_interrupted(isolated, monkeypatch): from artemis.runtime import DeviceExecutionLock job = isolated.jobs(service.create_batch(payload())["id"])[0] isolated.save_job({**job, "status": "running", "recovery_waiting": True}) monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {}) await service.reconcile_recovered_jobs() saved = isolated.job(job["session_id"]) assert saved["status"] == "interrupted" assert "服务中断" in saved["error"] assert not saved.get("recovery_waiting") @pytest.mark.asyncio async def test_batch_admission_idempotent_and_bound_device(isolated, monkeypatch): from apps.admin_console import server batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})()) enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]}) monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue) monkeypatch.setattr( "artemis.runtime.device_pool.validate_explicit_serial_async", AsyncMock(return_value=None) ) from artemis.runtime.device_pool import DeviceStatus monkeypatch.setattr( "artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]), ) await service.tick_batches() await service.tick_batches() assert enqueue.await_count == 1 assert enqueue.call_args.kwargs["device_serial"] == "test-device" assert isolated.jobs(batch["id"])[1]["status"] == "pending" @pytest.mark.asyncio async def test_paused_case_yields_batch_slot_to_next_case(isolated, monkeypatch): from apps.admin_console import server from artemis.runtime.device_pool import DeviceStatus batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") first = isolated.jobs(batch["id"])[0] isolated.save_job({**first, "status": "task_paused", "started_at": time.time()}) monkeypatch.setattr( service, "state", type("State", (), {"is_paused": False, "active_runs": {}})() ) enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]}) monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue) monkeypatch.setattr( "artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]), ) await service.tick_batches() assert enqueue.await_count == 1 assert enqueue.call_args.args[0] == ["观察首页"] assert [job["status"] for job in isolated.jobs(batch["id"])] == ["task_paused", "queued"] @pytest.mark.asyncio async def test_device_outage_pauses_without_consuming_pending_case(isolated, monkeypatch): batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})()) monkeypatch.setattr( "artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[]) ) await service.tick_batches() assert isolated.get("batch", batch["id"])["status"] == "paused" assert all(job["status"] == "pending" for job in isolated.jobs(batch["id"])) def test_stop_all_overrides_model_auto_resume(isolated, monkeypatch): from apps.admin_console.services import task_queue_service as queue batch = service.create_batch(payload()) isolated.put("batch", batch["id"], {**batch, "status": "paused", "pause_source": "model"}) monkeypatch.setattr("apps.admin_console.services.automation_store.store", isolated) monkeypatch.setattr(queue, "state", Mock(queue_items=[], active_connections={})) for method in ( "_terminate_all_device_owners", "_kill_all_local_runs", "_clear_pause_file", "ensure_worker_running", ): monkeypatch.setattr(queue.TaskQueueService, method, Mock()) assert queue.TaskQueueService._stop_all_tasks() saved = isolated.get("batch", batch["id"]) assert saved["pause_source"] == "user" assert "等待手动继续" in saved["reason"] @pytest.mark.asyncio @pytest.mark.parametrize("manual_pause", [False, True]) @pytest.mark.parametrize("stop_on_failure", [False, True]) async def test_model_pause_recovery_preserves_user_pause_and_failure_policy( isolated, monkeypatch, manual_pause, stop_on_failure ): from apps.admin_console import server from artemis.runtime.device_pool import DeviceStatus batch = service.create_batch(payload(stop_on_failure=stop_on_failure)) service.batch_action(batch["id"], "resume") first, second = isolated.jobs(batch["id"]) isolated.save_job({**first, "status": "running", "started_at": time.time()}) pause_state = type("State", (), {"is_paused": True, "active_runs": {}})() monkeypatch.setattr(service, "state", pause_state) enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]}) monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue) monkeypatch.setattr( "artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]), ) monkeypatch.setattr(reports, "export_batch", Mock()) await service.tick_batches() assert isolated.get("batch", batch["id"])["pause_source"] == "model" if manual_pause: service.batch_action(batch["id"], "pause") # Another paused-model tick must not overwrite the explicit user pause. await service.tick_batches() assert isolated.get("batch", batch["id"])["pause_source"] == "user" isolated.save_job({**first, "status": "failed"}) pause_state.is_paused = False await service.tick_batches() assert isolated.job(first["session_id"])["status"] == "failed" expected = "pending" if manual_pause else "skipped" if stop_on_failure else "queued" assert isolated.job(second["session_id"])["status"] == expected assert enqueue.await_count == (0 if manual_pause or stop_on_failure else 1) @pytest.mark.asyncio @pytest.mark.parametrize("code", ["initialization_failed", "execution_failed"]) async def test_failure_reason_retained_and_only_initialization_blocks_remaining_batch( isolated, monkeypatch, code ): from apps.admin_console import server from artemis.runtime import task_result from artemis.runtime.device_pool import DeviceStatus batch = service.create_batch(payload()) service.batch_action(batch["id"], "resume") first, second = isolated.jobs(batch["id"]) isolated.save_job({**first, "status": "running"}) monkeypatch.setattr(task_result, "TRACES_PATH", reports.TRACES_PATH) task_result.save_failure_result( first["session_id"], "PermissionError: awake lock denied", code=code ) monkeypatch.setattr(reports, "export_session", Mock(side_effect=OSError("export unavailable"))) monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})()) enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]}) monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue) monkeypatch.setattr( "artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]), ) await service.on_terminal({"session_id": first["session_id"]}, "failed") await service.tick_batches() saved = isolated.job(first["session_id"]) assert saved["error"] == "PermissionError: awake lock denied" assert saved["report_error"] == "export unavailable" if code == "initialization_failed": assert isolated.get("batch", batch["id"])["pause_source"] == "initialization" assert isolated.job(second["session_id"])["status"] == "pending" enqueue.assert_not_awaited() service.batch_action(batch["id"], "resume") await service.tick_batches() assert enqueue.await_count == 1 assert enqueue.call_args.kwargs["session_id"] == second["session_id"] assert isolated.job(first["session_id"])["status"] == "failed" @pytest.mark.asyncio async def test_fail_fast_includes_skipped_reports(isolated, monkeypatch): batch = service.create_batch(payload(stop_on_failure=True)) first = isolated.jobs(batch["id"])[0] isolated.save_job({**first, "status": "failed"}) service.batch_action(batch["id"], "resume") await service.tick_batches() assert [j["status"] for j in isolated.jobs(batch["id"])] == ["failed", "skipped"] archive = reports.export_batch(batch["id"]) with zipfile.ZipFile(archive) as z: assert len(z.namelist()) == 10 # summaries + reports, diagnostics and evidence per task diagnostics = [name for name in z.namelist() if name.endswith("模型请求诊断.json")] assert len(diagnostics) == 2 for name in diagnostics: assert json.loads(z.read(name))["schema_version"] == 1 assert len([name for name in z.namelist() if name.endswith("证据索引.json")]) == 2 assert sum(n.endswith(".docx") for n in z.namelist()) == 3 def test_batch_archive_contains_each_task_media_when_present(isolated, monkeypatch, tmp_path): batch = service.create_batch(payload()) first, second = isolated.jobs(batch["id"]) image1 = tmp_path / "first.png" image2 = tmp_path / "second.png" video = tmp_path / "recording.mp4" image1.write_bytes(b"first screenshot") image2.write_bytes(b"second screenshot") video.write_bytes(b"saved recording") images = {first["session_id"]: image1, second["session_id"]: image2} monkeypatch.setattr( reports, "session_screenshots", lambda report: [(images[report["id"]], "0001_步骤0001_操作前.png")], ) monkeypatch.setattr( reports, "session_videos", lambda report: ( [(video, "0001_recording.mp4")] if report["id"] == first["session_id"] else [] ), ) archive = reports.export_batch(batch["id"]) with zipfile.ZipFile(archive) as z: names = z.namelist() assert len([name for name in names if "/步骤截图/" in name]) == 2 assert len([name for name in names if "/任务录屏/" in name]) == 1 assert all(name.startswith("任务报告/") for name in names if "步骤截图" in name) assert z.read(next(name for name in names if "/任务录屏/" in name)) == b"saved recording" @pytest.mark.asyncio async def test_report_failure_does_not_strand_job(isolated, monkeypatch): job = isolated.jobs(service.create_batch(payload())["id"])[0] monkeypatch.setattr( reports, "export_session", lambda *a: (_ for _ in ()).throw(OSError("disk full")) ) await service.on_terminal({"session_id": job["session_id"]}, "completed") saved = isolated.job(job["session_id"]) assert saved["status"] == "completed" assert "disk full" in saved["report_error"] @pytest.mark.asyncio async def test_batch_timeout_stays_visible_in_job_report_and_history(client, isolated): job = isolated.jobs(service.create_batch(payload())["id"])[0] isolated.save_job( {**job, "status": "running", "timed_out": True, "error": "任务执行超时(30 秒)"} ) await service.on_terminal({"session_id": job["session_id"]}, "cancelled") saved = isolated.job(job["session_id"]) assert saved["status"] == "failed" assert "超时" in saved["error"] assert "超时" in reports.session_report(job["session_id"])["error"] history = client.get("/api/automation/history").json()["items"] assert any( item["session_id"] == job["session_id"] and "超时" in item["error"] for item in history ) @pytest.mark.asyncio @pytest.mark.parametrize("stop_on_failure", [False, True]) async def test_timed_out_case_releases_next_batch_case(isolated, monkeypatch, stop_on_failure): from apps.admin_console import server from artemis.runtime.device_pool import DeviceStatus batch = service.create_batch(payload(timeout_seconds=1, stop_on_failure=stop_on_failure)) service.batch_action(batch["id"], "resume") first = isolated.jobs(batch["id"])[0] isolated.save_job({**first, "status": "running", "started_at": time.time() - 2}) monkeypatch.setattr( service, "state", type("State", (), {"is_paused": False, "active_runs": {}})() ) stop = Mock(return_value=True) enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]}) monkeypatch.setattr(server.task_queue_service, "stop_tasks", stop) monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue) monkeypatch.setattr( "artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]), ) monkeypatch.setattr(reports, "export_batch", Mock()) await service.tick_batches() assert isolated.get("batch", batch["id"])["status"] == "running" assert "超时" in isolated.job(first["session_id"])["error"] stop.assert_called_once_with(session_id=first["session_id"]) await service.tick_batches() stop.assert_called_once() await service.on_terminal({"session_id": first["session_id"]}, "cancelled") await service.tick_batches() statuses = [job["status"] for job in isolated.jobs(batch["id"])] assert statuses == (["failed", "skipped"] if stop_on_failure else ["failed", "queued"]) assert enqueue.await_count == (0 if stop_on_failure else 1) @pytest.mark.asyncio async def test_queued_case_does_not_consume_execution_timeout(isolated, monkeypatch): from apps.admin_console import server batch = service.create_batch(payload(timeout_seconds=1)) service.batch_action(batch["id"], "resume") first = isolated.jobs(batch["id"])[0] isolated.save_job({**first, "status": "queued", "started_at": time.time() - 30}) monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})()) stop = Mock() monkeypatch.setattr(server.task_queue_service, "stop_tasks", stop) await service.tick_batches() stop.assert_not_called() assert isolated.job(first["session_id"])["status"] == "queued" assert isolated.get("batch", batch["id"])["status"] == "running" @pytest.mark.asyncio async def test_model_timeout_from_runner_log_reaches_failed_task_and_report(isolated): job = isolated.jobs(service.create_batch(payload())["id"])[0] folder = reports.TRACES_PATH / job["session_id"] folder.mkdir(parents=True) (folder / "stderr.log").write_text( "TimeoutError: model request timed out after 300 seconds\n", encoding="utf-8" ) await service.on_terminal({"session_id": job["session_id"]}, "failed") saved = isolated.job(job["session_id"]) assert saved["status"] == "failed" assert "超时" in saved["error"] assert reports.session_report(job["session_id"])["status"] == "failed" assert "超时" in reports.session_report(job["session_id"])["error"] def test_optional_timeout_warnings_are_not_reported_as_task_failure(tmp_path): (tmp_path / "stdout.log").write_text( "⚠ Could not clean unowned screen wake locks on emulator-5554: " "Timed out acquiring the Artemis awake-lease mutex.\n" "⚠ VisualStepSummarizer: Error generating summary for step 3: \n" "ℹ StepMemoryService: Retrying Step 3 (attempt 4) after 1.0s\n", encoding="utf-8", ) assert reports.failure_reason(tmp_path, "failed", None) == ( "任务未完成,运行日志中没有可提取的具体异常;请查看执行步骤。" ) (tmp_path / "stderr.log").write_text( "TimeoutError: model request timed out after 300 seconds\n", encoding="utf-8" ) reason = reports.failure_reason(tmp_path, "failed", None) assert "model request timed out" in reason assert "awake-lease" not in reason (tmp_path / "stderr.log").write_text( "DeviceBusyError: device queue reservation was cancelled\n", encoding="utf-8" ) assert reports.failure_reason(tmp_path, "failed", None) == ( "DeviceBusyError: device queue reservation was cancelled" ) def test_reports_do_not_execute_formulas_or_invent_pass(isolated): batch = service.create_batch(payload()) job = isolated.jobs(batch["id"])[0] isolated.save_job( {**job, "case": {"name": "=1+1", "goal": '=HYPERLINK("x")'}, "status": "completed"} ) report = reports.session_report(job["session_id"]) assert report["verdict"].startswith("未验证") assert reports.verdict({"tests": {"passed": 2, "failed": 1}}) == "验证失败" assert reports.verdict({"tests": {"passed": 2, "unchecked": 1}}) == "验证不完整" paths = reports.export_session(job["session_id"]) wb = load_workbook(next(p for p in paths if p.suffix == ".xlsx")) assert wb["任务结果"]["B3"].data_type == "s" with zipfile.ZipFile(next(p for p in paths if p.suffix == ".docx")) as doc: assert "未验证".encode() in doc.read("word/document.xml") @pytest.mark.parametrize("result_status", ["passed", "failed", "blocked", "inconclusive"]) def test_report_without_output_body_uses_effective_verdict_and_evidence(isolated, result_status): job = isolated.jobs(service.create_batch(payload())["id"])[0] isolated.save_job({**job, "status": "completed"}) folder = reports.TRACES_PATH / job["session_id"] folder.mkdir(parents=True) (folder / "run_outcome.json").write_text( json.dumps( { "task_status": "completed", "test_result": { "status": result_status, "reason": "实际验收依据", "evidence": ["记录中的证据"], }, } ), encoding="utf-8", ) report = reports.session_report(job["session_id"]) assert report["verdict"] in report["content"] assert "实际验收依据" in report["content"] assert "记录中的证据" in report["content"] assert "未保存独立输出正文" in report["content"] assert "没有保存最终结论" not in report["content"] assert report["test_result"]["status"] == result_status def test_report_without_output_body_keeps_human_review_authoritative(isolated): job = isolated.jobs(service.create_batch(payload())["id"])[0] isolated.save_job({**job, "status": "completed"}) folder = reports.TRACES_PATH / job["session_id"] folder.mkdir(parents=True) (folder / "run_outcome.json").write_text( json.dumps( { "test_result": {"status": "passed", "reason": "AI 原判通过"}, } ), encoding="utf-8", ) isolated.save_case_review(job["session_id"], "failed", "用户确认实际结果不符合预期") report = reports.session_report(job["session_id"]) assert "人工确认" in report["content"] assert "用户确认实际结果不符合预期" in report["content"] assert "AI 原判通过" not in report["content"] assert report["test_result"]["status"] == "failed" def test_report_projects_stored_android_receipts_without_rewriting_history(isolated): from tests.unit.data_engine.test_evidence_sources import android_receipt sid = isolated.jobs(service.create_batch(payload())["id"])[0]["session_id"] receipt = android_receipt()["payload"]["result"] raw_payload = json.dumps({"args": {"action": "screenshot"}, "result": json.dumps(receipt)}) with sqlite3.connect(isolated.path) as conn: conn.execute( "INSERT INTO sessions(session_id,initial_goal,start_time,status) VALUES (?,?,?,?)", (sid, "跨端截图", 1, "completed"), ) conn.execute( "INSERT INTO steps(step_id,session_id,step_number) VALUES (?,?,?)", ("s", sid, 1) ) conn.execute( "INSERT INTO traces(trace_id,session_id,step_id,type,name,timestamp,status,payload) VALUES (?,?,?,?,?,?,?,?)", ("capture", sid, "s", "tool", "android_file", 2, "success", raw_payload), ) report = reports.session_report(sid) source = report["evidence_sources"]["sources"][0] assert source["kind"] == "android" assert source["references"][0]["trace_id"] == "capture" assert report["evidence_sources"]["unattributed_step_count"] == 0 with sqlite3.connect(isolated.path) as conn: assert ( conn.execute("SELECT payload FROM traces WHERE trace_id='capture'").fetchone()[0] == raw_payload ) def test_failed_report_exposes_logged_reason_and_formats_chinese_markdown(isolated): job = isolated.jobs(service.create_batch(payload())["id"])[0] isolated.save_job({**job, "status": "failed"}) folder = reports.TRACES_PATH / job["session_id"] (folder / "notes").mkdir(parents=True) (folder / "stdout.log").write_text( "❌ [run] Error running automation: 模型连续 3 次未提供可执行的工具调用\n", encoding="utf-8", ) (folder / "notes" / "output.md").write_text( "## 结果\n- **网络**正常\n\n| 分类 | 状态 |\n| --- | --- |\n| 系统 | 未确认 |", encoding="utf-8", ) report = reports.session_report(job["session_id"]) assert "连续 3 次" in report["error"] assert report["verdict"] == "测试阻塞(AI确认)" assert report["test_result"]["status"] == "blocked" assert "连续 3 次" in report["test_result"]["reason"] assert "abc123secret" not in reports.failure_reason( folder, "failed", "认证失败 OPENAI_API_KEY=abc123secret Authorization: Bearer abc123secret" ) assert "超时" not in reports.failure_reason(folder, "failed", "LLMExhaustedError: 额度不足") paths = reports.export_session(job["session_id"]) doc = Document(next(path for path in paths if path.suffix == ".docx")) assert doc.styles["Normal"].font.name == "Microsoft YaHei" assert any("异常原因" in paragraph.text for paragraph in doc.paragraphs) assert any( run.bold and "网络" in run.text for paragraph in doc.paragraphs for run in paragraph.runs ) assert any( cell.text == "分类" for table in doc.tables for row in table.rows for cell in row.cells ) wb = load_workbook(next(path for path in paths if path.suffix == ".xlsx")) assert any( row[0].value == "异常原因" and "连续 3 次" in row[1].value for row in wb["任务结果"].iter_rows() ) assert wb["结论"]["B2"].value == "结果" assert wb["结果表1"]["A2"].value == "系统" assert wb["执行证据"]["A1"].font.name == "微软雅黑" assert wb["执行证据"].page_setup.orientation == "landscape" assert reports.execution_result_label({"status": "dispatched"}) == "动作已发出(需核对截图)" assert reports.conclusion_rows("## 结果\n- **通过**检查\n```txt\n| 不是 | 表格 |\n```") == [ ("类型", "结论与发现"), ("小节", "结果"), ("要点", "通过检查"), ("代码", "| 不是 | 表格 |"), ] def test_prompt_based_verdict_is_consistent_in_history_batch_and_reports( client, isolated, monkeypatch ): from artemis.runtime import case_result monkeypatch.setattr(case_result, "TRACES_PATH", reports.TRACES_PATH) batch = service.create_batch(payload()) job = isolated.jobs(batch["id"])[0] isolated.save_job( {**job, "status": "completed", "started_at": time.time() - 50, "ended_at": time.time()} ) result = { "status": "failed", "reason": "预期出现成功提示,实际显示错误", "evidence": ["步骤 3 点击后截图显示错误提示"], } folder = reports.TRACES_PATH / job["session_id"] folder.mkdir(parents=True) (folder / "run_outcome.json").write_text( json.dumps( {"task_status": "completed", "tests": {}, "test_result": result}, ensure_ascii=False ), encoding="utf-8", ) history = client.get("/api/automation/history").json()["items"][0] batch_job = client.get(f"/api/automation/batches/{batch['id']}").json()["jobs"][0] report = client.get(f"/api/automation/reports/{job['session_id']}").json() for item in (history, batch_job, report): assert item["status"] == "completed" assert item["test_result"] == {**result, "source": "ai"} assert report["verdict"] == "测试不通过(AI确认)" paths = reports.export_session(job["session_id"]) doc = Document(next(path for path in paths if path.suffix == ".docx")) doc_text = "\n".join( cell.text for table in doc.tables for row in table.rows for cell in row.cells ) doc_text += "\n" + "\n".join(paragraph.text for paragraph in doc.paragraphs) assert result["reason"] in doc_text assert result["evidence"][0] in doc_text wb = load_workbook(next(path for path in paths if path.suffix == ".xlsx")) values = [cell.value for row in wb["任务结果"].iter_rows() for cell in row] assert "测试不通过(AI确认)" in values assert result["reason"] in values def test_human_review_overrides_every_report_preserves_ai_and_invalidates_batch_cache( client, isolated, monkeypatch ): from artemis.runtime import case_result from apps.admin_console.routers import sessions as sessions_api monkeypatch.setattr(case_result, "TRACES_PATH", reports.TRACES_PATH) batch = service.create_batch(payload()) jobs = isolated.jobs(batch["id"]) for job in jobs: isolated.save_job({**job, "status": "completed", "ended_at": time.time()}) session_id = jobs[0]["session_id"] ai = {"status": "failed", "reason": "AI识别到预期不匹配", "evidence": ["步骤 3 截图"]} directory = reports.TRACES_PATH / session_id directory.mkdir(parents=True) outcome_file = directory / "run_outcome.json" original = json.dumps({"task_status": "completed", "test_result": ai}, ensure_ascii=False) outcome_file.write_text(original, encoding="utf-8") archive = reports.export_batch(batch["id"]) isolated.put("batch", batch["id"], {**batch, "report_ready": True}) response = client.put( f"/api/automation/reports/{session_id}/review", json={"status": "passed", "reason": "已人工核对,符合预期"}, ) assert response.status_code == 200 assert response.json()["status"] == "completed" assert response.json()["verdict"] == "测试通过(人工确认)" assert outcome_file.read_text(encoding="utf-8") == original assert not archive.exists() assert not isolated.get("batch", batch["id"])["report_ready"] assert len(isolated.get("case_review", session_id)["reviews"]) == 1 for result in ( response.json(), client.get("/api/automation/history", params={"q": session_id}).json()["items"][0], client.get(f"/api/automation/batches/{batch['id']}").json()["jobs"][0], ): assert result["test_result"]["source"] == "human" assert result["test_result"]["status"] == "passed" assert result["test_result"]["ai_result"] == {**ai, "source": "ai"} assert result["test_result"]["reviewed_at"] > 0 monkeypatch.setattr(sessions_api, "store", isolated) monkeypatch.setattr(sessions_api, "TRACES_PATH", reports.TRACES_PATH) monkeypatch.setattr( sessions_api.session_repo, "get_session_by_id", lambda sid: {"session_id": sid, "status": "completed"}, ) client.app.include_router(sessions_api.router) detail = client.get(f"/api/sessions/{session_id}").json() assert detail["test_result"]["source"] == "human" assert detail["batch_id"] == batch["id"] downloaded = client.get(f"/api/automation/batches/{batch['id']}/archive") assert downloaded.status_code == 200 with zipfile.ZipFile(BytesIO(downloaded.content)) as bundle: wb = load_workbook(BytesIO(bundle.read("批次结果.xlsx"))) assert wb.active["D2"].value == "测试通过(人工确认)" assert "AI确认" in wb.active["D3"].value doc = Document(BytesIO(bundle.read("批次汇总.docx"))) assert doc.tables[0].rows[1].cells[3].text == "测试通过(人工确认)" changed = client.put(f"/api/automation/reports/{session_id}/review", json={"status": "failed"}) assert changed.status_code == 200 assert changed.json()["verdict"] == "测试不通过(人工确认)" assert len(isolated.get("case_review", session_id)["reviews"]) == 2 assert outcome_file.read_text(encoding="utf-8") == original def test_skipped_flash_review_is_consistent_in_history_reports_and_human_review( client, isolated, monkeypatch ): from artemis.runtime import case_result monkeypatch.setattr(case_result, "TRACES_PATH", reports.TRACES_PATH) batch = service.create_batch(payload()) job = isolated.jobs(batch["id"])[0] isolated.save_job({**job, "status": "completed"}) folder = reports.TRACES_PATH / job["session_id"] folder.mkdir(parents=True) result = { "status": "inconclusive", "reason": "执行完成 · 未做 AI 判定", "evidence": [], "verification_skipped": True, } (folder / "run_outcome.json").write_text( json.dumps({"task_status": "completed", "tests": {}, "test_result": result}), encoding="utf-8", ) for item in ( client.get("/api/automation/history").json()["items"][0], client.get(f"/api/automation/batches/{batch['id']}").json()["jobs"][0], client.get(f"/api/automation/reports/{job['session_id']}").json(), ): assert item["status"] == "completed" assert item["test_result"] == {**result, "source": "system"} assert reports.session_report(job["session_id"])["verdict"] == "未做 AI 判定" paths = reports.export_session(job["session_id"]) workbook = load_workbook(next(path for path in paths if path.suffix == ".xlsx")) assert "未做 AI 判定" in [ cell.value for row in workbook["任务结果"].iter_rows() for cell in row ] doc = Document(next(path for path in paths if path.suffix == ".docx")) assert "未做 AI 判定" in "\n".join(paragraph.text for paragraph in doc.paragraphs) reviewed = client.put( f"/api/automation/reports/{job['session_id']}/review", json={"status": "passed", "reason": "已人工核对"}, ).json() assert reviewed["test_result"]["source"] == "human" assert reviewed["test_result"]["ai_result"] == {**result, "source": "system"} assert reviewed["verdict"] == "测试通过(人工确认)" def test_review_rejects_active_missing_invalid_and_packing_tasks(client, isolated, monkeypatch): job = isolated.jobs(service.create_batch(payload())["id"])[0] url = f"/api/automation/reports/{job['session_id']}/review" assert client.put(url, json={"status": "passed"}).status_code == 409 assert isolated.get("case_review", job["session_id"]) is None assert client.put(url, json={"status": "inconclusive"}).status_code == 422 assert client.put(url, json={"status": "passed", "source": "human"}).status_code == 422 assert client.put(url, json={"status": "passed", "reason": "字" * 2001}).status_code == 422 assert ( client.put("/api/automation/reports/missing/review", json={"status": "passed"}).status_code == 404 ) isolated.save_job({**job, "status": "completed"}) monkeypatch.setattr(api.archive_service, "status", lambda *args: {"status": "preparing"}) assert client.put(url, json={"status": "passed"}).status_code == 409 assert isolated.get("case_review", job["session_id"]) is None def test_batch_filter_searches_full_history_and_reviews_are_removed_with_batch(client, isolated): first, second = [service.create_batch(payload(name=name)) for name in ("第一批", "第二批")] for batch in (first, second): for job in isolated.jobs(batch["id"]): isolated.save_job({**job, "status": "completed"}) with isolated.connection() as conn: conn.execute( "INSERT INTO sessions(session_id,initial_goal,start_time,status) VALUES (?,?,?,?)", ("single-case", "单独检查", time.time(), "completed"), ) conn.commit() filtered = client.get( "/api/automation/history", params={"batch_id": first["id"], "q": "用例", "size": 1, "page": 2}, ).json() assert filtered["total"] == 2 assert len(filtered["items"]) == 1 assert filtered["items"][0]["batch_id"] == first["id"] assert ( client.get("/api/automation/history", params={"batch_id": "__batch__"}).json()["total"] == 4 ) singles = client.get("/api/automation/history", params={"batch_id": "__single__"}).json() assert [item["session_id"] for item in singles["items"]] == ["single-case"] assert client.get("/api/automation/history", params={"batch_id": "bad/path"}).status_code == 400 reviewed_id = isolated.jobs(first["id"])[0]["session_id"] isolated.save_case_review(reviewed_id, "passed", "人工核对") isolated.begin_delete_batch(first["id"]) isolated.delete_batch(first["id"]) assert isolated.get("case_review", reviewed_id) is None assert len(isolated.jobs(second["id"])) == 2 isolated.save_case_review("single-case", "failed", "不符合预期") isolated.delete_history("single-case") assert isolated.get("case_review", "single-case") is None def test_task_material_zip_contains_ordered_evidence_and_reports( client, isolated, tmp_path, monkeypatch ): job = isolated.jobs(service.create_batch(payload())["id"])[0] isolated.save_job({**job, "case": {**job["case"], "case_id": "1.1"}}) artifact_dir = reports.TRACES_PATH / job["session_id"] / "artifacts" / "android" artifact_dir.mkdir(parents=True) (artifact_dir / "安卓截图.png").write_bytes(b"native png") (artifact_dir / ".artemis-partial.tmp").write_bytes(b"incomplete") evidence = tmp_path / "evidence" evidence.mkdir() first = evidence / "first.png" second = evidence / "second.png" video = evidence / "recording.mp4" for path in (first, second, video): path.write_bytes(path.name.encode()) monkeypatch.setattr( reports, "session_screenshots", lambda report: [(first, "0001_步骤0001_操作前.png"), (second, "0002_步骤0002_操作后.png")], ) monkeypatch.setattr(reports, "session_videos", lambda report: [(video, "0001_recording.mp4")]) response = client.get(f"/api/automation/reports/{job['session_id']}/all") assert response.status_code == 200 with zipfile.ZipFile(BytesIO(response.content)) as archive: files = archive.namelist() assert all(name.startswith("1.1/") for name in files) assert any(name.endswith(".docx") for name in files) assert any(name.endswith(".xlsx") for name in files) assert files.index("1.1/步骤截图/0001_步骤0001_操作前.png") < files.index( "1.1/步骤截图/0002_步骤0002_操作后.png" ) assert archive.read("1.1/任务录屏/安卓/0001_recording.mp4") == b"recording.mp4" manifest = json.loads(archive.read("1.1/文件清单.json")) assert manifest["session_id"] == job["session_id"] assert manifest["artifacts"] == ["安卓截图.png"] assert archive.read("1.1/任务文件/安卓/安卓截图.png") == b"native png" assert not any("partial" in name for name in files) assert "1.1-all.zip" in response.headers["content-disposition"] def test_batch_zip_contains_each_tasks_android_artifacts(isolated): batch = service.create_batch(payload()) for index, job in enumerate(isolated.jobs(batch["id"])): root = reports.TRACES_PATH / job["session_id"] / "artifacts" / "android" root.mkdir(parents=True) (root / "结果.txt").write_text(f"任务{index}", encoding="utf-8") with zipfile.ZipFile(reports.export_batch(batch["id"])) as archive: names = [name for name in archive.namelist() if name.endswith("任务文件/安卓/结果.txt")] assert len(names) == 2 assert {archive.read(name).decode() for name in names} == {"任务0", "任务1"} def test_all_material_cache_reuses_reports_and_invalidates_on_review_or_new_files( isolated, monkeypatch ): job = isolated.jobs(service.create_batch(payload())["id"])[0] job["status"] = "completed" isolated.save_job(job) word = Mock(wraps=reports.word_report) monkeypatch.setattr(reports, "word_report", word) monkeypatch.setattr( reports, "diagnostics_json", lambda sid: json.dumps({"requests": [], "collected_at": time.time()}), ) first = reports.export_session_media(job["session_id"], "all") first_bytes = first.read_bytes() reports.export_session_media(job["session_id"], "all") assert word.call_count == 1 assert first.read_bytes() == first_bytes isolated.save_case_review(job["session_id"], "passed", "人工核对") reports.export_session_media(job["session_id"], "all") assert word.call_count == 2 folder = reports.TRACES_PATH / job["session_id"] / "artifacts" / "android" folder.mkdir(parents=True) (folder / "新资料.txt").write_text("new evidence", encoding="utf-8") reports.export_session_media(job["session_id"], "all") with zipfile.ZipFile(first) as archive: assert any(name.endswith("任务文件/安卓/新资料.txt") for name in archive.namelist()) assert not json.loads( archive.read(next(n for n in archive.namelist() if n.endswith("文件清单.json"))) )["snapshot"] # File-only changes rebuild the ZIP while keeping unchanged documents. assert word.call_count == 2 def test_different_tasks_export_concurrently_and_running_exports_are_snapshots( isolated, monkeypatch ): from concurrent.futures import ThreadPoolExecutor jobs = isolated.jobs(service.create_batch(payload())["id"]) barrier = threading.Barrier(2) def generate(report, path): barrier.wait(timeout=5) path.write_bytes(b"generated report") monkeypatch.setattr(reports, "word_report", generate) with ThreadPoolExecutor(max_workers=2) as pool: exports = [pool.submit(reports.export_session, j["session_id"], ["docx"]) for j in jobs] assert all(result.result(timeout=10)[0].is_file() for result in exports) count = Mock(side_effect=lambda report, path: path.write_bytes(b"snapshot")) monkeypatch.setattr(reports, "word_report", count) for _ in range(2): reports.export_session(jobs[0]["session_id"], ["docx"]) assert count.call_count == 2 def test_android_artifact_export_does_not_follow_other_session_links(isolated): root = reports.TRACES_PATH / "session" / "artifacts" root.mkdir(parents=True) other = reports.TRACES_PATH / "other-session" other.mkdir() (other / "private.txt").write_text("other task") try: (root / "android").symlink_to(other, target_is_directory=True) except OSError: pytest.skip("symlinks unavailable") assert reports.session_artifacts({"id": "session"}) == [] def test_report_source_inventory_preserves_human_verdict_and_raw_evidence( isolated, tmp_path, monkeypatch ): from apps.admin_console.services import evidence_service, task_inspection job = isolated.jobs(service.create_batch(payload())["id"])[0] sid = job["session_id"] action = { "action": "plug_read", "plugin_id": "data-reader", "call_id": "recorded-call", "read_only": True, "source": {"kind": "api", "target_id": "business-service", "label": "安卓业务后台"}, } with sqlite3.connect(isolated.path) as conn: conn.execute( "INSERT INTO sessions(session_id,initial_goal,start_time,end_time,status) VALUES (?,?,?,?,?)", (sid, "检查业务状态", 1700000010, 1700000120, "completed"), ) conn.execute( "INSERT INTO steps(step_id,session_id,step_number,action_taken) VALUES (?,?,?,?)", ("actual-step", sid, 7, json.dumps(action)), ) isolated.save_case_review(sid, "failed", "用户指出业务状态仍未落库") monkeypatch.setattr(evidence_service, "TRACES_PATH", reports.TRACES_PATH) report = reports.session_report(sid) source = report["evidence_sources"]["sources"][0] assert source["category"] == "auxiliary" assert source["references"][0]["step_id"] == "actual-step" assert report["test_result"]["status"] == "failed" assert report["test_result"]["source"] == "human" assert report["steps"][0]["action_taken"] == action index = evidence_service.export_index(sid, report, []) assert index["evidence_sources"] == report["evidence_sources"] monkeypatch.setattr(task_inspection, "store", isolated) monkeypatch.setattr(task_inspection, "session_repo", reports.SessionRepository()) assert ( task_inspection.inspect_task(sid, "report")["evidence_sources"] == report["evidence_sources"] ) from docx import Document from openpyxl import load_workbook word = tmp_path / "sources.docx" sheet = tmp_path / "sources.xlsx" reports.word_report(report, word) reports.excel_report(report, sheet) document = Document(word) word_text = " ".join( [p.text for p in document.paragraphs] + [cell.text for table in document.tables for row in table.rows for cell in row.cells] ) workbook = load_workbook(sheet) try: excel_text = " ".join( str(cell.value or "") for ws in workbook for row in ws for cell in row ) finally: workbook.close() for text in (word_text, excel_text): assert "接口数据" in text assert "安卓业务后台" in text assert "actual-step" in text assert "用户指出业务状态仍未落库" in text def test_reports_share_lifecycle_and_distinguish_guidance_receipt_from_consumption( isolated, tmp_path, monkeypatch ): from apps.admin_console.services import task_lifecycle from artemis.runtime import task_control job = isolated.jobs(service.create_batch(payload())["id"])[0] job.update( status="completed", created_at=1700000010, worker_started_at=1700000020, ended_at=1700000120, finalized_at=1700000125, ) isolated.save_job(job) monkeypatch.setattr(task_control, "TRACES_PATH", reports.TRACES_PATH) task_control.add_instruction(job["session_id"], "保留证据", instruction_id="unused-guidance") report = reports.session_report(job["session_id"]) assert report["lifecycle"] == task_lifecycle.task_lifecycle( job["session_id"], db_path=isolated.path ) assert report["lifecycle"]["durations"]["total"] == 115 assert report["lifecycle"]["durations"]["execution"] is None assert report["instructions"][0]["status"] == "unapplied" word = tmp_path / "lifecycle.docx" excel = tmp_path / "lifecycle.xlsx" reports.word_report(report, word) reports.excel_report(report, excel) document = Document(word) word_text = "\n".join(p.text for p in document.paragraphs) + "\n".join( cell.text for table in document.tables for row in table.rows for cell in row.cells ) book = load_workbook(excel) excel_text = "\n".join(str(cell.value) for sheet in book for row in sheet for cell in row) for content in (word_text, excel_text): assert "后台排队:10.0 秒" in content assert "执行与整理:未记录" in content assert "unused-guidance · 任务结束,未记录消费 · 接收" in content def test_video_export_accepts_finalized_worker_trace_and_rejects_external_path( isolated, tmp_path, monkeypatch ): job = isolated.jobs(service.create_batch(payload())["id"])[0] session_id = job["session_id"] recording_dir = reports.TRACES_PATH / f"web_123_{session_id[:8]}_PASS_2026-09-28" recording_dir.mkdir(parents=True) first = recording_dir / "recording.mp4" first.write_bytes(b"first segment") resumed = recording_dir / "resume_0001" resumed.mkdir() second = resumed / "recording.mp4" second.write_bytes(b"second segment") repository = Mock() repository.db_path = isolated.path repository.get_video_recording_for_session.return_value = {"local_video_path": str(first)} repository.get_session_by_id.return_value = {"video_filepath": str(first)} monkeypatch.setattr(reports, "SessionRepository", lambda: repository) archive_path = reports.export_session_media(session_id, "video") with zipfile.ZipFile(archive_path) as archive: names = archive.namelist() assert names == [ "任务录屏/安卓/0001_web_123_" + session_id[:8] + "_PASS_2026-09-28_recording.mp4", "任务录屏/安卓/0002_resume_0001_recording.mp4", "文件清单.json", ] assert archive.read(names[0]) == b"first segment" assert archive.read(names[1]) == b"second segment" external = tmp_path / "outside" / "recording.mp4" external.parent.mkdir() external.write_bytes(b"unrelated") repository.get_video_recording_for_session.return_value = {"local_video_path": str(external)} assert reports.session_videos({"id": session_id}) == [] def test_history_filters_all_persisted_rows_and_escapes_like(client, isolated): batch = service.create_batch(payload()) first, second = isolated.jobs(batch["id"]) isolated.save_job( { **first, "status": "completed", "created_at": 100, "case": {"name": "搜索_%", "goal": "abc", "tags": "回归"}, } ) isolated.save_job({**second, "status": "failed", "created_at": 200}) assert client.get("/api/automation/history", params={"q": "_%"}).json()["total"] == 1 assert ( client.get("/api/automation/history", params={"q": "回归"}).json()["items"][0]["session_id"] == first["session_id"] ) assert ( client.get( "/api/automation/history", params={"start": 150, "end": 250, "status": "failed"} ).json()["total"] == 1 ) assert ( client.get("/api/automation/history", params={"size": 1, "page": 2}).json()["items"][0][ "session_id" ] == first["session_id"] ) def test_history_returns_persisted_execution_mode_without_inventing_legacy_mode(client, isolated): with isolated.connection() as connection: for sid, device_info in [ ("flash", '{"profile":"flash"}'), ("pro", '{"profile":"pro"}'), ("legacy", "not json"), ]: connection.execute( "INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info) VALUES (?,?,?,?,?)", (sid, sid, 100, "completed", device_info), ) connection.commit() isolated.save_job( { "session_id": "queued-only", "status": "failed", "case": {"name": "no device"}, "task_config": {"profile": "pro"}, "created_at": 100, } ) rows = {row["session_id"]: row for row in client.get("/api/automation/history").json()["items"]} assert rows["flash"]["profile"] == "flash" assert rows["pro"]["profile"] == "pro" assert rows["legacy"]["profile"] is None assert rows["queued-only"]["profile"] == "pro" def test_batch_submission_is_atomic_and_idempotent(client, isolated): value = payload(request_id="same-request") first = client.post("/api/automation/batches", json=value) second = client.post("/api/automation/batches", json=value) assert first.status_code == second.status_code == 200 assert first.json()["id"] == second.json()["id"] assert len(isolated.jobs()) == 2 assert len(isolated.records("batch")) == 1 conflict = client.post("/api/automation/batches", json={**value, "name": "different"}) assert conflict.status_code == 409 lookup = client.get("/api/automation/batches/by-request/same-request").json() assert lookup["id"] == first.json()["id"] assert len(lookup["jobs"]) == 2 def test_removed_replay_is_inert_but_history_survives(client, isolated): batch = service.create_batch(payload()) job = isolated.jobs(batch["id"])[0] isolated.save_job({**job, "execution_mode": "strict", "workflow_id": "legacy"}) service.recover_batches() assert isolated.get("batch", batch["id"])["status"] == "cancelled" assert all(j["status"] == "cancelled" for j in isolated.jobs(batch["id"])) assert client.post(f"/api/automation/batches/{batch['id']}/resume").status_code == 409 assert client.post(f"/api/automation/batches/{batch['id']}/retry_failed").status_code == 409 assert client.get("/api/automation/workflows").status_code == 404 assert client.get("/api/automation/history").json()["total"] == 2 def test_exploration_history_groups_before_pagination_and_preserves_validations(client, isolated): with isolated.connection() as conn: for i, kind in enumerate(["exploration", "exploration", "validation", "validation"]): sid = f"run-{i}" conn.execute( "INSERT INTO sessions(session_id,initial_goal,start_time,status) VALUES (?,?,?,?)", (sid, "目标", i + 1, "completed"), ) record = { "exploration_id": "conversation", "kind": kind, "created_at": i + 1, "status": "completed", "case_version": 1, } conn.execute( "INSERT INTO automation_records VALUES (?,?,?,?)", ("exploration_run", sid, json.dumps(record), i + 1), ) conn.commit() all_rows = client.get("/api/automation/history").json() assert all_rows["total"] == 3 assert {r["session_id"] for r in all_rows["items"]} == {"run-1", "run-2", "run-3"} page = client.get("/api/automation/history?batch_id=__exploration__&size=1").json() assert page["total"] == 1 assert page["items"][0]["session_id"] == "run-1" assert page["items"][0]["exploration_run_count"] == 2 ordinary = client.get("/api/automation/history?batch_id=__single__").json() assert ordinary["total"] == 2 with isolated.connection() as conn: conn.execute( "INSERT INTO automation_records VALUES (?,?,?,?)", ( "exploration", "conversation", json.dumps({"status": "paused", "active_run_id": None}), 5, ), ) conn.commit() assert client.get("/api/automation/history?batch_id=__exploration__").json()["total"] == 0 def test_validation_history_uses_case_title_after_loading_its_job(client, isolated): from artemis.config.exploration import ExplorationConfig from artemis.exploration.models import CaseDraft from artemis.exploration.store import ExplorationStore repository = ExplorationStore(isolated) session = repository.create("探索设置", ExplorationConfig(), {}, "create") repository.save_case( session["id"], CaseDraft(name="深色主题检查", goal="完整执行提示词"), 0, "save" ) run = repository.reserve_run( session["id"], "validate", kind="validation", goal="完整执行提示词", task_config={}, case_version=1, ) isolated.save_job( { "session_id": run["id"], "status": "completed", "name": "完整执行提示词", "goal": "完整执行提示词", } ) with isolated.connection() as conn: conn.execute( "INSERT INTO sessions(session_id,initial_goal,start_time,status) VALUES (?,?,?,?)", (run["id"], "完整执行提示词", 1, "completed"), ) conn.commit() rows = client.get("/api/automation/history?q=深色主题检查").json()["items"] assert len(rows) == 1 assert rows[0]["name"] == "深色主题检查" assert rows[0]["initial_goal"] == "完整执行提示词" assert rows[0]["case_version"] == 1 @pytest.mark.asyncio async def test_queued_exploration_keeps_identity_without_a_session_row(isolated, monkeypatch): from apps.admin_console.routers import tasks from artemis.exploration.store import ExplorationStore from artemis.exploration.models import CreateExploration from apps.admin_console.services.exploration_service import ExplorationService service = ExplorationService(ExplorationStore(isolated)) session = service.create(CreateExploration(goal="排队探索", request_id="create")) run = service.repository.reserve_run( session["id"], "start", kind="exploration", goal="排队探索", task_config={"model": "gpt-6.1-sol"}, ) queue_item = {"session_id": run["id"], "goal": run["goal"], "status": "pending"} from types import SimpleNamespace monkeypatch.setattr( tasks, "state", SimpleNamespace( is_running=False, queue_items=[], queue_tasks=[queue_item], active_runs={}, active_connections={}, active_session_id=None, current_goal=None, current_profile=None, ipc_port=0, ), ) monkeypatch.setattr(tasks, "store", isolated) monkeypatch.setattr(tasks.task_queue_service, "ensure_worker_running", lambda: None) monkeypatch.setattr(tasks, "session_repo", Mock(get_latest_session=lambda: None)) monkeypatch.setattr(tasks.DeviceExecutionLock, "get_active_owner", lambda: None) monkeypatch.setattr(tasks.DeviceExecutionLock, "get_active_owners", lambda: {}) monkeypatch.setattr(tasks.DeviceExecutionLock, "get_queued_tasks", lambda: []) result = await tasks.get_status() assert result["queue"][0]["exploration_id"] == session["id"] assert result["queue"][0]["exploration_kind"] == "exploration" assert "exploration_id" not in queue_item # Do not mutate scheduler state.