1148 lines
50 KiB
Python
1148 lines
50 KiB
Python
"""Automation acceptance checks with isolated persistence and no real device side effects."""
|
||
|
||
from io import BytesIO
|
||
import json
|
||
import sqlite3
|
||
import threading
|
||
import time
|
||
from unittest.mock import AsyncMock, Mock
|
||
from uuid import UUID
|
||
import zipfile
|
||
|
||
from docx import Document
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
from openpyxl import load_workbook
|
||
import pytest
|
||
|
||
from apps.admin_console.routers import automation as api
|
||
from apps.admin_console.services import automation_service as service
|
||
from apps.admin_console.services import report_export as reports
|
||
from apps.admin_console.services import task_import
|
||
from apps.admin_console.services.automation_store import AutomationStore
|
||
from artemis.data_engine.storage import StorageManager
|
||
|
||
|
||
@pytest.fixture
|
||
def isolated(tmp_path, monkeypatch):
|
||
db = tmp_path / "data.db"
|
||
StorageManager(db_path=db, base_trace_dir=tmp_path)
|
||
store = AutomationStore(db)
|
||
for module in (api, service, reports):
|
||
monkeypatch.setattr(module, "store", store)
|
||
monkeypatch.setattr(reports, "TRACES_PATH", tmp_path / "traces")
|
||
monkeypatch.setattr(reports, "EXPORT_ROOT", tmp_path / "exports")
|
||
from apps.admin_console.database.repositories.session_repository import SessionRepository
|
||
from apps.admin_console.database.repositories.step_repository import StepRepository
|
||
|
||
monkeypatch.setattr(reports, "SessionRepository", lambda *a: SessionRepository(db))
|
||
monkeypatch.setattr(reports, "StepRepository", lambda *a: StepRepository(db))
|
||
return store
|
||
|
||
|
||
@pytest.fixture
|
||
def client(isolated):
|
||
app = FastAPI()
|
||
app.include_router(api.router)
|
||
return TestClient(app)
|
||
|
||
|
||
def payload(**extra):
|
||
return {
|
||
"name": "批次",
|
||
"cases": [{"name": "用例甲", "goal": "观察设置"}, {"name": "用例乙", "goal": "观察首页"}],
|
||
"device_serial": "test-device",
|
||
**extra,
|
||
}
|
||
|
||
|
||
def workbook_bytes(rows):
|
||
wb = load_workbook(BytesIO(task_import.template()))
|
||
for row in rows:
|
||
wb["任务"].append(row)
|
||
output = BytesIO()
|
||
wb.save(output)
|
||
return output.getvalue()
|
||
|
||
|
||
def test_template_preview_rejects_formula_and_bad_json():
|
||
data = workbook_bytes(
|
||
[
|
||
["正常", "设置", "冒烟", "打开设置", "", "观察设置", '{"profile":"flash"}'],
|
||
["公式", "", "", "", "", "=SUM(1,2)"],
|
||
["参数错误", "", "", "", "", "测试", "{"],
|
||
]
|
||
)
|
||
result = task_import.parse_workbook(data)
|
||
assert len(result["tasks"]) == 1
|
||
assert [e["row"] for e in result["errors"]] == [3, 4]
|
||
assert not result["valid"]
|
||
|
||
|
||
def test_prompt_only_and_per_row_config_validation():
|
||
goal = "先查看屏幕，再按实际画面判断。"
|
||
data = workbook_bytes(
|
||
[
|
||
[
|
||
"CASE-7",
|
||
"检查项目",
|
||
"子项",
|
||
"用例名称",
|
||
"仅作索引的约束",
|
||
goal,
|
||
'{"profile":"pro","model":"mimo-v2.6-flash","record_video":true}',
|
||
],
|
||
["CASE-8", "", "", "", "", "观察屏幕", '{"OPENAI_API_KEY":"never"}'],
|
||
]
|
||
)
|
||
preview = task_import.parse_workbook(data)
|
||
assert [e["row"] for e in preview["errors"]] == [3]
|
||
case = preview["tasks"][0]
|
||
assert task_import.case_goal(case) == goal
|
||
assert case["preconditions"] == "仅作索引的约束"
|
||
assert case["task_config"]["record_video"] is True
|
||
|
||
|
||
def test_invalid_archive_and_empty_template():
|
||
with pytest.raises(ValueError):
|
||
task_import.parse_workbook(b"not a zip")
|
||
assert not task_import.parse_workbook(task_import.template())["valid"]
|
||
|
||
|
||
def test_batch_archive_preparation_reports_progress_and_reuses_ready_file(
|
||
client, isolated, monkeypatch
|
||
):
|
||
batch = service.create_batch(payload())
|
||
batch_id = batch["id"]
|
||
started = threading.Event()
|
||
release = threading.Event()
|
||
calls = []
|
||
|
||
def slow_export(identifier, progress=None):
|
||
calls.append(identifier)
|
||
progress(0, 2, "正在打包任务 1/2")
|
||
started.set()
|
||
assert release.wait(5)
|
||
progress(2, 2, "正在完成压缩包")
|
||
path = reports.EXPORT_ROOT / "batches" / identifier / "批次报告.zip"
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_bytes(b"prepared archive")
|
||
return path
|
||
|
||
monkeypatch.setattr(reports, "export_batch", slow_export)
|
||
base = f"/api/automation/batches/{batch_id}/archive"
|
||
try:
|
||
response = client.post(base + "/prepare")
|
||
assert response.status_code == 200
|
||
assert response.json()["status"] == "preparing"
|
||
assert started.wait(2)
|
||
progress = client.get(base + "/status").json()
|
||
assert progress["total"] == 2
|
||
assert progress["stage"] == "正在打包任务 1/2"
|
||
assert client.get(base).status_code == 409
|
||
finally:
|
||
release.set()
|
||
for _ in range(50):
|
||
if client.get(base + "/status").json()["status"] == "ready":
|
||
break
|
||
time.sleep(0.02)
|
||
assert client.get(base + "/status").json()["status"] == "ready"
|
||
assert client.get(base).content == b"prepared archive"
|
||
assert calls == [batch_id]
|
||
|
||
|
||
def test_completed_batch_reuses_existing_archive(client, isolated, monkeypatch):
|
||
batch = service.create_batch(payload())
|
||
batch_id = batch["id"]
|
||
isolated.put("batch", batch_id, {**batch, "status": "completed", "report_ready": True})
|
||
path = reports.EXPORT_ROOT / "batches" / batch_id / "批次报告.zip"
|
||
path.parent.mkdir(parents=True)
|
||
path.write_bytes(b"existing archive")
|
||
exporter = Mock(side_effect=AssertionError("archive must not be rebuilt"))
|
||
monkeypatch.setattr(reports, "export_batch", exporter)
|
||
base = f"/api/automation/batches/{batch_id}/archive"
|
||
assert client.post(base + "/prepare").json()["status"] == "ready"
|
||
assert client.get(base).content == b"existing archive"
|
||
exporter.assert_not_called()
|
||
|
||
|
||
def test_preview_does_not_enqueue(client, isolated):
|
||
response = client.post(
|
||
"/api/automation/import/preview",
|
||
files={
|
||
"file": ("tasks.xlsx", workbook_bytes([["案例", "", "", "观察页面", "", "观察页面"]]))
|
||
},
|
||
)
|
||
assert response.status_code == 200
|
||
assert response.json()["valid"]
|
||
assert not isolated.jobs()
|
||
|
||
|
||
def test_preferences_and_required_device(client):
|
||
assert client.get("/api/automation/preferences").json()["formats"] == ["docx", "xlsx"]
|
||
assert client.put("/api/automation/preferences", json={"formats": ["xlsx"]}).status_code == 200
|
||
assert client.put("/api/automation/preferences", json={"formats": []}).status_code == 422
|
||
assert client.post("/api/automation/batches", json=payload(device_serial="")).status_code == 422
|
||
|
||
|
||
def test_batch_repetitions_are_ordered_independent_and_idempotent(client, isolated):
|
||
request = payload(repeat_count=3, request_id="repeat-test")
|
||
created = client.post("/api/automation/batches", json=request)
|
||
assert created.status_code == 200
|
||
batch = created.json()
|
||
assert (batch["case_count"], batch["repeat_count"], batch["total_executions"]) == (2, 3, 6)
|
||
jobs = isolated.jobs(batch["id"])
|
||
assert len({j["session_id"] for j in jobs}) == 6
|
||
assert [j["position"] for j in jobs] == list(range(1, 7))
|
||
assert [(j["iteration"], j["case_position"]) for j in jobs] == [
|
||
(1, 1),
|
||
(1, 2),
|
||
(2, 1),
|
||
(2, 2),
|
||
(3, 1),
|
||
(3, 2),
|
||
]
|
||
assert [j["case"]["goal"] for j in jobs] == ["观察设置", "观察首页"] * 3
|
||
assert all(j["status"] == "pending" for j in jobs)
|
||
assert client.post("/api/automation/batches", json=request).json()["id"] == batch["id"]
|
||
assert len(isolated.jobs(batch["id"])) == 6
|
||
assert (
|
||
client.post("/api/automation/batches", json={**request, "repeat_count": 2}).status_code
|
||
== 409
|
||
)
|
||
# Each selected failure is copied once, including failures from different rounds.
|
||
for job in (jobs[0], jobs[4]):
|
||
isolated.save_job({**job, "status": "failed"})
|
||
retry = service.batch_action(batch["id"], "retry_failed")
|
||
assert retry["repeat_count"] == 1
|
||
assert len(isolated.jobs(retry["id"])) == 2
|
||
report = reports.session_report(jobs[4]["session_id"])
|
||
assert "第 3 / 3 轮" in report["name"]
|
||
archive = reports.export_batch(batch["id"])
|
||
with zipfile.ZipFile(archive) as z:
|
||
assert len([n for n in z.namelist() if n.endswith(".xlsx")]) == 7
|
||
assert len(set(z.namelist())) == len(z.namelist())
|
||
|
||
|
||
@pytest.mark.parametrize("count", [0, -1, 1.5, True, "3", None])
|
||
def test_batch_rejects_invalid_repetitions(client, isolated, count):
|
||
assert (
|
||
client.post("/api/automation/batches", json=payload(repeat_count=count)).status_code == 422
|
||
)
|
||
assert isolated.records("batch") == []
|
||
|
||
|
||
def test_batch_limit_is_checked_before_creating_records(client, isolated, monkeypatch):
|
||
monkeypatch.setattr(service.settings, "ARTEMIS_BATCH_MAX_EXECUTIONS", 5)
|
||
assert client.get("/api/automation/preferences").json()["batch_max_executions"] == 5
|
||
response = client.post("/api/automation/batches", json=payload(repeat_count=3))
|
||
assert response.status_code == 409
|
||
assert "5" in response.json()["detail"]
|
||
assert not isolated.records("batch")
|
||
assert not isolated.jobs()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_loop_dispatch_continues_after_failure_and_resume_without_repeating(
|
||
isolated, monkeypatch
|
||
):
|
||
from apps.admin_console import server
|
||
from artemis.runtime.device_pool import DeviceStatus
|
||
|
||
batch = service.create_batch(payload(repeat_count=2))
|
||
service.batch_action(batch["id"], "resume")
|
||
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
||
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
||
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async",
|
||
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
||
)
|
||
for index in range(4):
|
||
await service.tick_batches()
|
||
await service.tick_batches() # Polling must not enqueue the same execution twice.
|
||
assert enqueue.await_count == index + 1
|
||
job = isolated.jobs(batch["id"])[index]
|
||
assert enqueue.call_args.kwargs["session_id"] == job["session_id"]
|
||
assert f"第 {job['iteration']} / 2 轮" in enqueue.call_args.kwargs["display_name"]
|
||
isolated.save_job({**job, "status": "failed" if index == 0 else "completed"})
|
||
if index == 1:
|
||
service.batch_action(batch["id"], "pause")
|
||
await service.tick_batches()
|
||
assert enqueue.await_count == 2
|
||
service.batch_action(batch["id"], "resume")
|
||
await service.tick_batches()
|
||
assert isolated.get("batch", batch["id"])["status"] == "completed"
|
||
assert enqueue.await_count == 4
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_repeat_fail_fast_skips_future_rounds(isolated):
|
||
batch = service.create_batch(payload(repeat_count=3, stop_on_failure=True))
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": "failed"})
|
||
service.batch_action(batch["id"], "resume")
|
||
await service.tick_batches()
|
||
assert [j["status"] for j in isolated.jobs(batch["id"])] == ["failed"] + ["skipped"] * 5
|
||
|
||
|
||
def test_batch_delete_removes_tasks_and_files_and_prevents_request_replay(client, isolated):
|
||
created = client.post(
|
||
"/api/automation/batches", json=payload(request_id="delete-example")
|
||
).json()
|
||
first, second = isolated.jobs(created["id"])
|
||
isolated.save_job({**first, "status": "completed"})
|
||
trace_dir = reports.TRACES_PATH / first["session_id"]
|
||
trace_dir.mkdir(parents=True)
|
||
(trace_dir / "result.json").write_text("{}", encoding="utf-8")
|
||
report_dir = reports.EXPORT_ROOT / "sessions" / first["session_id"]
|
||
report_dir.mkdir(parents=True)
|
||
(report_dir / "任务报告.docx").write_bytes(b"report")
|
||
batch_dir = reports.EXPORT_ROOT / "batches" / created["id"]
|
||
batch_dir.mkdir(parents=True)
|
||
(batch_dir / "批次报告.zip").write_bytes(b"zip")
|
||
mcp_task_dir = reports.EXPORT_ROOT.parent / "mcp_exports" / "sessions" / first["session_id"]
|
||
mcp_task_dir.mkdir(parents=True)
|
||
(mcp_task_dir / "报告.docx").write_bytes(b"report")
|
||
mcp_batch_dir = reports.EXPORT_ROOT.parent / "mcp_exports" / "batches" / created["id"]
|
||
mcp_batch_dir.mkdir(parents=True)
|
||
(mcp_batch_dir / "批次.zip").write_bytes(b"zip")
|
||
unrelated = reports.TRACES_PATH / "unrelated-task"
|
||
unrelated.mkdir()
|
||
(unrelated / "keep.txt").write_text("keep", encoding="utf-8")
|
||
|
||
response = client.delete(f"/api/automation/batches/{created['id']}")
|
||
assert response.status_code == 200
|
||
assert client.get(f"/api/automation/batches/{created['id']}").status_code == 404
|
||
assert isolated.job(first["session_id"]) is None
|
||
assert isolated.job(second["session_id"]) is None
|
||
assert not trace_dir.exists()
|
||
assert not report_dir.exists()
|
||
assert not batch_dir.exists()
|
||
assert not mcp_task_dir.exists()
|
||
assert not mcp_batch_dir.exists()
|
||
assert (unrelated / "keep.txt").read_text(encoding="utf-8") == "keep"
|
||
with pytest.raises(ValueError, match="不存在"):
|
||
reports.session_report(first["session_id"])
|
||
assert (
|
||
client.post(
|
||
"/api/automation/batches", json=payload(request_id="delete-example")
|
||
).status_code
|
||
== 409
|
||
)
|
||
|
||
|
||
@pytest.mark.parametrize("status", ["queued", "running", "task_paused", "resuming", "finalizing"])
|
||
def test_batch_delete_rejects_active_jobs(client, isolated, status):
|
||
batch = service.create_batch(payload())
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": status})
|
||
|
||
response = client.delete(f"/api/automation/batches/{batch['id']}")
|
||
assert response.status_code == 409
|
||
assert isolated.get("batch", batch["id"]) is not None
|
||
assert len(isolated.jobs(batch["id"])) == 2
|
||
|
||
|
||
def test_batch_deletion_reservation_blocks_new_dispatch(isolated):
|
||
batch = service.create_batch(payload())
|
||
job = isolated.jobs(batch["id"])[0]
|
||
|
||
isolated.begin_delete_batch(batch["id"])
|
||
|
||
with pytest.raises(ValueError, match="正在删除"):
|
||
isolated.save_job({**job, "status": "queued"})
|
||
with pytest.raises(ValueError, match="正在删除"):
|
||
isolated.put("batch", batch["id"], {**batch, "status": "running"})
|
||
assert isolated.jobs_for_sessions([job["session_id"]])[job["session_id"]]["status"] == "pending"
|
||
|
||
|
||
def test_batch_deletion_preserves_shared_screenshot_until_last_task(client, isolated):
|
||
first_batch = service.create_batch(payload())
|
||
second_batch = service.create_batch(payload(name="另一批次"))
|
||
first = isolated.jobs(first_batch["id"])[0]
|
||
other = isolated.jobs(second_batch["id"])[0]
|
||
shared_hash = "a" * 64
|
||
storage = StorageManager(isolated.path, reports.TRACES_PATH)
|
||
image_file = reports.TRACES_PATH / "images" / f"{shared_hash}.jpg"
|
||
image_file.parent.mkdir(parents=True)
|
||
image_file.write_bytes(b"shared image")
|
||
with sqlite3.connect(isolated.path) as conn:
|
||
conn.execute("INSERT INTO images(image_name) VALUES (?)", (shared_hash,))
|
||
for job in (first, other):
|
||
conn.execute(
|
||
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info) "
|
||
"VALUES (?,?,?,?,?)",
|
||
(job["session_id"], "观察设置", 1.0, "completed", "{}"),
|
||
)
|
||
conn.execute(
|
||
"INSERT INTO steps(step_id,session_id,pre_image_name) VALUES (?,?,?)",
|
||
(job["session_id"], job["session_id"], shared_hash),
|
||
)
|
||
|
||
assert client.delete(f"/api/automation/batches/{first_batch['id']}").status_code == 200
|
||
assert image_file.read_bytes() == b"shared image"
|
||
assert storage.get_session(UUID(other["session_id"])) is not None
|
||
assert client.delete(f"/api/automation/batches/{second_batch['id']}").status_code == 200
|
||
assert not image_file.exists()
|
||
|
||
|
||
def test_restart_never_repeats_started_work(isolated):
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**job, "status": "queued"})
|
||
service.recover_batches()
|
||
assert isolated.get("batch", batch["id"])["status"] == "paused"
|
||
assert [j["status"] for j in isolated.jobs(batch["id"])] == ["interrupted", "pending"]
|
||
retry = service.batch_action(batch["id"], "retry_failed")
|
||
assert retry["status"] == "draft"
|
||
assert len(isolated.jobs(retry["id"])) == 1
|
||
assert isolated.jobs(retry["id"])[0]["session_id"] != job["session_id"]
|
||
|
||
|
||
@pytest.mark.parametrize("liveness_source", ["pid", "device_lock"])
|
||
def test_recovery_preserves_live_runner_and_batch(isolated, monkeypatch, liveness_source):
|
||
import os
|
||
from types import SimpleNamespace
|
||
from artemis.runtime import DeviceExecutionLock
|
||
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**job, "status": "running"})
|
||
if liveness_source == "pid":
|
||
with sqlite3.connect(isolated.path) as conn:
|
||
conn.execute(
|
||
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) "
|
||
"VALUES (?,?,?,?,?,?)",
|
||
(job["session_id"], "goal", 1, "running", "{}", os.getpid()),
|
||
)
|
||
monkeypatch.setattr(
|
||
DeviceExecutionLock,
|
||
"get_active_owners",
|
||
lambda: (
|
||
{"device": SimpleNamespace(session_id=job["session_id"])}
|
||
if liveness_source == "device_lock"
|
||
else {}
|
||
),
|
||
)
|
||
service.recover_batches()
|
||
assert isolated.job(job["session_id"])["status"] == "running"
|
||
assert not isolated.job(job["session_id"]).get("error")
|
||
assert isolated.get("batch", batch["id"])["status"] == "running"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_natural_completion_resolves_old_recovery_error_in_job_report_and_history(
|
||
client, isolated
|
||
):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
message = "服务中断；未完成任务需手动重跑"
|
||
isolated.save_job({**job, "status": "interrupted", "error": message})
|
||
await service.on_terminal({"session_id": job["session_id"]}, "completed")
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "completed"
|
||
assert not saved.get("error")
|
||
assert saved["recovery_events"][0]["message"] == message
|
||
assert reports.session_report(job["session_id"])["error"] == ""
|
||
assert not client.get("/api/automation/history").json()["items"][0].get("error")
|
||
|
||
|
||
def test_recovery_repairs_completed_legacy_warning_but_preserves_real_failure(isolated):
|
||
batch = service.create_batch(payload())
|
||
first, second = isolated.jobs(batch["id"])
|
||
isolated.save_job({**first, "status": "completed", "error": "服务中断，需手动重跑"})
|
||
isolated.save_job({**second, "status": "failed", "error": "任务执行超时（30 秒）"})
|
||
assert reports.session_report(first["session_id"])["error"] == ""
|
||
service.recover_batches()
|
||
assert not isolated.job(first["session_id"]).get("error")
|
||
assert "服务中断" in isolated.job(first["session_id"])["recovery_events"][0]["message"]
|
||
assert "超时" in isolated.job(second["session_id"])["error"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_surviving_worker_is_finalized_when_it_exits_without_resubmission(
|
||
isolated, monkeypatch
|
||
):
|
||
import os
|
||
from artemis.runtime import DeviceExecutionLock
|
||
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**job, "status": "running"})
|
||
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
|
||
with sqlite3.connect(isolated.path) as conn:
|
||
conn.execute(
|
||
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) "
|
||
"VALUES (?,?,?,?,?,?)",
|
||
(job["session_id"], "goal", time.time(), "running", "{}", os.getpid()),
|
||
)
|
||
service.recover_batches()
|
||
await service.reconcile_recovered_jobs()
|
||
assert isolated.job(job["session_id"])["status"] == "running"
|
||
with sqlite3.connect(isolated.path) as conn:
|
||
conn.execute(
|
||
"UPDATE sessions SET pid=NULL,status='completed' WHERE session_id=?",
|
||
(job["session_id"],),
|
||
)
|
||
await service.reconcile_recovered_jobs()
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "completed"
|
||
assert saved["report_ready"]
|
||
assert not saved.get("recovery_waiting")
|
||
assert not saved.get("error")
|
||
assert len(isolated.jobs(batch["id"])) == 2
|
||
assert isolated.jobs(batch["id"])[1]["status"] == "pending"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_surviving_worker_that_dies_without_result_is_interrupted(isolated, monkeypatch):
|
||
from artemis.runtime import DeviceExecutionLock
|
||
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
isolated.save_job({**job, "status": "running", "recovery_waiting": True})
|
||
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
|
||
await service.reconcile_recovered_jobs()
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "interrupted"
|
||
assert "服务中断" in saved["error"]
|
||
assert not saved.get("recovery_waiting")
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_batch_admission_idempotent_and_bound_device(isolated, monkeypatch):
|
||
from apps.admin_console import server
|
||
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
||
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
||
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.validate_explicit_serial_async", AsyncMock(return_value=None)
|
||
)
|
||
from artemis.runtime.device_pool import DeviceStatus
|
||
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async",
|
||
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
||
)
|
||
await service.tick_batches()
|
||
await service.tick_batches()
|
||
assert enqueue.await_count == 1
|
||
assert enqueue.call_args.kwargs["device_serial"] == "test-device"
|
||
assert isolated.jobs(batch["id"])[1]["status"] == "pending"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_paused_case_yields_batch_slot_to_next_case(isolated, monkeypatch):
|
||
from apps.admin_console import server
|
||
from artemis.runtime.device_pool import DeviceStatus
|
||
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": "task_paused", "started_at": time.time()})
|
||
monkeypatch.setattr(
|
||
service, "state", type("State", (), {"is_paused": False, "active_runs": {}})()
|
||
)
|
||
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
||
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async",
|
||
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
||
)
|
||
await service.tick_batches()
|
||
assert enqueue.await_count == 1
|
||
assert enqueue.call_args.args[0] == ["观察首页"]
|
||
assert [job["status"] for job in isolated.jobs(batch["id"])] == ["task_paused", "queued"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_device_outage_pauses_without_consuming_pending_case(isolated, monkeypatch):
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[])
|
||
)
|
||
await service.tick_batches()
|
||
assert isolated.get("batch", batch["id"])["status"] == "paused"
|
||
assert all(job["status"] == "pending" for job in isolated.jobs(batch["id"]))
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fail_fast_includes_skipped_reports(isolated, monkeypatch):
|
||
batch = service.create_batch(payload(stop_on_failure=True))
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": "failed"})
|
||
service.batch_action(batch["id"], "resume")
|
||
await service.tick_batches()
|
||
assert [j["status"] for j in isolated.jobs(batch["id"])] == ["failed", "skipped"]
|
||
archive = reports.export_batch(batch["id"])
|
||
with zipfile.ZipFile(archive) as z:
|
||
assert len(z.namelist()) == 8 # summaries + two reports and evidence index per task
|
||
assert len([name for name in z.namelist() if name.endswith("证据索引.json")]) == 2
|
||
assert sum(n.endswith(".docx") for n in z.namelist()) == 3
|
||
|
||
|
||
def test_batch_archive_contains_each_task_media_when_present(isolated, monkeypatch, tmp_path):
|
||
batch = service.create_batch(payload())
|
||
first, second = isolated.jobs(batch["id"])
|
||
image1 = tmp_path / "first.png"
|
||
image2 = tmp_path / "second.png"
|
||
video = tmp_path / "recording.mp4"
|
||
image1.write_bytes(b"first screenshot")
|
||
image2.write_bytes(b"second screenshot")
|
||
video.write_bytes(b"saved recording")
|
||
images = {first["session_id"]: image1, second["session_id"]: image2}
|
||
monkeypatch.setattr(
|
||
reports,
|
||
"session_screenshots",
|
||
lambda report: [(images[report["id"]], "0001_步骤0001_操作前.png")],
|
||
)
|
||
monkeypatch.setattr(
|
||
reports,
|
||
"session_videos",
|
||
lambda report: (
|
||
[(video, "0001_recording.mp4")] if report["id"] == first["session_id"] else []
|
||
),
|
||
)
|
||
|
||
archive = reports.export_batch(batch["id"])
|
||
with zipfile.ZipFile(archive) as z:
|
||
names = z.namelist()
|
||
assert len([name for name in names if "/步骤截图/" in name]) == 2
|
||
assert len([name for name in names if "/任务录屏/" in name]) == 1
|
||
assert all(name.startswith("任务报告/") for name in names if "步骤截图" in name)
|
||
assert z.read(next(name for name in names if "/任务录屏/" in name)) == b"saved recording"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_report_failure_does_not_strand_job(isolated, monkeypatch):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
monkeypatch.setattr(
|
||
reports, "export_session", lambda *a: (_ for _ in ()).throw(OSError("disk full"))
|
||
)
|
||
await service.on_terminal({"session_id": job["session_id"]}, "completed")
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "completed"
|
||
assert "disk full" in saved["report_error"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_batch_timeout_stays_visible_in_job_report_and_history(client, isolated):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
isolated.save_job(
|
||
{**job, "status": "running", "timed_out": True, "error": "任务执行超时（30 秒）"}
|
||
)
|
||
await service.on_terminal({"session_id": job["session_id"]}, "cancelled")
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "failed"
|
||
assert "超时" in saved["error"]
|
||
assert "超时" in reports.session_report(job["session_id"])["error"]
|
||
history = client.get("/api/automation/history").json()["items"]
|
||
assert any(
|
||
item["session_id"] == job["session_id"] and "超时" in item["error"] for item in history
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
@pytest.mark.parametrize("stop_on_failure", [False, True])
|
||
async def test_timed_out_case_releases_next_batch_case(isolated, monkeypatch, stop_on_failure):
|
||
from apps.admin_console import server
|
||
from artemis.runtime.device_pool import DeviceStatus
|
||
|
||
batch = service.create_batch(payload(timeout_seconds=1, stop_on_failure=stop_on_failure))
|
||
service.batch_action(batch["id"], "resume")
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": "running", "started_at": time.time() - 2})
|
||
monkeypatch.setattr(
|
||
service, "state", type("State", (), {"is_paused": False, "active_runs": {}})()
|
||
)
|
||
stop = Mock(return_value=True)
|
||
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
||
monkeypatch.setattr(server.task_queue_service, "stop_tasks", stop)
|
||
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async",
|
||
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
||
)
|
||
monkeypatch.setattr(reports, "export_batch", Mock())
|
||
|
||
await service.tick_batches()
|
||
assert isolated.get("batch", batch["id"])["status"] == "running"
|
||
assert "超时" in isolated.job(first["session_id"])["error"]
|
||
stop.assert_called_once_with(session_id=first["session_id"])
|
||
await service.tick_batches()
|
||
stop.assert_called_once()
|
||
|
||
await service.on_terminal({"session_id": first["session_id"]}, "cancelled")
|
||
await service.tick_batches()
|
||
statuses = [job["status"] for job in isolated.jobs(batch["id"])]
|
||
assert statuses == (["failed", "skipped"] if stop_on_failure else ["failed", "queued"])
|
||
assert enqueue.await_count == (0 if stop_on_failure else 1)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_queued_case_does_not_consume_execution_timeout(isolated, monkeypatch):
|
||
from apps.admin_console import server
|
||
|
||
batch = service.create_batch(payload(timeout_seconds=1))
|
||
service.batch_action(batch["id"], "resume")
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": "queued", "started_at": time.time() - 30})
|
||
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
||
stop = Mock()
|
||
monkeypatch.setattr(server.task_queue_service, "stop_tasks", stop)
|
||
|
||
await service.tick_batches()
|
||
stop.assert_not_called()
|
||
assert isolated.job(first["session_id"])["status"] == "queued"
|
||
assert isolated.get("batch", batch["id"])["status"] == "running"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_model_timeout_from_runner_log_reaches_failed_task_and_report(isolated):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
folder = reports.TRACES_PATH / job["session_id"]
|
||
folder.mkdir(parents=True)
|
||
(folder / "stderr.log").write_text(
|
||
"TimeoutError: model request timed out after 300 seconds\n", encoding="utf-8"
|
||
)
|
||
|
||
await service.on_terminal({"session_id": job["session_id"]}, "failed")
|
||
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "failed"
|
||
assert "超时" in saved["error"]
|
||
assert reports.session_report(job["session_id"])["status"] == "failed"
|
||
assert "超时" in reports.session_report(job["session_id"])["error"]
|
||
|
||
|
||
def test_optional_timeout_warnings_are_not_reported_as_task_failure(tmp_path):
|
||
(tmp_path / "stdout.log").write_text(
|
||
"⚠ Could not clean unowned screen wake locks on emulator-5554: "
|
||
"Timed out acquiring the Artemis awake-lease mutex.\n"
|
||
"⚠ VisualStepSummarizer: Error generating summary for step 3: \n"
|
||
"ℹ StepMemoryService: Retrying Step 3 (attempt 4) after 1.0s\n",
|
||
encoding="utf-8",
|
||
)
|
||
assert reports.failure_reason(tmp_path, "failed", None) == (
|
||
"任务未完成，运行日志中没有可提取的具体异常；请查看执行步骤。"
|
||
)
|
||
|
||
(tmp_path / "stderr.log").write_text(
|
||
"TimeoutError: model request timed out after 300 seconds\n", encoding="utf-8"
|
||
)
|
||
reason = reports.failure_reason(tmp_path, "failed", None)
|
||
assert "model request timed out" in reason
|
||
assert "awake-lease" not in reason
|
||
|
||
(tmp_path / "stderr.log").write_text(
|
||
"DeviceBusyError: device queue reservation was cancelled\n", encoding="utf-8"
|
||
)
|
||
assert reports.failure_reason(tmp_path, "failed", None) == (
|
||
"DeviceBusyError: device queue reservation was cancelled"
|
||
)
|
||
|
||
|
||
def test_reports_do_not_execute_formulas_or_invent_pass(isolated):
|
||
batch = service.create_batch(payload())
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job(
|
||
{**job, "case": {"name": "=1+1", "goal": '=HYPERLINK("x")'}, "status": "completed"}
|
||
)
|
||
report = reports.session_report(job["session_id"])
|
||
assert report["verdict"].startswith("未验证")
|
||
assert reports.verdict({"tests": {"passed": 2, "failed": 1}}) == "验证失败"
|
||
assert reports.verdict({"tests": {"passed": 2, "unchecked": 1}}) == "验证不完整"
|
||
paths = reports.export_session(job["session_id"])
|
||
wb = load_workbook(next(p for p in paths if p.suffix == ".xlsx"))
|
||
assert wb["任务结果"]["B3"].data_type == "s"
|
||
with zipfile.ZipFile(next(p for p in paths if p.suffix == ".docx")) as doc:
|
||
assert "未验证".encode() in doc.read("word/document.xml")
|
||
|
||
|
||
def test_failed_report_exposes_logged_reason_and_formats_chinese_markdown(isolated):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
isolated.save_job({**job, "status": "failed"})
|
||
folder = reports.TRACES_PATH / job["session_id"]
|
||
(folder / "notes").mkdir(parents=True)
|
||
(folder / "stdout.log").write_text(
|
||
"❌ [run] Error running automation: 模型连续 3 次未提供可执行的工具调用\n",
|
||
encoding="utf-8",
|
||
)
|
||
(folder / "notes" / "output.md").write_text(
|
||
"## 结果\n- **网络**正常\n\n| 分类 | 状态 |\n| --- | --- |\n| 系统 | 未确认 |",
|
||
encoding="utf-8",
|
||
)
|
||
report = reports.session_report(job["session_id"])
|
||
assert "连续 3 次" in report["error"]
|
||
assert report["verdict"] == "测试阻塞（AI确认）"
|
||
assert report["test_result"]["status"] == "blocked"
|
||
assert "连续 3 次" in report["test_result"]["reason"]
|
||
assert "abc123secret" not in reports.failure_reason(
|
||
folder, "failed", "认证失败 OPENAI_API_KEY=abc123secret Authorization: Bearer abc123secret"
|
||
)
|
||
assert "超时" not in reports.failure_reason(folder, "failed", "LLMExhaustedError: 额度不足")
|
||
paths = reports.export_session(job["session_id"])
|
||
doc = Document(next(path for path in paths if path.suffix == ".docx"))
|
||
assert doc.styles["Normal"].font.name == "Microsoft YaHei"
|
||
assert any("异常原因" in paragraph.text for paragraph in doc.paragraphs)
|
||
assert any(
|
||
run.bold and "网络" in run.text for paragraph in doc.paragraphs for run in paragraph.runs
|
||
)
|
||
assert any(
|
||
cell.text == "分类" for table in doc.tables for row in table.rows for cell in row.cells
|
||
)
|
||
wb = load_workbook(next(path for path in paths if path.suffix == ".xlsx"))
|
||
assert any(
|
||
row[0].value == "异常原因" and "连续 3 次" in row[1].value
|
||
for row in wb["任务结果"].iter_rows()
|
||
)
|
||
assert wb["结论"]["B2"].value == "结果"
|
||
assert wb["结果表1"]["A2"].value == "系统"
|
||
assert wb["执行证据"]["A1"].font.name == "微软雅黑"
|
||
assert wb["执行证据"].page_setup.orientation == "landscape"
|
||
assert reports.execution_result_label({"status": "dispatched"}) == "动作已发出（需核对截图）"
|
||
assert reports.conclusion_rows("## 结果\n- **通过**检查\n```txt\n| 不是 | 表格 |\n```") == [
|
||
("类型", "结论与发现"),
|
||
("小节", "结果"),
|
||
("要点", "通过检查"),
|
||
("代码", "| 不是 | 表格 |"),
|
||
]
|
||
|
||
|
||
def test_prompt_based_verdict_is_consistent_in_history_batch_and_reports(
|
||
client, isolated, monkeypatch
|
||
):
|
||
from artemis.runtime import case_result
|
||
|
||
monkeypatch.setattr(case_result, "TRACES_PATH", reports.TRACES_PATH)
|
||
batch = service.create_batch(payload())
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job(
|
||
{**job, "status": "completed", "started_at": time.time() - 50, "ended_at": time.time()}
|
||
)
|
||
result = {
|
||
"status": "failed",
|
||
"reason": "预期出现成功提示，实际显示错误",
|
||
"evidence": ["步骤 3 点击后截图显示错误提示"],
|
||
}
|
||
folder = reports.TRACES_PATH / job["session_id"]
|
||
folder.mkdir(parents=True)
|
||
(folder / "run_outcome.json").write_text(
|
||
json.dumps(
|
||
{"task_status": "completed", "tests": {}, "test_result": result}, ensure_ascii=False
|
||
),
|
||
encoding="utf-8",
|
||
)
|
||
history = client.get("/api/automation/history").json()["items"][0]
|
||
batch_job = client.get(f"/api/automation/batches/{batch['id']}").json()["jobs"][0]
|
||
report = client.get(f"/api/automation/reports/{job['session_id']}").json()
|
||
for item in (history, batch_job, report):
|
||
assert item["status"] == "completed"
|
||
assert item["test_result"] == {**result, "source": "ai"}
|
||
assert report["verdict"] == "测试不通过（AI确认）"
|
||
paths = reports.export_session(job["session_id"])
|
||
doc = Document(next(path for path in paths if path.suffix == ".docx"))
|
||
doc_text = "\n".join(
|
||
cell.text for table in doc.tables for row in table.rows for cell in row.cells
|
||
)
|
||
doc_text += "\n" + "\n".join(paragraph.text for paragraph in doc.paragraphs)
|
||
assert result["reason"] in doc_text
|
||
assert result["evidence"][0] in doc_text
|
||
wb = load_workbook(next(path for path in paths if path.suffix == ".xlsx"))
|
||
values = [cell.value for row in wb["任务结果"].iter_rows() for cell in row]
|
||
assert "测试不通过（AI确认）" in values
|
||
assert result["reason"] in values
|
||
|
||
|
||
def test_human_review_overrides_every_report_preserves_ai_and_invalidates_batch_cache(
|
||
client, isolated, monkeypatch
|
||
):
|
||
from artemis.runtime import case_result
|
||
from apps.admin_console.routers import sessions as sessions_api
|
||
|
||
monkeypatch.setattr(case_result, "TRACES_PATH", reports.TRACES_PATH)
|
||
batch = service.create_batch(payload())
|
||
jobs = isolated.jobs(batch["id"])
|
||
for job in jobs:
|
||
isolated.save_job({**job, "status": "completed", "ended_at": time.time()})
|
||
session_id = jobs[0]["session_id"]
|
||
ai = {"status": "failed", "reason": "AI识别到预期不匹配", "evidence": ["步骤 3 截图"]}
|
||
directory = reports.TRACES_PATH / session_id
|
||
directory.mkdir(parents=True)
|
||
outcome_file = directory / "run_outcome.json"
|
||
original = json.dumps({"task_status": "completed", "test_result": ai}, ensure_ascii=False)
|
||
outcome_file.write_text(original, encoding="utf-8")
|
||
archive = reports.export_batch(batch["id"])
|
||
isolated.put("batch", batch["id"], {**batch, "report_ready": True})
|
||
|
||
response = client.put(
|
||
f"/api/automation/reports/{session_id}/review",
|
||
json={"status": "passed", "reason": "已人工核对，符合预期"},
|
||
)
|
||
assert response.status_code == 200
|
||
assert response.json()["status"] == "completed"
|
||
assert response.json()["verdict"] == "测试通过（人工确认）"
|
||
assert outcome_file.read_text(encoding="utf-8") == original
|
||
assert not archive.exists()
|
||
assert not isolated.get("batch", batch["id"])["report_ready"]
|
||
assert len(isolated.get("case_review", session_id)["reviews"]) == 1
|
||
for result in (
|
||
response.json(),
|
||
client.get("/api/automation/history", params={"q": session_id}).json()["items"][0],
|
||
client.get(f"/api/automation/batches/{batch['id']}").json()["jobs"][0],
|
||
):
|
||
assert result["test_result"]["source"] == "human"
|
||
assert result["test_result"]["status"] == "passed"
|
||
assert result["test_result"]["ai_result"] == {**ai, "source": "ai"}
|
||
assert result["test_result"]["reviewed_at"] > 0
|
||
|
||
monkeypatch.setattr(sessions_api, "store", isolated)
|
||
monkeypatch.setattr(sessions_api, "TRACES_PATH", reports.TRACES_PATH)
|
||
monkeypatch.setattr(
|
||
sessions_api.session_repo,
|
||
"get_session_by_id",
|
||
lambda sid: {"session_id": sid, "status": "completed"},
|
||
)
|
||
client.app.include_router(sessions_api.router)
|
||
detail = client.get(f"/api/sessions/{session_id}").json()
|
||
assert detail["test_result"]["source"] == "human"
|
||
assert detail["batch_id"] == batch["id"]
|
||
|
||
downloaded = client.get(f"/api/automation/batches/{batch['id']}/archive")
|
||
assert downloaded.status_code == 200
|
||
with zipfile.ZipFile(BytesIO(downloaded.content)) as bundle:
|
||
wb = load_workbook(BytesIO(bundle.read("批次结果.xlsx")))
|
||
assert wb.active["D2"].value == "测试通过（人工确认）"
|
||
assert "AI确认" in wb.active["D3"].value
|
||
doc = Document(BytesIO(bundle.read("批次汇总.docx")))
|
||
assert doc.tables[0].rows[1].cells[3].text == "测试通过（人工确认）"
|
||
|
||
changed = client.put(f"/api/automation/reports/{session_id}/review", json={"status": "failed"})
|
||
assert changed.status_code == 200
|
||
assert changed.json()["verdict"] == "测试不通过（人工确认）"
|
||
assert len(isolated.get("case_review", session_id)["reviews"]) == 2
|
||
assert outcome_file.read_text(encoding="utf-8") == original
|
||
|
||
|
||
def test_review_rejects_active_missing_invalid_and_packing_tasks(client, isolated, monkeypatch):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
url = f"/api/automation/reports/{job['session_id']}/review"
|
||
assert client.put(url, json={"status": "passed"}).status_code == 409
|
||
assert isolated.get("case_review", job["session_id"]) is None
|
||
assert client.put(url, json={"status": "inconclusive"}).status_code == 422
|
||
assert client.put(url, json={"status": "passed", "source": "human"}).status_code == 422
|
||
assert client.put(url, json={"status": "passed", "reason": "字" * 2001}).status_code == 422
|
||
assert (
|
||
client.put("/api/automation/reports/missing/review", json={"status": "passed"}).status_code
|
||
== 404
|
||
)
|
||
isolated.save_job({**job, "status": "completed"})
|
||
monkeypatch.setattr(api.archive_service, "status", lambda *args: {"status": "preparing"})
|
||
assert client.put(url, json={"status": "passed"}).status_code == 409
|
||
assert isolated.get("case_review", job["session_id"]) is None
|
||
|
||
|
||
def test_batch_filter_searches_full_history_and_reviews_are_removed_with_batch(client, isolated):
|
||
first, second = [service.create_batch(payload(name=name)) for name in ("第一批", "第二批")]
|
||
for batch in (first, second):
|
||
for job in isolated.jobs(batch["id"]):
|
||
isolated.save_job({**job, "status": "completed"})
|
||
with isolated.connection() as conn:
|
||
conn.execute(
|
||
"INSERT INTO sessions(session_id,initial_goal,start_time,status) VALUES (?,?,?,?)",
|
||
("single-case", "单独检查", time.time(), "completed"),
|
||
)
|
||
conn.commit()
|
||
filtered = client.get(
|
||
"/api/automation/history",
|
||
params={"batch_id": first["id"], "q": "用例", "size": 1, "page": 2},
|
||
).json()
|
||
assert filtered["total"] == 2
|
||
assert len(filtered["items"]) == 1
|
||
assert filtered["items"][0]["batch_id"] == first["id"]
|
||
assert (
|
||
client.get("/api/automation/history", params={"batch_id": "__batch__"}).json()["total"] == 4
|
||
)
|
||
singles = client.get("/api/automation/history", params={"batch_id": "__single__"}).json()
|
||
assert [item["session_id"] for item in singles["items"]] == ["single-case"]
|
||
assert client.get("/api/automation/history", params={"batch_id": "bad/path"}).status_code == 400
|
||
reviewed_id = isolated.jobs(first["id"])[0]["session_id"]
|
||
isolated.save_case_review(reviewed_id, "passed", "人工核对")
|
||
isolated.begin_delete_batch(first["id"])
|
||
isolated.delete_batch(first["id"])
|
||
assert isolated.get("case_review", reviewed_id) is None
|
||
assert len(isolated.jobs(second["id"])) == 2
|
||
isolated.save_case_review("single-case", "failed", "不符合预期")
|
||
isolated.delete_history("single-case")
|
||
assert isolated.get("case_review", "single-case") is None
|
||
|
||
|
||
def test_task_material_zip_contains_ordered_evidence_and_reports(
|
||
client, isolated, tmp_path, monkeypatch
|
||
):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
isolated.save_job({**job, "case": {**job["case"], "case_id": "1.1"}})
|
||
artifact_dir = reports.TRACES_PATH / job["session_id"] / "artifacts" / "android"
|
||
artifact_dir.mkdir(parents=True)
|
||
(artifact_dir / "安卓截图.png").write_bytes(b"native png")
|
||
(artifact_dir / ".artemis-partial.tmp").write_bytes(b"incomplete")
|
||
evidence = tmp_path / "evidence"
|
||
evidence.mkdir()
|
||
first = evidence / "first.png"
|
||
second = evidence / "second.png"
|
||
video = evidence / "recording.mp4"
|
||
for path in (first, second, video):
|
||
path.write_bytes(path.name.encode())
|
||
monkeypatch.setattr(
|
||
reports,
|
||
"session_screenshots",
|
||
lambda report: [(first, "0001_步骤0001_操作前.png"), (second, "0002_步骤0002_操作后.png")],
|
||
)
|
||
monkeypatch.setattr(reports, "session_videos", lambda report: [(video, "0001_recording.mp4")])
|
||
|
||
response = client.get(f"/api/automation/reports/{job['session_id']}/all")
|
||
assert response.status_code == 200
|
||
with zipfile.ZipFile(BytesIO(response.content)) as archive:
|
||
files = archive.namelist()
|
||
assert all(name.startswith("1.1/") for name in files)
|
||
assert any(name.endswith(".docx") for name in files)
|
||
assert any(name.endswith(".xlsx") for name in files)
|
||
assert files.index("1.1/步骤截图/0001_步骤0001_操作前.png") < files.index(
|
||
"1.1/步骤截图/0002_步骤0002_操作后.png"
|
||
)
|
||
assert archive.read("1.1/任务录屏/0001_recording.mp4") == b"recording.mp4"
|
||
manifest = json.loads(archive.read("1.1/文件清单.json"))
|
||
assert manifest["session_id"] == job["session_id"]
|
||
assert manifest["artifacts"] == ["安卓截图.png"]
|
||
assert archive.read("1.1/任务文件/安卓/安卓截图.png") == b"native png"
|
||
assert not any("partial" in name for name in files)
|
||
assert "1.1-all.zip" in response.headers["content-disposition"]
|
||
|
||
|
||
def test_batch_zip_contains_each_tasks_android_artifacts(isolated):
|
||
batch = service.create_batch(payload())
|
||
for index, job in enumerate(isolated.jobs(batch["id"])):
|
||
root = reports.TRACES_PATH / job["session_id"] / "artifacts" / "android"
|
||
root.mkdir(parents=True)
|
||
(root / "结果.txt").write_text(f"任务{index}", encoding="utf-8")
|
||
with zipfile.ZipFile(reports.export_batch(batch["id"])) as archive:
|
||
names = [name for name in archive.namelist() if name.endswith("任务文件/安卓/结果.txt")]
|
||
assert len(names) == 2
|
||
assert {archive.read(name).decode() for name in names} == {"任务0", "任务1"}
|
||
|
||
|
||
def test_android_artifact_export_does_not_follow_other_session_links(isolated):
|
||
root = reports.TRACES_PATH / "session" / "artifacts"
|
||
root.mkdir(parents=True)
|
||
other = reports.TRACES_PATH / "other-session"
|
||
other.mkdir()
|
||
(other / "private.txt").write_text("other task")
|
||
try:
|
||
(root / "android").symlink_to(other, target_is_directory=True)
|
||
except OSError:
|
||
pytest.skip("symlinks unavailable")
|
||
assert reports.session_artifacts({"id": "session"}) == []
|
||
|
||
|
||
def test_video_export_accepts_finalized_worker_trace_and_rejects_external_path(
|
||
isolated, tmp_path, monkeypatch
|
||
):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
session_id = job["session_id"]
|
||
recording_dir = reports.TRACES_PATH / f"web_123_{session_id[:8]}_PASS_2026-09-28"
|
||
recording_dir.mkdir(parents=True)
|
||
first = recording_dir / "recording.mp4"
|
||
first.write_bytes(b"first segment")
|
||
resumed = recording_dir / "resume_0001"
|
||
resumed.mkdir()
|
||
second = resumed / "recording.mp4"
|
||
second.write_bytes(b"second segment")
|
||
|
||
repository = Mock()
|
||
repository.get_video_recording_for_session.return_value = {"local_video_path": str(first)}
|
||
repository.get_session_by_id.return_value = {"video_filepath": str(first)}
|
||
monkeypatch.setattr(reports, "SessionRepository", lambda: repository)
|
||
|
||
archive_path = reports.export_session_media(session_id, "video")
|
||
with zipfile.ZipFile(archive_path) as archive:
|
||
names = archive.namelist()
|
||
assert names == [
|
||
"任务录屏/0001_web_123_" + session_id[:8] + "_PASS_2026-09-28_recording.mp4",
|
||
"任务录屏/0002_resume_0001_recording.mp4",
|
||
"文件清单.json",
|
||
]
|
||
assert archive.read(names[0]) == b"first segment"
|
||
assert archive.read(names[1]) == b"second segment"
|
||
|
||
external = tmp_path / "outside" / "recording.mp4"
|
||
external.parent.mkdir()
|
||
external.write_bytes(b"unrelated")
|
||
repository.get_video_recording_for_session.return_value = {"local_video_path": str(external)}
|
||
assert reports.session_videos({"id": session_id}) == []
|
||
|
||
|
||
def test_history_filters_all_persisted_rows_and_escapes_like(client, isolated):
|
||
batch = service.create_batch(payload())
|
||
first, second = isolated.jobs(batch["id"])
|
||
isolated.save_job(
|
||
{
|
||
**first,
|
||
"status": "completed",
|
||
"created_at": 100,
|
||
"case": {"name": "搜索_%", "goal": "abc", "tags": "回归"},
|
||
}
|
||
)
|
||
isolated.save_job({**second, "status": "failed", "created_at": 200})
|
||
assert client.get("/api/automation/history", params={"q": "_%"}).json()["total"] == 1
|
||
assert (
|
||
client.get("/api/automation/history", params={"q": "回归"}).json()["items"][0]["session_id"]
|
||
== first["session_id"]
|
||
)
|
||
assert (
|
||
client.get(
|
||
"/api/automation/history", params={"start": 150, "end": 250, "status": "failed"}
|
||
).json()["total"]
|
||
== 1
|
||
)
|
||
assert (
|
||
client.get("/api/automation/history", params={"size": 1, "page": 2}).json()["items"][0][
|
||
"session_id"
|
||
]
|
||
== first["session_id"]
|
||
)
|
||
|
||
|
||
def test_batch_submission_is_atomic_and_idempotent(client, isolated):
|
||
value = payload(request_id="same-request")
|
||
first = client.post("/api/automation/batches", json=value)
|
||
second = client.post("/api/automation/batches", json=value)
|
||
assert first.status_code == second.status_code == 200
|
||
assert first.json()["id"] == second.json()["id"]
|
||
assert len(isolated.jobs()) == 2
|
||
assert len(isolated.records("batch")) == 1
|
||
conflict = client.post("/api/automation/batches", json={**value, "name": "different"})
|
||
assert conflict.status_code == 409
|
||
lookup = client.get("/api/automation/batches/by-request/same-request").json()
|
||
assert lookup["id"] == first.json()["id"]
|
||
assert len(lookup["jobs"]) == 2
|
||
|
||
|
||
def test_removed_replay_is_inert_but_history_survives(client, isolated):
|
||
batch = service.create_batch(payload())
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**job, "execution_mode": "strict", "workflow_id": "legacy"})
|
||
service.recover_batches()
|
||
assert isolated.get("batch", batch["id"])["status"] == "cancelled"
|
||
assert all(j["status"] == "cancelled" for j in isolated.jobs(batch["id"]))
|
||
assert client.post(f"/api/automation/batches/{batch['id']}/resume").status_code == 409
|
||
assert client.post(f"/api/automation/batches/{batch['id']}/retry_failed").status_code == 409
|
||
assert client.get("/api/automation/workflows").status_code == 404
|
||
assert client.get("/api/automation/history").json()["total"] == 2
|