1802 lines
78 KiB
Python
1802 lines
78 KiB
Python
"""Automation acceptance checks with isolated persistence and no real device side effects."""
|
||
|
||
from io import BytesIO
|
||
import json
|
||
import sqlite3
|
||
import threading
|
||
import time
|
||
from unittest.mock import AsyncMock, Mock
|
||
from uuid import UUID
|
||
import zipfile
|
||
|
||
from docx import Document
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
from openpyxl import load_workbook
|
||
import pytest
|
||
|
||
from apps.admin_console.routers import automation as api
|
||
from apps.admin_console.services import automation_service as service
|
||
from apps.admin_console.services import report_export as reports
|
||
from apps.admin_console.services import task_import
|
||
from apps.admin_console.services.automation_store import AutomationStore
|
||
from artemis.data_engine.storage import StorageManager
|
||
|
||
|
||
@pytest.fixture
|
||
def isolated(tmp_path, monkeypatch):
|
||
db = tmp_path / "data.db"
|
||
StorageManager(db_path=db, base_trace_dir=tmp_path)
|
||
store = AutomationStore(db)
|
||
for module in (api, service, reports):
|
||
monkeypatch.setattr(module, "store", store)
|
||
monkeypatch.setattr(reports, "TRACES_PATH", tmp_path / "traces")
|
||
monkeypatch.setattr(reports, "EXPORT_ROOT", tmp_path / "exports")
|
||
from apps.admin_console.database.repositories.session_repository import SessionRepository
|
||
from apps.admin_console.database.repositories.step_repository import StepRepository
|
||
|
||
monkeypatch.setattr(reports, "SessionRepository", lambda *a: SessionRepository(db))
|
||
monkeypatch.setattr(reports, "StepRepository", lambda *a: StepRepository(db))
|
||
return store
|
||
|
||
|
||
@pytest.fixture
|
||
def client(isolated):
|
||
app = FastAPI()
|
||
app.include_router(api.router)
|
||
return TestClient(app)
|
||
|
||
|
||
def payload(**extra):
|
||
return {
|
||
"name": "批次",
|
||
"cases": [{"name": "用例甲", "goal": "观察设置"}, {"name": "用例乙", "goal": "观察首页"}],
|
||
"device_serial": "test-device",
|
||
**extra,
|
||
}
|
||
|
||
|
||
def workbook_bytes(rows):
|
||
wb = load_workbook(BytesIO(task_import.template()))
|
||
for row in rows:
|
||
wb["任务"].append(row)
|
||
output = BytesIO()
|
||
wb.save(output)
|
||
return output.getvalue()
|
||
|
||
|
||
def test_template_preview_rejects_formula_and_bad_json():
|
||
data = workbook_bytes(
|
||
[
|
||
["正常", "设置", "冒烟", "打开设置", "", "观察设置", '{"profile":"flash"}'],
|
||
["公式", "", "", "", "", "=SUM(1,2)"],
|
||
["参数错误", "", "", "", "", "测试", "{"],
|
||
]
|
||
)
|
||
result = task_import.parse_workbook(data)
|
||
assert len(result["tasks"]) == 1
|
||
assert [e["row"] for e in result["errors"]] == [3, 4]
|
||
assert not result["valid"]
|
||
|
||
|
||
def test_prompt_only_and_per_row_config_validation():
|
||
goal = "先查看屏幕，再按实际画面判断。"
|
||
data = workbook_bytes(
|
||
[
|
||
[
|
||
"CASE-7",
|
||
"检查项目",
|
||
"子项",
|
||
"用例名称",
|
||
"仅作索引的约束",
|
||
goal,
|
||
'{"profile":"pro","model":"mimo-v2.6-flash","record_video":true}',
|
||
],
|
||
["CASE-8", "", "", "", "", "观察屏幕", '{"OPENAI_API_KEY":"never"}'],
|
||
]
|
||
)
|
||
preview = task_import.parse_workbook(data)
|
||
assert [e["row"] for e in preview["errors"]] == [3]
|
||
case = preview["tasks"][0]
|
||
assert task_import.case_goal(case) == goal
|
||
assert case["preconditions"] == "仅作索引的约束"
|
||
assert case["task_config"]["record_video"] is True
|
||
|
||
|
||
def test_invalid_archive_and_empty_template():
|
||
with pytest.raises(ValueError):
|
||
task_import.parse_workbook(b"not a zip")
|
||
assert not task_import.parse_workbook(task_import.template())["valid"]
|
||
|
||
|
||
def test_pc_json_import_does_not_require_a_device_and_rejects_unknown_targets():
|
||
data = workbook_bytes(
|
||
[
|
||
["PC-1", "电脑", "", "浏览器", "", "检查桌面", '{"execution_target":"computer"}'],
|
||
["BAD-2", "", "", "", "", "检查页面", '{"execution_target":"pc"}'],
|
||
]
|
||
)
|
||
result = task_import.parse_workbook(data)
|
||
assert [item["row"] for item in result["errors"]] == [3]
|
||
options = result["tasks"][0]["task_config"]
|
||
assert options["execution_target"] == "computer"
|
||
assert options["device_serial"] is None
|
||
|
||
|
||
def test_batch_archive_preparation_reports_progress_and_reuses_ready_file(
|
||
client, isolated, monkeypatch
|
||
):
|
||
batch = service.create_batch(payload())
|
||
batch_id = batch["id"]
|
||
started = threading.Event()
|
||
release = threading.Event()
|
||
calls = []
|
||
|
||
def slow_export(identifier, progress=None):
|
||
calls.append(identifier)
|
||
progress(0, 2, "正在打包任务 1/2")
|
||
started.set()
|
||
assert release.wait(5)
|
||
progress(2, 2, "正在完成压缩包")
|
||
path = reports.EXPORT_ROOT / "batches" / identifier / "批次报告.zip"
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_bytes(b"prepared archive")
|
||
return path
|
||
|
||
monkeypatch.setattr(reports, "export_batch", slow_export)
|
||
base = f"/api/automation/batches/{batch_id}/archive"
|
||
try:
|
||
response = client.post(base + "/prepare")
|
||
assert response.status_code == 200
|
||
assert response.json()["status"] == "preparing"
|
||
assert started.wait(2)
|
||
progress = client.get(base + "/status").json()
|
||
assert progress["total"] == 2
|
||
assert progress["stage"] == "正在打包任务 1/2"
|
||
assert client.get(base).status_code == 409
|
||
finally:
|
||
release.set()
|
||
for _ in range(50):
|
||
if client.get(base + "/status").json()["status"] == "ready":
|
||
break
|
||
time.sleep(0.02)
|
||
assert client.get(base + "/status").json()["status"] == "ready"
|
||
assert client.get(base).content == b"prepared archive"
|
||
assert calls == [batch_id]
|
||
|
||
|
||
def test_completed_batch_reuses_existing_archive(client, isolated, monkeypatch):
|
||
batch = service.create_batch(payload())
|
||
batch_id = batch["id"]
|
||
isolated.put("batch", batch_id, {**batch, "status": "completed", "report_ready": True})
|
||
path = reports.EXPORT_ROOT / "batches" / batch_id / "批次报告.zip"
|
||
path.parent.mkdir(parents=True)
|
||
path.write_bytes(b"existing archive")
|
||
reports.remember(path, reports.BATCH_ARCHIVE_SIGNATURE)
|
||
exporter = Mock(side_effect=AssertionError("archive must not be rebuilt"))
|
||
monkeypatch.setattr(reports, "export_batch", exporter)
|
||
base = f"/api/automation/batches/{batch_id}/archive"
|
||
assert client.post(base + "/prepare").json()["status"] == "ready"
|
||
assert client.get(base).content == b"existing archive"
|
||
exporter.assert_not_called()
|
||
|
||
|
||
@pytest.mark.parametrize("signature", [None, "artemis-batch-layout-3"])
|
||
def test_legacy_batch_archive_is_rebuilt_with_runtime_diagnostics(client, isolated, signature):
|
||
batch = service.create_batch(payload())
|
||
batch_id = batch["id"]
|
||
isolated.put("batch", batch_id, {**batch, "status": "completed", "report_ready": True})
|
||
path = reports.EXPORT_ROOT / "batches" / batch_id / "批次报告.zip"
|
||
path.parent.mkdir(parents=True)
|
||
path.write_bytes(b"legacy archive")
|
||
if signature:
|
||
reports.remember(path, signature)
|
||
base = f"/api/automation/batches/{batch_id}/archive"
|
||
assert client.get(base + "/status").json()["status"] == "idle"
|
||
response = client.get(base)
|
||
assert response.status_code == 200
|
||
assert zipfile.is_zipfile(BytesIO(response.content))
|
||
with zipfile.ZipFile(BytesIO(response.content)) as archive:
|
||
assert len([name for name in archive.namelist() if name.endswith("运行环境诊断.json")]) == 2
|
||
assert client.get(base + "/status").json()["status"] == "ready"
|
||
|
||
|
||
def test_preview_does_not_enqueue(client, isolated):
|
||
response = client.post(
|
||
"/api/automation/import/preview",
|
||
files={
|
||
"file": ("tasks.xlsx", workbook_bytes([["案例", "", "", "观察页面", "", "观察页面"]]))
|
||
},
|
||
)
|
||
assert response.status_code == 200
|
||
assert response.json()["valid"]
|
||
assert not isolated.jobs()
|
||
|
||
|
||
def test_preferences_and_required_device(client):
|
||
assert client.get("/api/automation/preferences").json()["formats"] == ["docx", "xlsx"]
|
||
assert client.put("/api/automation/preferences", json={"formats": ["xlsx"]}).status_code == 200
|
||
assert client.put("/api/automation/preferences", json={"formats": []}).status_code == 422
|
||
assert client.post("/api/automation/batches", json=payload(device_serial="")).status_code == 422
|
||
|
||
|
||
def test_batch_repetitions_are_ordered_independent_and_idempotent(client, isolated):
|
||
request = payload(repeat_count=3, request_id="repeat-test")
|
||
created = client.post("/api/automation/batches", json=request)
|
||
assert created.status_code == 200
|
||
batch = created.json()
|
||
assert (batch["case_count"], batch["repeat_count"], batch["total_executions"]) == (2, 3, 6)
|
||
jobs = isolated.jobs(batch["id"])
|
||
assert len({j["session_id"] for j in jobs}) == 6
|
||
assert [j["position"] for j in jobs] == list(range(1, 7))
|
||
assert [(j["iteration"], j["case_position"]) for j in jobs] == [
|
||
(1, 1),
|
||
(1, 2),
|
||
(2, 1),
|
||
(2, 2),
|
||
(3, 1),
|
||
(3, 2),
|
||
]
|
||
assert [j["case"]["goal"] for j in jobs] == ["观察设置", "观察首页"] * 3
|
||
assert all(j["status"] == "pending" for j in jobs)
|
||
assert client.post("/api/automation/batches", json=request).json()["id"] == batch["id"]
|
||
assert len(isolated.jobs(batch["id"])) == 6
|
||
assert (
|
||
client.post("/api/automation/batches", json={**request, "repeat_count": 2}).status_code
|
||
== 409
|
||
)
|
||
# Each selected failure is copied once, including failures from different rounds.
|
||
for job in (jobs[0], jobs[4]):
|
||
isolated.save_job({**job, "status": "failed"})
|
||
retry = service.batch_action(batch["id"], "retry_failed")
|
||
assert retry["repeat_count"] == 1
|
||
assert len(isolated.jobs(retry["id"])) == 2
|
||
report = reports.session_report(jobs[4]["session_id"])
|
||
assert "第 3 / 3 轮" in report["name"]
|
||
archive = reports.export_batch(batch["id"])
|
||
with zipfile.ZipFile(archive) as z:
|
||
assert len([n for n in z.namelist() if n.endswith(".xlsx")]) == 7
|
||
assert len(set(z.namelist())) == len(z.namelist())
|
||
|
||
|
||
@pytest.mark.parametrize("count", [0, -1, 1.5, True, "3", None])
|
||
def test_batch_rejects_invalid_repetitions(client, isolated, count):
|
||
assert (
|
||
client.post("/api/automation/batches", json=payload(repeat_count=count)).status_code == 422
|
||
)
|
||
assert isolated.records("batch") == []
|
||
|
||
|
||
def test_batch_limit_is_checked_before_creating_records(client, isolated, monkeypatch):
|
||
monkeypatch.setattr(service.settings, "ARTEMIS_BATCH_MAX_EXECUTIONS", 5)
|
||
assert client.get("/api/automation/preferences").json()["batch_max_executions"] == 5
|
||
response = client.post("/api/automation/batches", json=payload(repeat_count=3))
|
||
assert response.status_code == 409
|
||
assert "5" in response.json()["detail"]
|
||
assert not isolated.records("batch")
|
||
assert not isolated.jobs()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_loop_dispatch_continues_after_failure_and_resume_without_repeating(
|
||
isolated, monkeypatch
|
||
):
|
||
from apps.admin_console import server
|
||
from artemis.runtime.device_pool import DeviceStatus
|
||
|
||
batch = service.create_batch(payload(repeat_count=2))
|
||
service.batch_action(batch["id"], "resume")
|
||
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
||
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
||
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async",
|
||
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
||
)
|
||
for index in range(4):
|
||
await service.tick_batches()
|
||
await service.tick_batches() # Polling must not enqueue the same execution twice.
|
||
assert enqueue.await_count == index + 1
|
||
job = isolated.jobs(batch["id"])[index]
|
||
assert enqueue.call_args.kwargs["session_id"] == job["session_id"]
|
||
assert f"第 {job['iteration']} / 2 轮" in enqueue.call_args.kwargs["display_name"]
|
||
isolated.save_job({**job, "status": "failed" if index == 0 else "completed"})
|
||
if index == 1:
|
||
service.batch_action(batch["id"], "pause")
|
||
await service.tick_batches()
|
||
assert enqueue.await_count == 2
|
||
service.batch_action(batch["id"], "resume")
|
||
await service.tick_batches()
|
||
assert isolated.get("batch", batch["id"])["status"] == "completed"
|
||
assert enqueue.await_count == 4
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_repeat_fail_fast_skips_future_rounds(isolated):
|
||
batch = service.create_batch(payload(repeat_count=3, stop_on_failure=True))
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": "failed"})
|
||
service.batch_action(batch["id"], "resume")
|
||
await service.tick_batches()
|
||
assert [j["status"] for j in isolated.jobs(batch["id"])] == ["failed"] + ["skipped"] * 5
|
||
|
||
|
||
def test_batch_delete_removes_tasks_and_files_and_prevents_request_replay(client, isolated):
|
||
created = client.post(
|
||
"/api/automation/batches", json=payload(request_id="delete-example")
|
||
).json()
|
||
first, second = isolated.jobs(created["id"])
|
||
isolated.save_job({**first, "status": "completed"})
|
||
trace_dir = reports.TRACES_PATH / first["session_id"]
|
||
trace_dir.mkdir(parents=True)
|
||
(trace_dir / "result.json").write_text("{}", encoding="utf-8")
|
||
report_dir = reports.EXPORT_ROOT / "sessions" / first["session_id"]
|
||
report_dir.mkdir(parents=True)
|
||
(report_dir / "任务报告.docx").write_bytes(b"report")
|
||
batch_dir = reports.EXPORT_ROOT / "batches" / created["id"]
|
||
batch_dir.mkdir(parents=True)
|
||
(batch_dir / "批次报告.zip").write_bytes(b"zip")
|
||
mcp_task_dir = reports.EXPORT_ROOT.parent / "mcp_exports" / "sessions" / first["session_id"]
|
||
mcp_task_dir.mkdir(parents=True)
|
||
(mcp_task_dir / "报告.docx").write_bytes(b"report")
|
||
mcp_batch_dir = reports.EXPORT_ROOT.parent / "mcp_exports" / "batches" / created["id"]
|
||
mcp_batch_dir.mkdir(parents=True)
|
||
(mcp_batch_dir / "批次.zip").write_bytes(b"zip")
|
||
unrelated = reports.TRACES_PATH / "unrelated-task"
|
||
unrelated.mkdir()
|
||
(unrelated / "keep.txt").write_text("keep", encoding="utf-8")
|
||
|
||
response = client.delete(f"/api/automation/batches/{created['id']}")
|
||
assert response.status_code == 200
|
||
assert client.get(f"/api/automation/batches/{created['id']}").status_code == 404
|
||
assert isolated.job(first["session_id"]) is None
|
||
assert isolated.job(second["session_id"]) is None
|
||
assert not trace_dir.exists()
|
||
assert not report_dir.exists()
|
||
assert not batch_dir.exists()
|
||
assert not mcp_task_dir.exists()
|
||
assert not mcp_batch_dir.exists()
|
||
assert (unrelated / "keep.txt").read_text(encoding="utf-8") == "keep"
|
||
with pytest.raises(ValueError, match="不存在"):
|
||
reports.session_report(first["session_id"])
|
||
assert (
|
||
client.post(
|
||
"/api/automation/batches", json=payload(request_id="delete-example")
|
||
).status_code
|
||
== 409
|
||
)
|
||
|
||
|
||
@pytest.mark.parametrize("status", ["queued", "running", "task_paused", "resuming", "finalizing"])
|
||
def test_batch_delete_rejects_active_jobs(client, isolated, status):
|
||
batch = service.create_batch(payload())
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": status})
|
||
|
||
response = client.delete(f"/api/automation/batches/{batch['id']}")
|
||
assert response.status_code == 409
|
||
assert isolated.get("batch", batch["id"]) is not None
|
||
assert len(isolated.jobs(batch["id"])) == 2
|
||
|
||
|
||
def test_batch_deletion_reservation_blocks_new_dispatch(isolated):
|
||
batch = service.create_batch(payload())
|
||
job = isolated.jobs(batch["id"])[0]
|
||
|
||
isolated.begin_delete_batch(batch["id"])
|
||
|
||
with pytest.raises(ValueError, match="正在删除"):
|
||
isolated.save_job({**job, "status": "queued"})
|
||
with pytest.raises(ValueError, match="正在删除"):
|
||
isolated.put("batch", batch["id"], {**batch, "status": "running"})
|
||
assert isolated.jobs_for_sessions([job["session_id"]])[job["session_id"]]["status"] == "pending"
|
||
|
||
|
||
def test_batch_deletion_preserves_shared_screenshot_until_last_task(client, isolated):
|
||
first_batch = service.create_batch(payload())
|
||
second_batch = service.create_batch(payload(name="另一批次"))
|
||
first = isolated.jobs(first_batch["id"])[0]
|
||
other = isolated.jobs(second_batch["id"])[0]
|
||
shared_hash = "a" * 64
|
||
storage = StorageManager(isolated.path, reports.TRACES_PATH)
|
||
image_file = reports.TRACES_PATH / "images" / f"{shared_hash}.jpg"
|
||
image_file.parent.mkdir(parents=True)
|
||
image_file.write_bytes(b"shared image")
|
||
with sqlite3.connect(isolated.path) as conn:
|
||
conn.execute("INSERT INTO images(image_name) VALUES (?)", (shared_hash,))
|
||
for job in (first, other):
|
||
conn.execute(
|
||
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info) "
|
||
"VALUES (?,?,?,?,?)",
|
||
(job["session_id"], "观察设置", 1.0, "completed", "{}"),
|
||
)
|
||
conn.execute(
|
||
"INSERT INTO steps(step_id,session_id,pre_image_name) VALUES (?,?,?)",
|
||
(job["session_id"], job["session_id"], shared_hash),
|
||
)
|
||
|
||
assert client.delete(f"/api/automation/batches/{first_batch['id']}").status_code == 200
|
||
assert image_file.read_bytes() == b"shared image"
|
||
assert storage.get_session(UUID(other["session_id"])) is not None
|
||
assert client.delete(f"/api/automation/batches/{second_batch['id']}").status_code == 200
|
||
assert not image_file.exists()
|
||
|
||
|
||
@pytest.mark.parametrize("model_paused", [False, True])
|
||
def test_restart_never_repeats_started_work(isolated, model_paused):
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
if model_paused:
|
||
isolated.put(
|
||
"batch",
|
||
batch["id"],
|
||
{**isolated.get("batch", batch["id"]), "status": "paused", "pause_source": "model"},
|
||
)
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**job, "status": "queued"})
|
||
service.recover_batches()
|
||
assert isolated.get("batch", batch["id"])["status"] == "paused"
|
||
assert isolated.get("batch", batch["id"])["pause_source"] == "recovery"
|
||
assert [j["status"] for j in isolated.jobs(batch["id"])] == ["interrupted", "pending"]
|
||
retry = service.batch_action(batch["id"], "retry_failed")
|
||
assert retry["status"] == "draft"
|
||
assert len(isolated.jobs(retry["id"])) == 1
|
||
assert isolated.jobs(retry["id"])[0]["session_id"] != job["session_id"]
|
||
|
||
|
||
@pytest.mark.parametrize("liveness_source", ["pid", "device_lock"])
|
||
def test_recovery_preserves_live_runner_and_batch(isolated, monkeypatch, liveness_source):
|
||
import os
|
||
from types import SimpleNamespace
|
||
from artemis.runtime import DeviceExecutionLock
|
||
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**job, "status": "running"})
|
||
if liveness_source == "pid":
|
||
with sqlite3.connect(isolated.path) as conn:
|
||
conn.execute(
|
||
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) "
|
||
"VALUES (?,?,?,?,?,?)",
|
||
(job["session_id"], "goal", 1, "running", "{}", os.getpid()),
|
||
)
|
||
monkeypatch.setattr(
|
||
DeviceExecutionLock,
|
||
"get_active_owners",
|
||
lambda: (
|
||
{"device": SimpleNamespace(session_id=job["session_id"])}
|
||
if liveness_source == "device_lock"
|
||
else {}
|
||
),
|
||
)
|
||
service.recover_batches()
|
||
assert isolated.job(job["session_id"])["status"] == "running"
|
||
assert not isolated.job(job["session_id"]).get("error")
|
||
assert isolated.get("batch", batch["id"])["status"] == "running"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_natural_completion_resolves_old_recovery_error_in_job_report_and_history(
|
||
client, isolated
|
||
):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
message = "服务中断；未完成任务需手动重跑"
|
||
isolated.save_job({**job, "status": "interrupted", "error": message})
|
||
await service.on_terminal({"session_id": job["session_id"]}, "completed")
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "completed"
|
||
assert not saved.get("error")
|
||
assert saved["recovery_events"][0]["message"] == message
|
||
assert reports.session_report(job["session_id"])["error"] == ""
|
||
assert not client.get("/api/automation/history").json()["items"][0].get("error")
|
||
|
||
|
||
def test_recovery_repairs_completed_legacy_warning_but_preserves_real_failure(isolated):
|
||
batch = service.create_batch(payload())
|
||
first, second = isolated.jobs(batch["id"])
|
||
isolated.save_job({**first, "status": "completed", "error": "服务中断，需手动重跑"})
|
||
isolated.save_job({**second, "status": "failed", "error": "任务执行超时（30 秒）"})
|
||
assert reports.session_report(first["session_id"])["error"] == ""
|
||
service.recover_batches()
|
||
assert not isolated.job(first["session_id"]).get("error")
|
||
assert "服务中断" in isolated.job(first["session_id"])["recovery_events"][0]["message"]
|
||
assert "超时" in isolated.job(second["session_id"])["error"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_surviving_worker_is_finalized_when_it_exits_without_resubmission(
|
||
isolated, monkeypatch
|
||
):
|
||
import os
|
||
from artemis.runtime import DeviceExecutionLock
|
||
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**job, "status": "running"})
|
||
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
|
||
with sqlite3.connect(isolated.path) as conn:
|
||
conn.execute(
|
||
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info,pid) "
|
||
"VALUES (?,?,?,?,?,?)",
|
||
(job["session_id"], "goal", time.time(), "running", "{}", os.getpid()),
|
||
)
|
||
service.recover_batches()
|
||
await service.reconcile_recovered_jobs()
|
||
assert isolated.job(job["session_id"])["status"] == "running"
|
||
with sqlite3.connect(isolated.path) as conn:
|
||
conn.execute(
|
||
"UPDATE sessions SET pid=NULL,status='completed' WHERE session_id=?",
|
||
(job["session_id"],),
|
||
)
|
||
await service.reconcile_recovered_jobs()
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "completed"
|
||
assert saved["report_ready"]
|
||
assert not saved.get("recovery_waiting")
|
||
assert not saved.get("error")
|
||
assert len(isolated.jobs(batch["id"])) == 2
|
||
assert isolated.jobs(batch["id"])[1]["status"] == "pending"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_surviving_worker_that_dies_without_result_is_interrupted(isolated, monkeypatch):
|
||
from artemis.runtime import DeviceExecutionLock
|
||
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
isolated.save_job({**job, "status": "running", "recovery_waiting": True})
|
||
monkeypatch.setattr(DeviceExecutionLock, "get_active_owners", lambda: {})
|
||
await service.reconcile_recovered_jobs()
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "interrupted"
|
||
assert "服务中断" in saved["error"]
|
||
assert not saved.get("recovery_waiting")
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_batch_admission_idempotent_and_bound_device(isolated, monkeypatch):
|
||
from apps.admin_console import server
|
||
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
||
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
||
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.validate_explicit_serial_async", AsyncMock(return_value=None)
|
||
)
|
||
from artemis.runtime.device_pool import DeviceStatus
|
||
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async",
|
||
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
||
)
|
||
await service.tick_batches()
|
||
await service.tick_batches()
|
||
assert enqueue.await_count == 1
|
||
assert enqueue.call_args.kwargs["device_serial"] == "test-device"
|
||
assert isolated.jobs(batch["id"])[1]["status"] == "pending"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_paused_case_yields_batch_slot_to_next_case(isolated, monkeypatch):
|
||
from apps.admin_console import server
|
||
from artemis.runtime.device_pool import DeviceStatus
|
||
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": "task_paused", "started_at": time.time()})
|
||
monkeypatch.setattr(
|
||
service, "state", type("State", (), {"is_paused": False, "active_runs": {}})()
|
||
)
|
||
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
||
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async",
|
||
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
||
)
|
||
await service.tick_batches()
|
||
assert enqueue.await_count == 1
|
||
assert enqueue.call_args.args[0] == ["观察首页"]
|
||
assert [job["status"] for job in isolated.jobs(batch["id"])] == ["task_paused", "queued"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_device_outage_pauses_without_consuming_pending_case(isolated, monkeypatch):
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async", AsyncMock(return_value=[])
|
||
)
|
||
await service.tick_batches()
|
||
assert isolated.get("batch", batch["id"])["status"] == "paused"
|
||
assert all(job["status"] == "pending" for job in isolated.jobs(batch["id"]))
|
||
|
||
|
||
def test_stop_all_overrides_model_auto_resume(isolated, monkeypatch):
|
||
from apps.admin_console.services import task_queue_service as queue
|
||
|
||
batch = service.create_batch(payload())
|
||
isolated.put("batch", batch["id"], {**batch, "status": "paused", "pause_source": "model"})
|
||
monkeypatch.setattr("apps.admin_console.services.automation_store.store", isolated)
|
||
monkeypatch.setattr(queue, "state", Mock(queue_items=[], active_connections={}))
|
||
for method in (
|
||
"_terminate_all_device_owners",
|
||
"_kill_all_local_runs",
|
||
"_clear_pause_file",
|
||
"ensure_worker_running",
|
||
):
|
||
monkeypatch.setattr(queue.TaskQueueService, method, Mock())
|
||
assert queue.TaskQueueService._stop_all_tasks()
|
||
saved = isolated.get("batch", batch["id"])
|
||
assert saved["pause_source"] == "user"
|
||
assert "等待手动继续" in saved["reason"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
@pytest.mark.parametrize("manual_pause", [False, True])
|
||
@pytest.mark.parametrize("stop_on_failure", [False, True])
|
||
async def test_model_pause_recovery_preserves_user_pause_and_failure_policy(
|
||
isolated, monkeypatch, manual_pause, stop_on_failure
|
||
):
|
||
from apps.admin_console import server
|
||
from artemis.runtime.device_pool import DeviceStatus
|
||
|
||
batch = service.create_batch(payload(stop_on_failure=stop_on_failure))
|
||
service.batch_action(batch["id"], "resume")
|
||
first, second = isolated.jobs(batch["id"])
|
||
isolated.save_job({**first, "status": "running", "started_at": time.time()})
|
||
pause_state = type("State", (), {"is_paused": True, "active_runs": {}})()
|
||
monkeypatch.setattr(service, "state", pause_state)
|
||
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
||
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async",
|
||
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
||
)
|
||
monkeypatch.setattr(reports, "export_batch", Mock())
|
||
await service.tick_batches()
|
||
assert isolated.get("batch", batch["id"])["pause_source"] == "model"
|
||
if manual_pause:
|
||
service.batch_action(batch["id"], "pause")
|
||
# Another paused-model tick must not overwrite the explicit user pause.
|
||
await service.tick_batches()
|
||
assert isolated.get("batch", batch["id"])["pause_source"] == "user"
|
||
isolated.save_job({**first, "status": "failed"})
|
||
pause_state.is_paused = False
|
||
await service.tick_batches()
|
||
assert isolated.job(first["session_id"])["status"] == "failed"
|
||
expected = "pending" if manual_pause else "skipped" if stop_on_failure else "queued"
|
||
assert isolated.job(second["session_id"])["status"] == expected
|
||
assert enqueue.await_count == (0 if manual_pause or stop_on_failure else 1)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
@pytest.mark.parametrize("code", ["initialization_failed", "execution_failed"])
|
||
async def test_failure_reason_retained_and_only_initialization_blocks_remaining_batch(
|
||
isolated, monkeypatch, code
|
||
):
|
||
from apps.admin_console import server
|
||
from artemis.runtime import task_result
|
||
from artemis.runtime.device_pool import DeviceStatus
|
||
|
||
batch = service.create_batch(payload())
|
||
service.batch_action(batch["id"], "resume")
|
||
first, second = isolated.jobs(batch["id"])
|
||
isolated.save_job({**first, "status": "running"})
|
||
monkeypatch.setattr(task_result, "TRACES_PATH", reports.TRACES_PATH)
|
||
task_result.save_failure_result(
|
||
first["session_id"], "PermissionError: awake lock denied", code=code
|
||
)
|
||
monkeypatch.setattr(reports, "export_session", Mock(side_effect=OSError("export unavailable")))
|
||
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
||
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
||
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async",
|
||
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
||
)
|
||
|
||
await service.on_terminal({"session_id": first["session_id"]}, "failed")
|
||
await service.tick_batches()
|
||
saved = isolated.job(first["session_id"])
|
||
assert saved["error"] == "PermissionError: awake lock denied"
|
||
assert saved["report_error"] == "export unavailable"
|
||
if code == "initialization_failed":
|
||
assert isolated.get("batch", batch["id"])["pause_source"] == "initialization"
|
||
assert isolated.job(second["session_id"])["status"] == "pending"
|
||
enqueue.assert_not_awaited()
|
||
service.batch_action(batch["id"], "resume")
|
||
await service.tick_batches()
|
||
assert enqueue.await_count == 1
|
||
assert enqueue.call_args.kwargs["session_id"] == second["session_id"]
|
||
assert isolated.job(first["session_id"])["status"] == "failed"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fail_fast_includes_skipped_reports(isolated, monkeypatch):
|
||
batch = service.create_batch(payload(stop_on_failure=True))
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": "failed"})
|
||
service.batch_action(batch["id"], "resume")
|
||
await service.tick_batches()
|
||
assert [j["status"] for j in isolated.jobs(batch["id"])] == ["failed", "skipped"]
|
||
archive = reports.export_batch(batch["id"])
|
||
with zipfile.ZipFile(archive) as z:
|
||
assert len(z.namelist()) == 12 # summaries + reports, diagnostics and evidence per task
|
||
diagnostics = [name for name in z.namelist() if name.endswith("模型请求诊断.json")]
|
||
assert len(diagnostics) == 2
|
||
for name in diagnostics:
|
||
assert json.loads(z.read(name))["schema_version"] == 1
|
||
assert len([name for name in z.namelist() if name.endswith("证据索引.json")]) == 2
|
||
assert sum(n.endswith(".docx") for n in z.namelist()) == 3
|
||
|
||
|
||
def test_batch_archive_contains_each_task_media_when_present(isolated, monkeypatch, tmp_path):
|
||
batch = service.create_batch(payload())
|
||
first, second = isolated.jobs(batch["id"])
|
||
image1 = tmp_path / "first.png"
|
||
image2 = tmp_path / "second.png"
|
||
video = tmp_path / "recording.mp4"
|
||
image1.write_bytes(b"first screenshot")
|
||
image2.write_bytes(b"second screenshot")
|
||
video.write_bytes(b"saved recording")
|
||
images = {first["session_id"]: image1, second["session_id"]: image2}
|
||
monkeypatch.setattr(
|
||
reports,
|
||
"session_screenshots",
|
||
lambda report: [(images[report["id"]], "0001_步骤0001_操作前.png")],
|
||
)
|
||
monkeypatch.setattr(
|
||
reports,
|
||
"session_videos",
|
||
lambda report: (
|
||
[(video, "0001_recording.mp4")] if report["id"] == first["session_id"] else []
|
||
),
|
||
)
|
||
|
||
archive = reports.export_batch(batch["id"])
|
||
with zipfile.ZipFile(archive) as z:
|
||
names = z.namelist()
|
||
assert len([name for name in names if "/步骤截图/" in name]) == 2
|
||
assert len([name for name in names if "/任务录屏/" in name]) == 1
|
||
assert all(name.startswith("任务报告/") for name in names if "步骤截图" in name)
|
||
assert z.read(next(name for name in names if "/任务录屏/" in name)) == b"saved recording"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_report_failure_does_not_strand_job(isolated, monkeypatch):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
monkeypatch.setattr(
|
||
reports, "export_session", lambda *a: (_ for _ in ()).throw(OSError("disk full"))
|
||
)
|
||
await service.on_terminal({"session_id": job["session_id"]}, "completed")
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "completed"
|
||
assert "disk full" in saved["report_error"]
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_batch_timeout_stays_visible_in_job_report_and_history(client, isolated):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
isolated.save_job(
|
||
{**job, "status": "running", "timed_out": True, "error": "任务执行超时（30 秒）"}
|
||
)
|
||
await service.on_terminal({"session_id": job["session_id"]}, "cancelled")
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "failed"
|
||
assert "超时" in saved["error"]
|
||
assert "超时" in reports.session_report(job["session_id"])["error"]
|
||
history = client.get("/api/automation/history").json()["items"]
|
||
assert any(
|
||
item["session_id"] == job["session_id"] and "超时" in item["error"] for item in history
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
@pytest.mark.parametrize("stop_on_failure", [False, True])
|
||
async def test_timed_out_case_releases_next_batch_case(isolated, monkeypatch, stop_on_failure):
|
||
from apps.admin_console import server
|
||
from artemis.runtime.device_pool import DeviceStatus
|
||
|
||
batch = service.create_batch(payload(timeout_seconds=1, stop_on_failure=stop_on_failure))
|
||
service.batch_action(batch["id"], "resume")
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": "running", "started_at": time.time() - 2})
|
||
monkeypatch.setattr(
|
||
service, "state", type("State", (), {"is_paused": False, "active_runs": {}})()
|
||
)
|
||
stop = Mock(return_value=True)
|
||
enqueue = AsyncMock(return_value={"tasks": [{"device_serial": "test-device"}]})
|
||
monkeypatch.setattr(server.task_queue_service, "stop_tasks", stop)
|
||
monkeypatch.setattr(server.task_queue_service, "enqueue_tasks", enqueue)
|
||
monkeypatch.setattr(
|
||
"artemis.runtime.device_pool.try_list_devices_async",
|
||
AsyncMock(return_value=[DeviceStatus(serial="test-device", state="device")]),
|
||
)
|
||
monkeypatch.setattr(reports, "export_batch", Mock())
|
||
|
||
await service.tick_batches()
|
||
assert isolated.get("batch", batch["id"])["status"] == "running"
|
||
assert "超时" in isolated.job(first["session_id"])["error"]
|
||
stop.assert_called_once_with(session_id=first["session_id"])
|
||
await service.tick_batches()
|
||
stop.assert_called_once()
|
||
|
||
await service.on_terminal({"session_id": first["session_id"]}, "cancelled")
|
||
await service.tick_batches()
|
||
statuses = [job["status"] for job in isolated.jobs(batch["id"])]
|
||
assert statuses == (["failed", "skipped"] if stop_on_failure else ["failed", "queued"])
|
||
assert enqueue.await_count == (0 if stop_on_failure else 1)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_queued_case_does_not_consume_execution_timeout(isolated, monkeypatch):
|
||
from apps.admin_console import server
|
||
|
||
batch = service.create_batch(payload(timeout_seconds=1))
|
||
service.batch_action(batch["id"], "resume")
|
||
first = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**first, "status": "queued", "started_at": time.time() - 30})
|
||
monkeypatch.setattr(service, "state", type("State", (), {"is_paused": False})())
|
||
stop = Mock()
|
||
monkeypatch.setattr(server.task_queue_service, "stop_tasks", stop)
|
||
|
||
await service.tick_batches()
|
||
stop.assert_not_called()
|
||
assert isolated.job(first["session_id"])["status"] == "queued"
|
||
assert isolated.get("batch", batch["id"])["status"] == "running"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_model_timeout_from_runner_log_reaches_failed_task_and_report(isolated):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
folder = reports.TRACES_PATH / job["session_id"]
|
||
folder.mkdir(parents=True)
|
||
(folder / "stderr.log").write_text(
|
||
"TimeoutError: model request timed out after 300 seconds\n", encoding="utf-8"
|
||
)
|
||
|
||
await service.on_terminal({"session_id": job["session_id"]}, "failed")
|
||
|
||
saved = isolated.job(job["session_id"])
|
||
assert saved["status"] == "failed"
|
||
assert "超时" in saved["error"]
|
||
assert reports.session_report(job["session_id"])["status"] == "failed"
|
||
assert "超时" in reports.session_report(job["session_id"])["error"]
|
||
|
||
|
||
def test_optional_timeout_warnings_are_not_reported_as_task_failure(tmp_path):
|
||
(tmp_path / "stdout.log").write_text(
|
||
"⚠ Could not clean unowned screen wake locks on emulator-5554: "
|
||
"Timed out acquiring the Artemis awake-lease mutex.\n"
|
||
"⚠ VisualStepSummarizer: Error generating summary for step 3: \n"
|
||
"ℹ StepMemoryService: Retrying Step 3 (attempt 4) after 1.0s\n",
|
||
encoding="utf-8",
|
||
)
|
||
assert reports.failure_reason(tmp_path, "failed", None) == (
|
||
"任务未完成，运行日志中没有可提取的具体异常；请查看执行步骤。"
|
||
)
|
||
|
||
(tmp_path / "stderr.log").write_text(
|
||
"TimeoutError: model request timed out after 300 seconds\n", encoding="utf-8"
|
||
)
|
||
reason = reports.failure_reason(tmp_path, "failed", None)
|
||
assert "model request timed out" in reason
|
||
assert "awake-lease" not in reason
|
||
|
||
(tmp_path / "stderr.log").write_text(
|
||
"DeviceBusyError: device queue reservation was cancelled\n", encoding="utf-8"
|
||
)
|
||
assert reports.failure_reason(tmp_path, "failed", None) == (
|
||
"DeviceBusyError: device queue reservation was cancelled"
|
||
)
|
||
|
||
|
||
def test_reports_do_not_execute_formulas_or_invent_pass(isolated):
|
||
batch = service.create_batch(payload())
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job(
|
||
{**job, "case": {"name": "=1+1", "goal": '=HYPERLINK("x")'}, "status": "completed"}
|
||
)
|
||
report = reports.session_report(job["session_id"])
|
||
assert report["verdict"].startswith("未验证")
|
||
assert reports.verdict({"tests": {"passed": 2, "failed": 1}}) == "验证失败"
|
||
assert reports.verdict({"tests": {"passed": 2, "unchecked": 1}}) == "验证不完整"
|
||
paths = reports.export_session(job["session_id"])
|
||
wb = load_workbook(next(p for p in paths if p.suffix == ".xlsx"))
|
||
assert wb["任务结果"]["B3"].data_type == "s"
|
||
with zipfile.ZipFile(next(p for p in paths if p.suffix == ".docx")) as doc:
|
||
assert "未验证".encode() in doc.read("word/document.xml")
|
||
|
||
|
||
@pytest.mark.parametrize("result_status", ["passed", "failed", "blocked", "inconclusive"])
|
||
def test_report_without_output_body_uses_effective_verdict_and_evidence(isolated, result_status):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
isolated.save_job({**job, "status": "completed"})
|
||
folder = reports.TRACES_PATH / job["session_id"]
|
||
folder.mkdir(parents=True)
|
||
(folder / "run_outcome.json").write_text(
|
||
json.dumps(
|
||
{
|
||
"task_status": "completed",
|
||
"test_result": {
|
||
"status": result_status,
|
||
"reason": "实际验收依据",
|
||
"evidence": ["记录中的证据"],
|
||
},
|
||
}
|
||
),
|
||
encoding="utf-8",
|
||
)
|
||
report = reports.session_report(job["session_id"])
|
||
assert report["verdict"] in report["content"]
|
||
assert "实际验收依据" in report["content"]
|
||
assert "记录中的证据" in report["content"]
|
||
assert "未保存独立输出正文" in report["content"]
|
||
assert "没有保存最终结论" not in report["content"]
|
||
assert report["test_result"]["status"] == result_status
|
||
|
||
|
||
def test_report_without_output_body_keeps_human_review_authoritative(isolated):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
isolated.save_job({**job, "status": "completed"})
|
||
folder = reports.TRACES_PATH / job["session_id"]
|
||
folder.mkdir(parents=True)
|
||
(folder / "run_outcome.json").write_text(
|
||
json.dumps(
|
||
{
|
||
"test_result": {"status": "passed", "reason": "AI 原判通过"},
|
||
}
|
||
),
|
||
encoding="utf-8",
|
||
)
|
||
isolated.save_case_review(job["session_id"], "failed", "用户确认实际结果不符合预期")
|
||
report = reports.session_report(job["session_id"])
|
||
assert "人工确认" in report["content"]
|
||
assert "用户确认实际结果不符合预期" in report["content"]
|
||
assert "AI 原判通过" not in report["content"]
|
||
assert report["test_result"]["status"] == "failed"
|
||
|
||
|
||
def test_report_projects_stored_android_receipts_without_rewriting_history(isolated):
|
||
from tests.unit.data_engine.test_evidence_sources import android_receipt
|
||
|
||
sid = isolated.jobs(service.create_batch(payload())["id"])[0]["session_id"]
|
||
receipt = android_receipt()["payload"]["result"]
|
||
raw_payload = json.dumps({"args": {"action": "screenshot"}, "result": json.dumps(receipt)})
|
||
with sqlite3.connect(isolated.path) as conn:
|
||
conn.execute(
|
||
"INSERT INTO sessions(session_id,initial_goal,start_time,status) VALUES (?,?,?,?)",
|
||
(sid, "跨端截图", 1, "completed"),
|
||
)
|
||
conn.execute(
|
||
"INSERT INTO steps(step_id,session_id,step_number) VALUES (?,?,?)", ("s", sid, 1)
|
||
)
|
||
conn.execute(
|
||
"INSERT INTO traces(trace_id,session_id,step_id,type,name,timestamp,status,payload) VALUES (?,?,?,?,?,?,?,?)",
|
||
("capture", sid, "s", "tool", "android_file", 2, "success", raw_payload),
|
||
)
|
||
report = reports.session_report(sid)
|
||
source = report["evidence_sources"]["sources"][0]
|
||
assert source["kind"] == "android"
|
||
assert source["references"][0]["trace_id"] == "capture"
|
||
assert report["evidence_sources"]["unattributed_step_count"] == 0
|
||
with sqlite3.connect(isolated.path) as conn:
|
||
assert (
|
||
conn.execute("SELECT payload FROM traces WHERE trace_id='capture'").fetchone()[0]
|
||
== raw_payload
|
||
)
|
||
|
||
|
||
def test_failed_report_exposes_logged_reason_and_formats_chinese_markdown(isolated):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
isolated.save_job({**job, "status": "failed"})
|
||
folder = reports.TRACES_PATH / job["session_id"]
|
||
(folder / "notes").mkdir(parents=True)
|
||
(folder / "stdout.log").write_text(
|
||
"❌ [run] Error running automation: 模型连续 3 次未提供可执行的工具调用\n",
|
||
encoding="utf-8",
|
||
)
|
||
(folder / "notes" / "output.md").write_text(
|
||
"## 结果\n- **网络**正常\n\n| 分类 | 状态 |\n| --- | --- |\n| 系统 | 未确认 |",
|
||
encoding="utf-8",
|
||
)
|
||
report = reports.session_report(job["session_id"])
|
||
assert "连续 3 次" in report["error"]
|
||
assert report["verdict"] == "测试阻塞（AI确认）"
|
||
assert report["test_result"]["status"] == "blocked"
|
||
assert "连续 3 次" in report["test_result"]["reason"]
|
||
assert "abc123secret" not in reports.failure_reason(
|
||
folder, "failed", "认证失败 OPENAI_API_KEY=abc123secret Authorization: Bearer abc123secret"
|
||
)
|
||
assert "超时" not in reports.failure_reason(folder, "failed", "LLMExhaustedError: 额度不足")
|
||
paths = reports.export_session(job["session_id"])
|
||
doc = Document(next(path for path in paths if path.suffix == ".docx"))
|
||
assert doc.styles["Normal"].font.name == "Microsoft YaHei"
|
||
assert any("异常原因" in paragraph.text for paragraph in doc.paragraphs)
|
||
assert any(
|
||
run.bold and "网络" in run.text for paragraph in doc.paragraphs for run in paragraph.runs
|
||
)
|
||
assert any(
|
||
cell.text == "分类" for table in doc.tables for row in table.rows for cell in row.cells
|
||
)
|
||
wb = load_workbook(next(path for path in paths if path.suffix == ".xlsx"))
|
||
assert any(
|
||
row[0].value == "异常原因" and "连续 3 次" in row[1].value
|
||
for row in wb["任务结果"].iter_rows()
|
||
)
|
||
assert wb["结论"]["B2"].value == "结果"
|
||
assert wb["结果表1"]["A2"].value == "系统"
|
||
assert wb["执行证据"]["A1"].font.name == "微软雅黑"
|
||
assert wb["执行证据"].page_setup.orientation == "landscape"
|
||
assert reports.execution_result_label({"status": "dispatched"}) == "动作已发出（需核对截图）"
|
||
assert reports.conclusion_rows("## 结果\n- **通过**检查\n```txt\n| 不是 | 表格 |\n```") == [
|
||
("类型", "结论与发现"),
|
||
("小节", "结果"),
|
||
("要点", "通过检查"),
|
||
("代码", "| 不是 | 表格 |"),
|
||
]
|
||
|
||
|
||
def test_prompt_based_verdict_is_consistent_in_history_batch_and_reports(
|
||
client, isolated, monkeypatch
|
||
):
|
||
from artemis.runtime import case_result
|
||
|
||
monkeypatch.setattr(case_result, "TRACES_PATH", reports.TRACES_PATH)
|
||
batch = service.create_batch(payload())
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job(
|
||
{**job, "status": "completed", "started_at": time.time() - 50, "ended_at": time.time()}
|
||
)
|
||
result = {
|
||
"status": "failed",
|
||
"reason": "预期出现成功提示，实际显示错误",
|
||
"evidence": ["步骤 3 点击后截图显示错误提示"],
|
||
}
|
||
folder = reports.TRACES_PATH / job["session_id"]
|
||
folder.mkdir(parents=True)
|
||
(folder / "run_outcome.json").write_text(
|
||
json.dumps(
|
||
{"task_status": "completed", "tests": {}, "test_result": result}, ensure_ascii=False
|
||
),
|
||
encoding="utf-8",
|
||
)
|
||
history = client.get("/api/automation/history").json()["items"][0]
|
||
batch_job = client.get(f"/api/automation/batches/{batch['id']}").json()["jobs"][0]
|
||
report = client.get(f"/api/automation/reports/{job['session_id']}").json()
|
||
for item in (history, batch_job, report):
|
||
assert item["status"] == "completed"
|
||
assert item["test_result"] == {**result, "source": "ai"}
|
||
assert report["verdict"] == "测试不通过（AI确认）"
|
||
paths = reports.export_session(job["session_id"])
|
||
doc = Document(next(path for path in paths if path.suffix == ".docx"))
|
||
doc_text = "\n".join(
|
||
cell.text for table in doc.tables for row in table.rows for cell in row.cells
|
||
)
|
||
doc_text += "\n" + "\n".join(paragraph.text for paragraph in doc.paragraphs)
|
||
assert result["reason"] in doc_text
|
||
assert result["evidence"][0] in doc_text
|
||
wb = load_workbook(next(path for path in paths if path.suffix == ".xlsx"))
|
||
values = [cell.value for row in wb["任务结果"].iter_rows() for cell in row]
|
||
assert "测试不通过（AI确认）" in values
|
||
assert result["reason"] in values
|
||
|
||
|
||
def test_human_review_overrides_every_report_preserves_ai_and_invalidates_batch_cache(
|
||
client, isolated, monkeypatch
|
||
):
|
||
from artemis.runtime import case_result
|
||
from apps.admin_console.routers import sessions as sessions_api
|
||
|
||
monkeypatch.setattr(case_result, "TRACES_PATH", reports.TRACES_PATH)
|
||
batch = service.create_batch(payload())
|
||
jobs = isolated.jobs(batch["id"])
|
||
for job in jobs:
|
||
isolated.save_job({**job, "status": "completed", "ended_at": time.time()})
|
||
session_id = jobs[0]["session_id"]
|
||
ai = {"status": "failed", "reason": "AI识别到预期不匹配", "evidence": ["步骤 3 截图"]}
|
||
directory = reports.TRACES_PATH / session_id
|
||
directory.mkdir(parents=True)
|
||
outcome_file = directory / "run_outcome.json"
|
||
original = json.dumps({"task_status": "completed", "test_result": ai}, ensure_ascii=False)
|
||
outcome_file.write_text(original, encoding="utf-8")
|
||
archive = reports.export_batch(batch["id"])
|
||
isolated.put("batch", batch["id"], {**batch, "report_ready": True})
|
||
|
||
response = client.put(
|
||
f"/api/automation/reports/{session_id}/review",
|
||
json={"status": "passed", "reason": "已人工核对，符合预期"},
|
||
)
|
||
assert response.status_code == 200
|
||
assert response.json()["status"] == "completed"
|
||
assert response.json()["verdict"] == "测试通过（人工确认）"
|
||
assert outcome_file.read_text(encoding="utf-8") == original
|
||
assert not archive.exists()
|
||
assert not isolated.get("batch", batch["id"])["report_ready"]
|
||
assert len(isolated.get("case_review", session_id)["reviews"]) == 1
|
||
for result in (
|
||
response.json(),
|
||
client.get("/api/automation/history", params={"q": session_id}).json()["items"][0],
|
||
client.get(f"/api/automation/batches/{batch['id']}").json()["jobs"][0],
|
||
):
|
||
assert result["test_result"]["source"] == "human"
|
||
assert result["test_result"]["status"] == "passed"
|
||
assert result["test_result"]["ai_result"] == {**ai, "source": "ai"}
|
||
assert result["test_result"]["reviewed_at"] > 0
|
||
|
||
monkeypatch.setattr(sessions_api, "store", isolated)
|
||
monkeypatch.setattr(sessions_api, "TRACES_PATH", reports.TRACES_PATH)
|
||
monkeypatch.setattr(
|
||
sessions_api.session_repo,
|
||
"get_session_by_id",
|
||
lambda sid: {"session_id": sid, "status": "completed"},
|
||
)
|
||
client.app.include_router(sessions_api.router)
|
||
detail = client.get(f"/api/sessions/{session_id}").json()
|
||
assert detail["test_result"]["source"] == "human"
|
||
assert detail["batch_id"] == batch["id"]
|
||
|
||
downloaded = client.get(f"/api/automation/batches/{batch['id']}/archive")
|
||
assert downloaded.status_code == 200
|
||
with zipfile.ZipFile(BytesIO(downloaded.content)) as bundle:
|
||
wb = load_workbook(BytesIO(bundle.read("批次结果.xlsx")))
|
||
assert wb.active["D2"].value == "测试通过（人工确认）"
|
||
assert "AI确认" in wb.active["D3"].value
|
||
doc = Document(BytesIO(bundle.read("批次汇总.docx")))
|
||
assert doc.tables[0].rows[1].cells[3].text == "测试通过（人工确认）"
|
||
|
||
changed = client.put(f"/api/automation/reports/{session_id}/review", json={"status": "failed"})
|
||
assert changed.status_code == 200
|
||
assert changed.json()["verdict"] == "测试不通过（人工确认）"
|
||
assert len(isolated.get("case_review", session_id)["reviews"]) == 2
|
||
assert outcome_file.read_text(encoding="utf-8") == original
|
||
|
||
|
||
def test_skipped_flash_review_is_consistent_in_history_reports_and_human_review(
|
||
client, isolated, monkeypatch
|
||
):
|
||
from artemis.runtime import case_result
|
||
|
||
monkeypatch.setattr(case_result, "TRACES_PATH", reports.TRACES_PATH)
|
||
batch = service.create_batch(payload())
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**job, "status": "completed"})
|
||
folder = reports.TRACES_PATH / job["session_id"]
|
||
folder.mkdir(parents=True)
|
||
result = {
|
||
"status": "inconclusive",
|
||
"reason": "执行完成 · 未做 AI 判定",
|
||
"evidence": [],
|
||
"verification_skipped": True,
|
||
}
|
||
(folder / "run_outcome.json").write_text(
|
||
json.dumps({"task_status": "completed", "tests": {}, "test_result": result}),
|
||
encoding="utf-8",
|
||
)
|
||
for item in (
|
||
client.get("/api/automation/history").json()["items"][0],
|
||
client.get(f"/api/automation/batches/{batch['id']}").json()["jobs"][0],
|
||
client.get(f"/api/automation/reports/{job['session_id']}").json(),
|
||
):
|
||
assert item["status"] == "completed"
|
||
assert item["test_result"] == {**result, "source": "system"}
|
||
assert reports.session_report(job["session_id"])["verdict"] == "未做 AI 判定"
|
||
paths = reports.export_session(job["session_id"])
|
||
workbook = load_workbook(next(path for path in paths if path.suffix == ".xlsx"))
|
||
assert "未做 AI 判定" in [
|
||
cell.value for row in workbook["任务结果"].iter_rows() for cell in row
|
||
]
|
||
doc = Document(next(path for path in paths if path.suffix == ".docx"))
|
||
assert "未做 AI 判定" in "\n".join(paragraph.text for paragraph in doc.paragraphs)
|
||
reviewed = client.put(
|
||
f"/api/automation/reports/{job['session_id']}/review",
|
||
json={"status": "passed", "reason": "已人工核对"},
|
||
).json()
|
||
assert reviewed["test_result"]["source"] == "human"
|
||
assert reviewed["test_result"]["ai_result"] == {**result, "source": "system"}
|
||
assert reviewed["verdict"] == "测试通过（人工确认）"
|
||
|
||
|
||
def test_review_rejects_active_missing_invalid_and_packing_tasks(client, isolated, monkeypatch):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
url = f"/api/automation/reports/{job['session_id']}/review"
|
||
assert client.put(url, json={"status": "passed"}).status_code == 409
|
||
assert isolated.get("case_review", job["session_id"]) is None
|
||
assert client.put(url, json={"status": "inconclusive"}).status_code == 422
|
||
assert client.put(url, json={"status": "passed", "source": "human"}).status_code == 422
|
||
assert client.put(url, json={"status": "passed", "reason": "字" * 2001}).status_code == 422
|
||
assert (
|
||
client.put("/api/automation/reports/missing/review", json={"status": "passed"}).status_code
|
||
== 404
|
||
)
|
||
isolated.save_job({**job, "status": "completed"})
|
||
monkeypatch.setattr(api.archive_service, "status", lambda *args: {"status": "preparing"})
|
||
assert client.put(url, json={"status": "passed"}).status_code == 409
|
||
assert isolated.get("case_review", job["session_id"]) is None
|
||
|
||
|
||
def test_batch_filter_searches_full_history_and_reviews_are_removed_with_batch(client, isolated):
|
||
first, second = [service.create_batch(payload(name=name)) for name in ("第一批", "第二批")]
|
||
for batch in (first, second):
|
||
for job in isolated.jobs(batch["id"]):
|
||
isolated.save_job({**job, "status": "completed"})
|
||
with isolated.connection() as conn:
|
||
conn.execute(
|
||
"INSERT INTO sessions(session_id,initial_goal,start_time,status) VALUES (?,?,?,?)",
|
||
("single-case", "单独检查", time.time(), "completed"),
|
||
)
|
||
conn.commit()
|
||
filtered = client.get(
|
||
"/api/automation/history",
|
||
params={"batch_id": first["id"], "q": "用例", "size": 1, "page": 2},
|
||
).json()
|
||
assert filtered["total"] == 2
|
||
assert len(filtered["items"]) == 1
|
||
assert filtered["items"][0]["batch_id"] == first["id"]
|
||
assert (
|
||
client.get("/api/automation/history", params={"batch_id": "__batch__"}).json()["total"] == 4
|
||
)
|
||
singles = client.get("/api/automation/history", params={"batch_id": "__single__"}).json()
|
||
assert [item["session_id"] for item in singles["items"]] == ["single-case"]
|
||
assert client.get("/api/automation/history", params={"batch_id": "bad/path"}).status_code == 400
|
||
reviewed_id = isolated.jobs(first["id"])[0]["session_id"]
|
||
isolated.save_case_review(reviewed_id, "passed", "人工核对")
|
||
isolated.begin_delete_batch(first["id"])
|
||
isolated.delete_batch(first["id"])
|
||
assert isolated.get("case_review", reviewed_id) is None
|
||
assert len(isolated.jobs(second["id"])) == 2
|
||
isolated.save_case_review("single-case", "failed", "不符合预期")
|
||
isolated.delete_history("single-case")
|
||
assert isolated.get("case_review", "single-case") is None
|
||
|
||
|
||
def test_task_material_zip_contains_ordered_evidence_and_reports(
|
||
client, isolated, tmp_path, monkeypatch
|
||
):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
isolated.save_job({**job, "case": {**job["case"], "case_id": "1.1"}})
|
||
artifact_dir = reports.TRACES_PATH / job["session_id"] / "artifacts" / "android"
|
||
artifact_dir.mkdir(parents=True)
|
||
(artifact_dir / "安卓截图.png").write_bytes(b"native png")
|
||
(artifact_dir / ".artemis-partial.tmp").write_bytes(b"incomplete")
|
||
evidence = tmp_path / "evidence"
|
||
evidence.mkdir()
|
||
first = evidence / "first.png"
|
||
second = evidence / "second.png"
|
||
video = evidence / "recording.mp4"
|
||
for path in (first, second, video):
|
||
path.write_bytes(path.name.encode())
|
||
monkeypatch.setattr(
|
||
reports,
|
||
"session_screenshots",
|
||
lambda report: [(first, "0001_步骤0001_操作前.png"), (second, "0002_步骤0002_操作后.png")],
|
||
)
|
||
monkeypatch.setattr(reports, "session_videos", lambda report: [(video, "0001_recording.mp4")])
|
||
|
||
response = client.get(f"/api/automation/reports/{job['session_id']}/all")
|
||
assert response.status_code == 200
|
||
with zipfile.ZipFile(BytesIO(response.content)) as archive:
|
||
files = archive.namelist()
|
||
assert all(name.startswith("1.1/") for name in files)
|
||
assert any(name.endswith(".docx") for name in files)
|
||
assert any(name.endswith(".xlsx") for name in files)
|
||
assert files.index("1.1/步骤截图/0001_步骤0001_操作前.png") < files.index(
|
||
"1.1/步骤截图/0002_步骤0002_操作后.png"
|
||
)
|
||
assert archive.read("1.1/任务录屏/安卓/0001_recording.mp4") == b"recording.mp4"
|
||
manifest = json.loads(archive.read("1.1/文件清单.json"))
|
||
assert manifest["session_id"] == job["session_id"]
|
||
assert manifest["artifacts"] == ["安卓截图.png"]
|
||
assert archive.read("1.1/任务文件/安卓/安卓截图.png") == b"native png"
|
||
assert not any("partial" in name for name in files)
|
||
assert "1.1-all.zip" in response.headers["content-disposition"]
|
||
|
||
|
||
def test_batch_zip_contains_each_tasks_android_artifacts(isolated):
|
||
batch = service.create_batch(payload())
|
||
for index, job in enumerate(isolated.jobs(batch["id"])):
|
||
root = reports.TRACES_PATH / job["session_id"] / "artifacts" / "android"
|
||
root.mkdir(parents=True)
|
||
(root / "结果.txt").write_text(f"任务{index}", encoding="utf-8")
|
||
with zipfile.ZipFile(reports.export_batch(batch["id"])) as archive:
|
||
names = [name for name in archive.namelist() if name.endswith("任务文件/安卓/结果.txt")]
|
||
assert len(names) == 2
|
||
assert {archive.read(name).decode() for name in names} == {"任务0", "任务1"}
|
||
|
||
|
||
@pytest.mark.parametrize("pause_source", ["device", "initialization", "user", "model"])
|
||
def test_restart_retains_original_pause_reason_and_archives_recovery(isolated, pause_source):
|
||
batch = service.create_batch(payload())
|
||
reason = "原始暂停原因"
|
||
isolated.put(
|
||
"batch",
|
||
batch["id"],
|
||
{**batch, "status": "paused", "pause_source": pause_source, "reason": reason},
|
||
)
|
||
service.recover_batches()
|
||
recovered = isolated.get("batch", batch["id"])
|
||
assert recovered["status"] == "paused"
|
||
assert recovered["pause_source"] == "recovery"
|
||
assert recovered["reason"] == reason
|
||
event = recovered["recovery_events"][0]
|
||
assert event["previous_reason"] == reason
|
||
assert event["previous_pause_source"] == pause_source
|
||
assert "服务已重启" in event["message"]
|
||
assert all(job["status"] == "pending" for job in isolated.jobs(batch["id"]))
|
||
service.recover_batches()
|
||
assert isolated.get("batch", batch["id"])["reason"] == reason
|
||
|
||
|
||
def test_runtime_evidence_is_exported_from_the_task_without_resampling(isolated, monkeypatch):
|
||
batch = service.create_batch(payload())
|
||
first, second = isolated.jobs(batch["id"])
|
||
snapshot = {
|
||
"schema_version": 1,
|
||
"failure": {
|
||
"captured_at": 1,
|
||
"runtime": {"source_commit": "worker-commit"},
|
||
"memory": {"available_bytes": 163},
|
||
},
|
||
}
|
||
directory = reports.TRACES_PATH / first["session_id"]
|
||
directory.mkdir(parents=True)
|
||
(directory / "runtime_diagnostics.json").write_text(json.dumps(snapshot), encoding="utf-8")
|
||
isolated.put(
|
||
"batch",
|
||
batch["id"],
|
||
{**batch, "status": "paused", "reason": "设备离线", "pause_source": "device"},
|
||
)
|
||
service.recover_batches()
|
||
|
||
def assert_snapshots(archive_path):
|
||
with zipfile.ZipFile(archive_path) as archive:
|
||
records = [
|
||
json.loads(archive.read(name))
|
||
for name in archive.namelist()
|
||
if name.endswith("运行环境诊断.json")
|
||
]
|
||
recorded = next(record for record in records if record["runtime"])
|
||
assert recorded["runtime"] == snapshot
|
||
assert recorded["batch"]["reason"] == "设备离线"
|
||
assert recorded["batch"]["recovery_events"][0]["previous_pause_source"] == "device"
|
||
return records
|
||
|
||
assert_snapshots(reports.export_session_media(first["session_id"], "all"))
|
||
records = assert_snapshots(reports.export_batch(batch["id"]))
|
||
assert len(records) == 2
|
||
assert next(record for record in records if record["runtime"] is None)["runtime"] is None
|
||
|
||
|
||
def test_all_material_cache_reuses_reports_and_invalidates_on_review_or_new_files(
|
||
isolated, monkeypatch
|
||
):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
job["status"] = "completed"
|
||
isolated.save_job(job)
|
||
word = Mock(wraps=reports.word_report)
|
||
monkeypatch.setattr(reports, "word_report", word)
|
||
monkeypatch.setattr(
|
||
reports,
|
||
"diagnostics_json",
|
||
lambda sid: json.dumps({"requests": [], "collected_at": time.time()}),
|
||
)
|
||
first = reports.export_session_media(job["session_id"], "all")
|
||
first_bytes = first.read_bytes()
|
||
reports.export_session_media(job["session_id"], "all")
|
||
assert word.call_count == 1
|
||
assert first.read_bytes() == first_bytes
|
||
isolated.save_case_review(job["session_id"], "passed", "人工核对")
|
||
reports.export_session_media(job["session_id"], "all")
|
||
assert word.call_count == 2
|
||
folder = reports.TRACES_PATH / job["session_id"] / "artifacts" / "android"
|
||
folder.mkdir(parents=True)
|
||
(folder / "新资料.txt").write_text("new evidence", encoding="utf-8")
|
||
reports.export_session_media(job["session_id"], "all")
|
||
with zipfile.ZipFile(first) as archive:
|
||
assert any(name.endswith("任务文件/安卓/新资料.txt") for name in archive.namelist())
|
||
assert not json.loads(
|
||
archive.read(next(n for n in archive.namelist() if n.endswith("文件清单.json")))
|
||
)["snapshot"]
|
||
# File-only changes rebuild the ZIP while keeping unchanged documents.
|
||
assert word.call_count == 2
|
||
|
||
|
||
def test_different_tasks_export_concurrently_and_running_exports_are_snapshots(
|
||
isolated, monkeypatch
|
||
):
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
|
||
jobs = isolated.jobs(service.create_batch(payload())["id"])
|
||
barrier = threading.Barrier(2)
|
||
|
||
def generate(report, path):
|
||
barrier.wait(timeout=5)
|
||
path.write_bytes(b"generated report")
|
||
|
||
monkeypatch.setattr(reports, "word_report", generate)
|
||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||
exports = [pool.submit(reports.export_session, j["session_id"], ["docx"]) for j in jobs]
|
||
assert all(result.result(timeout=10)[0].is_file() for result in exports)
|
||
count = Mock(side_effect=lambda report, path: path.write_bytes(b"snapshot"))
|
||
monkeypatch.setattr(reports, "word_report", count)
|
||
for _ in range(2):
|
||
reports.export_session(jobs[0]["session_id"], ["docx"])
|
||
assert count.call_count == 2
|
||
|
||
|
||
def test_android_artifact_export_does_not_follow_other_session_links(isolated):
|
||
root = reports.TRACES_PATH / "session" / "artifacts"
|
||
root.mkdir(parents=True)
|
||
other = reports.TRACES_PATH / "other-session"
|
||
other.mkdir()
|
||
(other / "private.txt").write_text("other task")
|
||
try:
|
||
(root / "android").symlink_to(other, target_is_directory=True)
|
||
except OSError:
|
||
pytest.skip("symlinks unavailable")
|
||
assert reports.session_artifacts({"id": "session"}) == []
|
||
|
||
|
||
def test_report_source_inventory_preserves_human_verdict_and_raw_evidence(
|
||
isolated, tmp_path, monkeypatch
|
||
):
|
||
from apps.admin_console.services import evidence_service, task_inspection
|
||
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
sid = job["session_id"]
|
||
action = {
|
||
"action": "plug_read",
|
||
"plugin_id": "data-reader",
|
||
"call_id": "recorded-call",
|
||
"read_only": True,
|
||
"source": {"kind": "api", "target_id": "business-service", "label": "安卓业务后台"},
|
||
}
|
||
with sqlite3.connect(isolated.path) as conn:
|
||
conn.execute(
|
||
"INSERT INTO sessions(session_id,initial_goal,start_time,end_time,status) VALUES (?,?,?,?,?)",
|
||
(sid, "检查业务状态", 1700000010, 1700000120, "completed"),
|
||
)
|
||
conn.execute(
|
||
"INSERT INTO steps(step_id,session_id,step_number,action_taken) VALUES (?,?,?,?)",
|
||
("actual-step", sid, 7, json.dumps(action)),
|
||
)
|
||
isolated.save_case_review(sid, "failed", "用户指出业务状态仍未落库")
|
||
monkeypatch.setattr(evidence_service, "TRACES_PATH", reports.TRACES_PATH)
|
||
report = reports.session_report(sid)
|
||
source = report["evidence_sources"]["sources"][0]
|
||
assert source["category"] == "auxiliary"
|
||
assert source["references"][0]["step_id"] == "actual-step"
|
||
assert report["test_result"]["status"] == "failed"
|
||
assert report["test_result"]["source"] == "human"
|
||
assert report["steps"][0]["action_taken"] == action
|
||
index = evidence_service.export_index(sid, report, [])
|
||
assert index["evidence_sources"] == report["evidence_sources"]
|
||
monkeypatch.setattr(task_inspection, "store", isolated)
|
||
monkeypatch.setattr(task_inspection, "session_repo", reports.SessionRepository())
|
||
assert (
|
||
task_inspection.inspect_task(sid, "report")["evidence_sources"]
|
||
== report["evidence_sources"]
|
||
)
|
||
from docx import Document
|
||
from openpyxl import load_workbook
|
||
|
||
word = tmp_path / "sources.docx"
|
||
sheet = tmp_path / "sources.xlsx"
|
||
reports.word_report(report, word)
|
||
reports.excel_report(report, sheet)
|
||
document = Document(word)
|
||
word_text = " ".join(
|
||
[p.text for p in document.paragraphs]
|
||
+ [cell.text for table in document.tables for row in table.rows for cell in row.cells]
|
||
)
|
||
workbook = load_workbook(sheet)
|
||
try:
|
||
excel_text = " ".join(
|
||
str(cell.value or "") for ws in workbook for row in ws for cell in row
|
||
)
|
||
finally:
|
||
workbook.close()
|
||
for text in (word_text, excel_text):
|
||
assert "接口数据" in text
|
||
assert "安卓业务后台" in text
|
||
assert "actual-step" in text
|
||
assert "用户指出业务状态仍未落库" in text
|
||
|
||
|
||
def test_reports_share_lifecycle_and_distinguish_guidance_receipt_from_consumption(
|
||
isolated, tmp_path, monkeypatch
|
||
):
|
||
from apps.admin_console.services import task_lifecycle
|
||
from artemis.runtime import task_control
|
||
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
job.update(
|
||
status="completed",
|
||
created_at=1700000010,
|
||
worker_started_at=1700000020,
|
||
ended_at=1700000120,
|
||
finalized_at=1700000125,
|
||
)
|
||
isolated.save_job(job)
|
||
monkeypatch.setattr(task_control, "TRACES_PATH", reports.TRACES_PATH)
|
||
task_control.add_instruction(job["session_id"], "保留证据", instruction_id="unused-guidance")
|
||
report = reports.session_report(job["session_id"])
|
||
assert report["lifecycle"] == task_lifecycle.task_lifecycle(
|
||
job["session_id"], db_path=isolated.path
|
||
)
|
||
assert report["lifecycle"]["durations"]["total"] == 115
|
||
assert report["lifecycle"]["durations"]["execution"] is None
|
||
assert report["instructions"][0]["status"] == "unapplied"
|
||
word = tmp_path / "lifecycle.docx"
|
||
excel = tmp_path / "lifecycle.xlsx"
|
||
reports.word_report(report, word)
|
||
reports.excel_report(report, excel)
|
||
document = Document(word)
|
||
word_text = "\n".join(p.text for p in document.paragraphs) + "\n".join(
|
||
cell.text for table in document.tables for row in table.rows for cell in row.cells
|
||
)
|
||
book = load_workbook(excel)
|
||
excel_text = "\n".join(str(cell.value) for sheet in book for row in sheet for cell in row)
|
||
for content in (word_text, excel_text):
|
||
assert "后台排队：10.0 秒" in content
|
||
assert "执行与整理：未记录" in content
|
||
assert "unused-guidance · 任务结束，未记录消费 · 接收" in content
|
||
|
||
|
||
def test_video_export_accepts_finalized_worker_trace_and_rejects_external_path(
|
||
isolated, tmp_path, monkeypatch
|
||
):
|
||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||
session_id = job["session_id"]
|
||
recording_dir = reports.TRACES_PATH / f"web_123_{session_id[:8]}_PASS_2026-09-28"
|
||
recording_dir.mkdir(parents=True)
|
||
first = recording_dir / "recording.mp4"
|
||
first.write_bytes(b"first segment")
|
||
resumed = recording_dir / "resume_0001"
|
||
resumed.mkdir()
|
||
second = resumed / "recording.mp4"
|
||
second.write_bytes(b"second segment")
|
||
|
||
repository = Mock()
|
||
repository.db_path = isolated.path
|
||
repository.get_video_recording_for_session.return_value = {"local_video_path": str(first)}
|
||
repository.get_session_by_id.return_value = {"video_filepath": str(first)}
|
||
monkeypatch.setattr(reports, "SessionRepository", lambda: repository)
|
||
|
||
archive_path = reports.export_session_media(session_id, "video")
|
||
with zipfile.ZipFile(archive_path) as archive:
|
||
names = archive.namelist()
|
||
assert names == [
|
||
"任务录屏/安卓/0001_web_123_" + session_id[:8] + "_PASS_2026-09-28_recording.mp4",
|
||
"任务录屏/安卓/0002_resume_0001_recording.mp4",
|
||
"文件清单.json",
|
||
]
|
||
assert archive.read(names[0]) == b"first segment"
|
||
assert archive.read(names[1]) == b"second segment"
|
||
|
||
external = tmp_path / "outside" / "recording.mp4"
|
||
external.parent.mkdir()
|
||
external.write_bytes(b"unrelated")
|
||
repository.get_video_recording_for_session.return_value = {"local_video_path": str(external)}
|
||
assert reports.session_videos({"id": session_id}) == []
|
||
|
||
|
||
def test_history_filters_all_persisted_rows_and_escapes_like(client, isolated):
|
||
batch = service.create_batch(payload())
|
||
first, second = isolated.jobs(batch["id"])
|
||
isolated.save_job(
|
||
{
|
||
**first,
|
||
"status": "completed",
|
||
"created_at": 100,
|
||
"case": {"name": "搜索_%", "goal": "abc", "tags": "回归"},
|
||
}
|
||
)
|
||
isolated.save_job({**second, "status": "failed", "created_at": 200})
|
||
assert client.get("/api/automation/history", params={"q": "_%"}).json()["total"] == 1
|
||
assert (
|
||
client.get("/api/automation/history", params={"q": "回归"}).json()["items"][0]["session_id"]
|
||
== first["session_id"]
|
||
)
|
||
assert (
|
||
client.get(
|
||
"/api/automation/history", params={"start": 150, "end": 250, "status": "failed"}
|
||
).json()["total"]
|
||
== 1
|
||
)
|
||
assert (
|
||
client.get("/api/automation/history", params={"size": 1, "page": 2}).json()["items"][0][
|
||
"session_id"
|
||
]
|
||
== first["session_id"]
|
||
)
|
||
|
||
|
||
def test_history_returns_persisted_execution_mode_without_inventing_legacy_mode(client, isolated):
|
||
with isolated.connection() as connection:
|
||
for sid, device_info in [
|
||
("flash", '{"profile":"flash"}'),
|
||
("pro", '{"profile":"pro"}'),
|
||
("legacy", "not json"),
|
||
]:
|
||
connection.execute(
|
||
"INSERT INTO sessions(session_id,initial_goal,start_time,status,device_info) VALUES (?,?,?,?,?)",
|
||
(sid, sid, 100, "completed", device_info),
|
||
)
|
||
connection.commit()
|
||
isolated.save_job(
|
||
{
|
||
"session_id": "queued-only",
|
||
"status": "failed",
|
||
"case": {"name": "no device"},
|
||
"task_config": {"profile": "pro"},
|
||
"created_at": 100,
|
||
}
|
||
)
|
||
rows = {row["session_id"]: row for row in client.get("/api/automation/history").json()["items"]}
|
||
assert rows["flash"]["profile"] == "flash"
|
||
assert rows["pro"]["profile"] == "pro"
|
||
assert rows["legacy"]["profile"] is None
|
||
assert rows["queued-only"]["profile"] == "pro"
|
||
|
||
|
||
def test_batch_submission_is_atomic_and_idempotent(client, isolated):
|
||
value = payload(request_id="same-request")
|
||
first = client.post("/api/automation/batches", json=value)
|
||
second = client.post("/api/automation/batches", json=value)
|
||
assert first.status_code == second.status_code == 200
|
||
assert first.json()["id"] == second.json()["id"]
|
||
assert len(isolated.jobs()) == 2
|
||
assert len(isolated.records("batch")) == 1
|
||
conflict = client.post("/api/automation/batches", json={**value, "name": "different"})
|
||
assert conflict.status_code == 409
|
||
lookup = client.get("/api/automation/batches/by-request/same-request").json()
|
||
assert lookup["id"] == first.json()["id"]
|
||
assert len(lookup["jobs"]) == 2
|
||
|
||
|
||
def test_removed_replay_is_inert_but_history_survives(client, isolated):
|
||
batch = service.create_batch(payload())
|
||
job = isolated.jobs(batch["id"])[0]
|
||
isolated.save_job({**job, "execution_mode": "strict", "workflow_id": "legacy"})
|
||
service.recover_batches()
|
||
assert isolated.get("batch", batch["id"])["status"] == "cancelled"
|
||
assert all(j["status"] == "cancelled" for j in isolated.jobs(batch["id"]))
|
||
assert client.post(f"/api/automation/batches/{batch['id']}/resume").status_code == 409
|
||
assert client.post(f"/api/automation/batches/{batch['id']}/retry_failed").status_code == 409
|
||
assert client.get("/api/automation/workflows").status_code == 404
|
||
assert client.get("/api/automation/history").json()["total"] == 2
|
||
|
||
|
||
def test_exploration_history_groups_before_pagination_and_preserves_validations(client, isolated):
|
||
with isolated.connection() as conn:
|
||
for i, kind in enumerate(["exploration", "exploration", "validation", "validation"]):
|
||
sid = f"run-{i}"
|
||
conn.execute(
|
||
"INSERT INTO sessions(session_id,initial_goal,start_time,status) VALUES (?,?,?,?)",
|
||
(sid, "目标", i + 1, "completed"),
|
||
)
|
||
record = {
|
||
"exploration_id": "conversation",
|
||
"kind": kind,
|
||
"created_at": i + 1,
|
||
"status": "completed",
|
||
"case_version": 1,
|
||
}
|
||
conn.execute(
|
||
"INSERT INTO automation_records VALUES (?,?,?,?)",
|
||
("exploration_run", sid, json.dumps(record), i + 1),
|
||
)
|
||
conn.commit()
|
||
all_rows = client.get("/api/automation/history").json()
|
||
assert all_rows["total"] == 3
|
||
assert {r["session_id"] for r in all_rows["items"]} == {"run-1", "run-2", "run-3"}
|
||
page = client.get("/api/automation/history?batch_id=__exploration__&size=1").json()
|
||
assert page["total"] == 1
|
||
assert page["items"][0]["session_id"] == "run-1"
|
||
assert page["items"][0]["exploration_run_count"] == 2
|
||
ordinary = client.get("/api/automation/history?batch_id=__single__").json()
|
||
assert ordinary["total"] == 2
|
||
with isolated.connection() as conn:
|
||
conn.execute(
|
||
"INSERT INTO automation_records VALUES (?,?,?,?)",
|
||
(
|
||
"exploration",
|
||
"conversation",
|
||
json.dumps({"status": "paused", "active_run_id": None}),
|
||
5,
|
||
),
|
||
)
|
||
conn.commit()
|
||
assert client.get("/api/automation/history?batch_id=__exploration__").json()["total"] == 0
|
||
|
||
|
||
def test_validation_history_uses_case_title_after_loading_its_job(client, isolated):
|
||
from artemis.config.exploration import ExplorationConfig
|
||
from artemis.exploration.models import CaseDraft
|
||
from artemis.exploration.store import ExplorationStore
|
||
|
||
repository = ExplorationStore(isolated)
|
||
session = repository.create("探索设置", ExplorationConfig(), {}, "create")
|
||
repository.save_case(
|
||
session["id"], CaseDraft(name="深色主题检查", goal="完整执行提示词"), 0, "save"
|
||
)
|
||
run = repository.reserve_run(
|
||
session["id"],
|
||
"validate",
|
||
kind="validation",
|
||
goal="完整执行提示词",
|
||
task_config={},
|
||
case_version=1,
|
||
)
|
||
isolated.save_job(
|
||
{
|
||
"session_id": run["id"],
|
||
"status": "completed",
|
||
"name": "完整执行提示词",
|
||
"goal": "完整执行提示词",
|
||
}
|
||
)
|
||
with isolated.connection() as conn:
|
||
conn.execute(
|
||
"INSERT INTO sessions(session_id,initial_goal,start_time,status) VALUES (?,?,?,?)",
|
||
(run["id"], "完整执行提示词", 1, "completed"),
|
||
)
|
||
conn.commit()
|
||
rows = client.get("/api/automation/history?q=深色主题检查").json()["items"]
|
||
assert len(rows) == 1
|
||
assert rows[0]["name"] == "深色主题检查"
|
||
assert rows[0]["initial_goal"] == "完整执行提示词"
|
||
assert rows[0]["case_version"] == 1
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_queued_exploration_keeps_identity_without_a_session_row(isolated, monkeypatch):
|
||
from apps.admin_console.routers import tasks
|
||
from artemis.exploration.store import ExplorationStore
|
||
from artemis.exploration.models import CreateExploration
|
||
from apps.admin_console.services.exploration_service import ExplorationService
|
||
|
||
service = ExplorationService(ExplorationStore(isolated))
|
||
session = service.create(CreateExploration(goal="排队探索", request_id="create"))
|
||
run = service.repository.reserve_run(
|
||
session["id"],
|
||
"start",
|
||
kind="exploration",
|
||
goal="排队探索",
|
||
task_config={"model": "gpt-6.1-sol"},
|
||
)
|
||
queue_item = {"session_id": run["id"], "goal": run["goal"], "status": "pending"}
|
||
from types import SimpleNamespace
|
||
|
||
monkeypatch.setattr(
|
||
tasks,
|
||
"state",
|
||
SimpleNamespace(
|
||
is_running=False,
|
||
queue_items=[],
|
||
queue_tasks=[queue_item],
|
||
active_runs={},
|
||
active_connections={},
|
||
active_session_id=None,
|
||
current_goal=None,
|
||
current_profile=None,
|
||
ipc_port=0,
|
||
),
|
||
)
|
||
monkeypatch.setattr(tasks, "store", isolated)
|
||
monkeypatch.setattr(tasks.task_queue_service, "ensure_worker_running", lambda: None)
|
||
monkeypatch.setattr(tasks, "session_repo", Mock(get_latest_session=lambda: None))
|
||
monkeypatch.setattr(tasks.DeviceExecutionLock, "get_active_owner", lambda: None)
|
||
monkeypatch.setattr(tasks.DeviceExecutionLock, "get_active_owners", lambda: {})
|
||
monkeypatch.setattr(tasks.DeviceExecutionLock, "get_queued_tasks", lambda: [])
|
||
result = await tasks.get_status()
|
||
assert result["queue"][0]["exploration_id"] == session["id"]
|
||
assert result["queue"][0]["exploration_kind"] == "exploration"
|
||
assert "exploration_id" not in queue_item # Do not mutate scheduler state.
|