feat: add favorite task templates and optimize evidence exports
This commit is contained in:
1 parent
e1ea099592
commit
a4a5430922
46 files changed
+1714
-188
No files matched your search
@@ -158,6 +158,7 @@ def test_completed_batch_reuses_existing_archive(client, isolated, monkeypatch):
|
||||
path = reports.EXPORT_ROOT / "batches" / batch_id / "批次报告.zip"
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_bytes(b"existing archive")
|
||||
reports.remember(path, reports.BATCH_ARCHIVE_SIGNATURE)
|
||||
exporter = Mock(side_effect=AssertionError("archive must not be rebuilt"))
|
||||
monkeypatch.setattr(reports, "export_batch", exporter)
|
||||
base = f"/api/automation/batches/{batch_id}/archive"
|
||||
@@ -166,6 +167,21 @@ def test_completed_batch_reuses_existing_archive(client, isolated, monkeypatch):
|
||||
exporter.assert_not_called()
|
||||
|
||||
|
||||
def test_legacy_batch_archive_is_rebuilt_with_current_recording_layout(client, isolated):
|
||||
batch = service.create_batch(payload())
|
||||
batch_id = batch["id"]
|
||||
isolated.put("batch", batch_id, {**batch, "status": "completed", "report_ready": True})
|
||||
path = reports.EXPORT_ROOT / "batches" / batch_id / "批次报告.zip"
|
||||
path.parent.mkdir(parents=True)
|
||||
path.write_bytes(b"legacy archive")
|
||||
base = f"/api/automation/batches/{batch_id}/archive"
|
||||
assert client.get(base + "/status").json()["status"] == "idle"
|
||||
response = client.get(base)
|
||||
assert response.status_code == 200
|
||||
assert zipfile.is_zipfile(BytesIO(response.content))
|
||||
assert client.get(base + "/status").json()["status"] == "ready"
|
||||
|
||||
|
||||
def test_preview_does_not_enqueue(client, isolated):
|
||||
response = client.post(
|
||||
"/api/automation/import/preview",
|
||||
@@ -1021,7 +1037,7 @@ def test_task_material_zip_contains_ordered_evidence_and_reports(
|
||||
assert files.index("1.1/步骤截图/0001_步骤0001_操作前.png") < files.index(
|
||||
"1.1/步骤截图/0002_步骤0002_操作后.png"
|
||||
)
|
||||
assert archive.read("1.1/任务录屏/0001_recording.mp4") == b"recording.mp4"
|
||||
assert archive.read("1.1/任务录屏/安卓/0001_recording.mp4") == b"recording.mp4"
|
||||
manifest = json.loads(archive.read("1.1/文件清单.json"))
|
||||
assert manifest["session_id"] == job["session_id"]
|
||||
assert manifest["artifacts"] == ["安卓截图.png"]
|
||||
@@ -1042,6 +1058,63 @@ def test_batch_zip_contains_each_tasks_android_artifacts(isolated):
|
||||
assert {archive.read(name).decode() for name in names} == {"任务0", "任务1"}
|
||||
|
||||
|
||||
def test_all_material_cache_reuses_reports_and_invalidates_on_review_or_new_files(
|
||||
isolated, monkeypatch
|
||||
):
|
||||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||||
job["status"] = "completed"
|
||||
isolated.save_job(job)
|
||||
word = Mock(wraps=reports.word_report)
|
||||
monkeypatch.setattr(reports, "word_report", word)
|
||||
monkeypatch.setattr(
|
||||
reports,
|
||||
"diagnostics_json",
|
||||
lambda sid: json.dumps({"requests": [], "collected_at": time.time()}),
|
||||
)
|
||||
first = reports.export_session_media(job["session_id"], "all")
|
||||
first_bytes = first.read_bytes()
|
||||
reports.export_session_media(job["session_id"], "all")
|
||||
assert word.call_count == 1
|
||||
assert first.read_bytes() == first_bytes
|
||||
isolated.save_case_review(job["session_id"], "passed", "人工核对")
|
||||
reports.export_session_media(job["session_id"], "all")
|
||||
assert word.call_count == 2
|
||||
folder = reports.TRACES_PATH / job["session_id"] / "artifacts" / "android"
|
||||
folder.mkdir(parents=True)
|
||||
(folder / "新资料.txt").write_text("new evidence", encoding="utf-8")
|
||||
reports.export_session_media(job["session_id"], "all")
|
||||
with zipfile.ZipFile(first) as archive:
|
||||
assert any(name.endswith("任务文件/安卓/新资料.txt") for name in archive.namelist())
|
||||
assert not json.loads(
|
||||
archive.read(next(n for n in archive.namelist() if n.endswith("文件清单.json")))
|
||||
)["snapshot"]
|
||||
# File-only changes rebuild the ZIP while keeping unchanged documents.
|
||||
assert word.call_count == 2
|
||||
|
||||
|
||||
def test_different_tasks_export_concurrently_and_running_exports_are_snapshots(
|
||||
isolated, monkeypatch
|
||||
):
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
jobs = isolated.jobs(service.create_batch(payload())["id"])
|
||||
barrier = threading.Barrier(2)
|
||||
|
||||
def generate(report, path):
|
||||
barrier.wait(timeout=5)
|
||||
path.write_bytes(b"generated report")
|
||||
|
||||
monkeypatch.setattr(reports, "word_report", generate)
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
exports = [pool.submit(reports.export_session, j["session_id"], ["docx"]) for j in jobs]
|
||||
assert all(result.result(timeout=10)[0].is_file() for result in exports)
|
||||
count = Mock(side_effect=lambda report, path: path.write_bytes(b"snapshot"))
|
||||
monkeypatch.setattr(reports, "word_report", count)
|
||||
for _ in range(2):
|
||||
reports.export_session(jobs[0]["session_id"], ["docx"])
|
||||
assert count.call_count == 2
|
||||
|
||||
|
||||
def test_android_artifact_export_does_not_follow_other_session_links(isolated):
|
||||
root = reports.TRACES_PATH / "session" / "artifacts"
|
||||
root.mkdir(parents=True)
|
||||
@@ -1078,8 +1151,8 @@ def test_video_export_accepts_finalized_worker_trace_and_rejects_external_path(
|
||||
with zipfile.ZipFile(archive_path) as archive:
|
||||
names = archive.namelist()
|
||||
assert names == [
|
||||
"任务录屏/0001_web_123_" + session_id[:8] + "_PASS_2026-09-28_recording.mp4",
|
||||
"任务录屏/0002_resume_0001_recording.mp4",
|
||||
"任务录屏/安卓/0001_web_123_" + session_id[:8] + "_PASS_2026-09-28_recording.mp4",
|
||||
"任务录屏/安卓/0002_resume_0001_recording.mp4",
|
||||
"文件清单.json",
|
||||
]
|
||||
assert archive.read(names[0]) == b"first segment"
|
||||
|
||||
@@ -0,0 +1,325 @@
|
||||
"""Favorite snapshots and transport retries never replay an old result or task."""
|
||||
|
||||
import asyncio
|
||||
from copy import deepcopy
|
||||
import json
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import ValidationError
|
||||
import pytest
|
||||
|
||||
from apps.admin_console.services import automation_service
|
||||
from apps.admin_console.services.automation_store import AutomationStore
|
||||
from apps.admin_console.services.favorites import (
|
||||
FavoriteCreate,
|
||||
FavoriteRun,
|
||||
FavoriteService,
|
||||
FavoriteUpdate,
|
||||
)
|
||||
from artemis.config.exploration import ExplorationConfig
|
||||
from artemis.data_engine.storage import StorageManager
|
||||
from artemis.exploration.models import CaseDraft
|
||||
from artemis.exploration.store import ExplorationStore
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def favorites(tmp_path, monkeypatch):
|
||||
storage = AutomationStore(tmp_path / "history.db")
|
||||
StorageManager(storage.path, tmp_path / "traces")
|
||||
monkeypatch.setattr(automation_service, "store", storage)
|
||||
return FavoriteService(storage)
|
||||
|
||||
|
||||
def task(service, session_id="task", **options):
|
||||
config = {
|
||||
"profile": "pro",
|
||||
"model": "original-model",
|
||||
"reasoning_effort": "high",
|
||||
"device_serial": "original-device",
|
||||
"record_video": True,
|
||||
**options,
|
||||
}
|
||||
job = {
|
||||
"session_id": session_id,
|
||||
"status": "completed",
|
||||
"case": {"name": "原用例", "goal": "检查真实页面"},
|
||||
"execution_config": {
|
||||
"runtime_overrides": config,
|
||||
"expected_output": "保留证据",
|
||||
"enable_outputter": True,
|
||||
},
|
||||
}
|
||||
service.store.save_job(job)
|
||||
return service.create(FavoriteCreate(source_id=session_id))
|
||||
|
||||
|
||||
def test_snapshot_survives_source_deletion_and_edits_do_not_modify_source(favorites):
|
||||
item = task(favorites)
|
||||
updated = favorites.update(
|
||||
item["id"], FavoriteUpdate(name="常用检查", task_config={"model": "new-model"})
|
||||
)
|
||||
assert updated["snapshot"]["runtime_overrides"]["model"] == "new-model"
|
||||
assert (
|
||||
favorites.store.job("task")["execution_config"]["runtime_overrides"]["model"]
|
||||
== "original-model"
|
||||
)
|
||||
favorites.store.delete_history()
|
||||
assert favorites.get(item["id"])["snapshot"]["goal"] == "检查真实页面"
|
||||
assert favorites.store.job("task") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rerun_preserves_settings_and_is_durably_idempotent(favorites, monkeypatch):
|
||||
item = task(favorites)
|
||||
enqueue = AsyncMock(return_value={"tasks": [{"session_id": "new-task"}], "status": "queued"})
|
||||
monkeypatch.setattr("apps.admin_console.routers.tasks.run_task", enqueue)
|
||||
value = FavoriteRun(request_id="same-submission", task_config={"reasoning_effort": "max"})
|
||||
first, second = await asyncio.gather(
|
||||
favorites.run(item["id"], value), favorites.run(item["id"], value)
|
||||
)
|
||||
assert first == second
|
||||
enqueue.assert_awaited_once()
|
||||
request = enqueue.call_args.args[0]
|
||||
assert request.session_id != "task"
|
||||
assert request.runtime_overrides.model == "original-model"
|
||||
assert request.runtime_overrides.reasoning_effort == "max"
|
||||
assert request.runtime_overrides.record_video is True
|
||||
assert request.runtime_overrides.device_serial == "original-device"
|
||||
assert request.expected_output == "保留证据"
|
||||
assert favorites.get(item["id"])["snapshot"]["runtime_overrides"]["reasoning_effort"] == "high"
|
||||
favorites.delete(item["id"])
|
||||
assert await FavoriteService(favorites.store).run(item["id"], value) == first
|
||||
enqueue.assert_awaited_once()
|
||||
with pytest.raises(ValueError, match="request_id"):
|
||||
await favorites.run(
|
||||
item["id"],
|
||||
FavoriteRun(request_id="same-submission", task_config={"model": "different"}),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lost_admission_reply_reuses_snapshot_and_session(favorites, monkeypatch):
|
||||
item = task(favorites)
|
||||
enqueue = AsyncMock(side_effect=[ConnectionError("reply lost"), {"tasks": [{}]}])
|
||||
monkeypatch.setattr("apps.admin_console.routers.tasks.run_task", enqueue)
|
||||
value = FavoriteRun(request_id="retry")
|
||||
with pytest.raises(ConnectionError):
|
||||
await favorites.run(item["id"], value)
|
||||
favorites.update(item["id"], FavoriteUpdate(task_config={"model": "changed"}))
|
||||
await FavoriteService(favorites.store).run(item["id"], value)
|
||||
requests = [call.args[0] for call in enqueue.call_args_list]
|
||||
assert requests[0].session_id == requests[1].session_id
|
||||
assert requests[1].runtime_overrides.model == "original-model"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_favorite_keeps_one_round_and_original_repetitions(favorites):
|
||||
batch = automation_service.create_batch(
|
||||
{
|
||||
"name": "压测",
|
||||
"repeat_count": 3,
|
||||
"profile": "flash",
|
||||
"device_serial": "device",
|
||||
"cases": [
|
||||
{"name": "甲", "goal": "检查甲", "task_config": {"model": "model-a"}},
|
||||
{
|
||||
"name": "乙",
|
||||
"goal": "检查乙",
|
||||
"task_config": {"profile": "pro", "model": "model-b"},
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
item = favorites.create(FavoriteCreate(source_kind="batch", source_id=batch["id"]))
|
||||
assert len(item["snapshot"]["cases"]) == 2
|
||||
assert item["snapshot"]["repeat_count"] == 3
|
||||
result = await favorites.run(
|
||||
item["id"],
|
||||
FavoriteRun(
|
||||
request_id="batch-rerun",
|
||||
task_config={"reasoning_effort": "max"},
|
||||
case_configs={1: {"model": "changed-b"}},
|
||||
),
|
||||
)
|
||||
jobs = favorites.store.jobs(result["batch_id"])
|
||||
assert len(jobs) == 6
|
||||
assert [j["case"]["name"] for j in jobs] == ["甲", "乙"] * 3
|
||||
assert [j["case"]["task_config"]["model"] for j in jobs] == ["model-a", "changed-b"] * 3
|
||||
assert all(j["case"]["task_config"]["reasoning_effort"] == "max" for j in jobs)
|
||||
assert favorites.store.get("batch", result["batch_id"])["status"] == "running"
|
||||
again = await favorites.run(
|
||||
item["id"],
|
||||
FavoriteRun(
|
||||
request_id="batch-rerun",
|
||||
task_config={"reasoning_effort": "max"},
|
||||
case_configs={1: {"model": "changed-b"}},
|
||||
),
|
||||
)
|
||||
assert again == result
|
||||
assert len(favorites.store.records("batch")) == 2
|
||||
|
||||
|
||||
def test_exploration_case_pins_version_without_inheriting_expensive_model(favorites):
|
||||
repo = ExplorationStore(favorites.store)
|
||||
session = repo.create(
|
||||
"探索目标",
|
||||
ExplorationConfig(model="exploration-model", reasoning_effort="max"),
|
||||
{"model": "exploration-model", "device_serial": "device"},
|
||||
"create",
|
||||
)
|
||||
repo.save_case(
|
||||
session["id"],
|
||||
CaseDraft(name="完成用例", goal="验证主页", preconditions="已登录"),
|
||||
0,
|
||||
"case-v1",
|
||||
)
|
||||
item = favorites.create(FavoriteCreate(source_kind="exploration_case", source_id=session["id"]))
|
||||
assert item["case_version"] == 1
|
||||
assert item["snapshot"]["goal"] == "前置条件:\n已登录\n\n验证主页"
|
||||
assert item["snapshot"]["runtime_overrides"]["profile"] == "flash"
|
||||
assert item["snapshot"]["runtime_overrides"]["model"] != "exploration-model"
|
||||
repo.save_case(session["id"], CaseDraft(name="新用例", goal="新目标"), 1, "case-v2")
|
||||
assert favorites.get(item["id"])["snapshot"]["goal"].endswith("验证主页")
|
||||
exploration = favorites.create(
|
||||
FavoriteCreate(source_kind="exploration", source_id=session["id"])
|
||||
)
|
||||
assert exploration["snapshot"]["config"]["model"] == "exploration-model"
|
||||
|
||||
|
||||
def test_old_task_recovers_real_model_and_marks_missing_settings(favorites):
|
||||
with favorites.store.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions(session_id,initial_goal,status,device_info) VALUES (?,?,?,?)",
|
||||
(
|
||||
"legacy",
|
||||
"检查浏览器",
|
||||
"completed",
|
||||
json.dumps({"profile": "flash", "serial": "computer:local"}),
|
||||
),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO traces(trace_id,session_id,type,timestamp,payload) VALUES (?,?,?,?,?)",
|
||||
(
|
||||
"request",
|
||||
"legacy",
|
||||
"llm_diagnostic",
|
||||
1,
|
||||
json.dumps(
|
||||
{"role": "FlashRunner", "model": "actual-model", "reasoning_effort": "max"}
|
||||
),
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
item = favorites.create(FavoriteCreate(source_id="legacy"))
|
||||
options = item["snapshot"]["runtime_overrides"]
|
||||
assert options["model"] == "actual-model"
|
||||
assert options["reasoning_effort"] == "max"
|
||||
assert options["execution_target"] == "computer"
|
||||
assert "model" not in item["inherited_fields"]
|
||||
assert "recording" in item["inherited_fields"]
|
||||
|
||||
|
||||
def test_api_rejects_credentials_and_arbitrary_snapshots(favorites, monkeypatch):
|
||||
from apps.admin_console.routers import favorites as api
|
||||
|
||||
monkeypatch.setattr(api, "favorites", favorites)
|
||||
app = FastAPI()
|
||||
app.include_router(api.router)
|
||||
client = TestClient(app)
|
||||
item = task(favorites)
|
||||
assert len(client.get("/api/automation/favorites").json()) == 1
|
||||
assert (
|
||||
client.patch(
|
||||
f"/api/automation/favorites/{item['id']}", json={"task_config": {"api_key": "never"}}
|
||||
).status_code
|
||||
== 422
|
||||
)
|
||||
assert (
|
||||
client.post(
|
||||
"/api/automation/favorites", json={"source_id": "task", "snapshot": {}}
|
||||
).status_code
|
||||
== 422
|
||||
)
|
||||
assert (
|
||||
client.patch(f"/api/automation/favorites/{item['id']}", json={"name": " "}).status_code
|
||||
== 409
|
||||
)
|
||||
assert client.delete(f"/api/automation/favorites/{item['id']}").status_code == 200
|
||||
assert favorites.store.job("task")
|
||||
with pytest.raises(ValidationError):
|
||||
FavoriteRun(request_id="run", task_config={"password": "never"})
|
||||
|
||||
|
||||
def test_collection_defaults_cannot_modify_original_snapshot(favorites):
|
||||
item = task(favorites)
|
||||
original = deepcopy(item)
|
||||
favorites.updated(item, FavoriteUpdate(task_config={"model": "temporary"}))
|
||||
assert item == original
|
||||
|
||||
|
||||
def test_partial_recording_update_keeps_other_favorite_settings(favorites):
|
||||
item = task(favorites, recording={"codec": "h264", "fps": 10, "max_long_edge": 720})
|
||||
updated = favorites.updated(item, FavoriteUpdate(task_config={"recording": {"codec": "h265"}}))
|
||||
assert updated["snapshot"]["runtime_overrides"]["recording"] == {
|
||||
"codec": "h265",
|
||||
"fps": 10,
|
||||
"max_long_edge": 720,
|
||||
}
|
||||
assert item["snapshot"]["runtime_overrides"]["recording"]["codec"] == "h264"
|
||||
|
||||
|
||||
def test_legacy_empty_recording_configuration_uses_current_defaults():
|
||||
from apps.admin_console.services.execution_config import execution_config
|
||||
from artemis.config.agent import load_agent_config
|
||||
|
||||
snapshot = execution_config({"profile": "flash", "runtime_overrides": {"recording": None}})
|
||||
assert snapshot["runtime_overrides"]["recording"] == load_agent_config().recording.model_dump()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("rejected", [False, True])
|
||||
async def test_restart_exploration_creates_new_conversation_and_reports_rejection(
|
||||
favorites, monkeypatch, rejected
|
||||
):
|
||||
from apps.admin_console.services.exploration_service import ExplorationService
|
||||
|
||||
repo = ExplorationStore(favorites.store)
|
||||
original = repo.create(
|
||||
"探索目标",
|
||||
ExplorationConfig(model="explore-model", reasoning_effort="max"),
|
||||
{"device_serial": "device", "record_video": True},
|
||||
"source",
|
||||
)
|
||||
item = favorites.create(FavoriteCreate(source_kind="exploration", source_id=original["id"]))
|
||||
monkeypatch.setattr(
|
||||
"apps.admin_console.services.exploration_service.exploration_service",
|
||||
ExplorationService(repo),
|
||||
)
|
||||
enqueue = AsyncMock(
|
||||
return_value=(
|
||||
{"status": "rejected", "error": "设备离线"}
|
||||
if rejected
|
||||
else {"status": "queued", "tasks": [{}]}
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr("apps.admin_console.routers.tasks.run_task", enqueue)
|
||||
value = FavoriteRun(request_id="new-exploration", task_config={"model": "selected-model"})
|
||||
if rejected:
|
||||
with pytest.raises(ValueError, match="设备离线"):
|
||||
await favorites.run(item["id"], value)
|
||||
with pytest.raises(ValueError, match="设备离线"):
|
||||
await favorites.run(item["id"], value)
|
||||
else:
|
||||
result = await favorites.run(item["id"], value)
|
||||
assert result["exploration_id"] != original["id"]
|
||||
assert repo.run(result["session_id"])["exploration_id"] == result["exploration_id"]
|
||||
assert await favorites.run(item["id"], value) == result
|
||||
request = enqueue.call_args.args[0]
|
||||
assert request.conversation_id != original["id"]
|
||||
assert request.runtime_overrides.model == "selected-model"
|
||||
assert request.runtime_overrides.reasoning_effort == "max"
|
||||
assert request.runtime_overrides.record_video is True
|
||||
enqueue.assert_awaited_once()
|
||||
assert repo.get(original["id"]) == original
|
||||
Reference in new issue
Block a user