feat: add favorite task templates and optimize evidence exports

This commit is contained in:
chenmingxuan committed 2026-10-18 19:44:51 +08:00
1 parent e1ea099592
commit a4a5430922
46 files changed
+1714 -188

No files matched your search

+76 -3
View File
@@ -158,6 +158,7 @@ def test_completed_batch_reuses_existing_archive(client, isolated, monkeypatch):
path = reports.EXPORT_ROOT / "batches" / batch_id / "批次报告.zip"
path.parent.mkdir(parents=True)
path.write_bytes(b"existing archive")
reports.remember(path, reports.BATCH_ARCHIVE_SIGNATURE)
exporter = Mock(side_effect=AssertionError("archive must not be rebuilt"))
monkeypatch.setattr(reports, "export_batch", exporter)
base = f"/api/automation/batches/{batch_id}/archive"
@@ -166,6 +167,21 @@ def test_completed_batch_reuses_existing_archive(client, isolated, monkeypatch):
exporter.assert_not_called()
def test_legacy_batch_archive_is_rebuilt_with_current_recording_layout(client, isolated):
batch = service.create_batch(payload())
batch_id = batch["id"]
isolated.put("batch", batch_id, {**batch, "status": "completed", "report_ready": True})
path = reports.EXPORT_ROOT / "batches" / batch_id / "批次报告.zip"
path.parent.mkdir(parents=True)
path.write_bytes(b"legacy archive")
base = f"/api/automation/batches/{batch_id}/archive"
assert client.get(base + "/status").json()["status"] == "idle"
response = client.get(base)
assert response.status_code == 200
assert zipfile.is_zipfile(BytesIO(response.content))
assert client.get(base + "/status").json()["status"] == "ready"
def test_preview_does_not_enqueue(client, isolated):
response = client.post(
"/api/automation/import/preview",
@@ -1021,7 +1037,7 @@ def test_task_material_zip_contains_ordered_evidence_and_reports(
assert files.index("1.1/步骤截图/0001_步骤0001_操作前.png") < files.index(
"1.1/步骤截图/0002_步骤0002_操作后.png"
)
assert archive.read("1.1/任务录屏/0001_recording.mp4") == b"recording.mp4"
assert archive.read("1.1/任务录屏/安卓/0001_recording.mp4") == b"recording.mp4"
manifest = json.loads(archive.read("1.1/文件清单.json"))
assert manifest["session_id"] == job["session_id"]
assert manifest["artifacts"] == ["安卓截图.png"]
@@ -1042,6 +1058,63 @@ def test_batch_zip_contains_each_tasks_android_artifacts(isolated):
assert {archive.read(name).decode() for name in names} == {"任务0", "任务1"}
def test_all_material_cache_reuses_reports_and_invalidates_on_review_or_new_files(
isolated, monkeypatch
):
job = isolated.jobs(service.create_batch(payload())["id"])[0]
job["status"] = "completed"
isolated.save_job(job)
word = Mock(wraps=reports.word_report)
monkeypatch.setattr(reports, "word_report", word)
monkeypatch.setattr(
reports,
"diagnostics_json",
lambda sid: json.dumps({"requests": [], "collected_at": time.time()}),
)
first = reports.export_session_media(job["session_id"], "all")
first_bytes = first.read_bytes()
reports.export_session_media(job["session_id"], "all")
assert word.call_count == 1
assert first.read_bytes() == first_bytes
isolated.save_case_review(job["session_id"], "passed", "人工核对")
reports.export_session_media(job["session_id"], "all")
assert word.call_count == 2
folder = reports.TRACES_PATH / job["session_id"] / "artifacts" / "android"
folder.mkdir(parents=True)
(folder / "新资料.txt").write_text("new evidence", encoding="utf-8")
reports.export_session_media(job["session_id"], "all")
with zipfile.ZipFile(first) as archive:
assert any(name.endswith("任务文件/安卓/新资料.txt") for name in archive.namelist())
assert not json.loads(
archive.read(next(n for n in archive.namelist() if n.endswith("文件清单.json")))
)["snapshot"]
# File-only changes rebuild the ZIP while keeping unchanged documents.
assert word.call_count == 2
def test_different_tasks_export_concurrently_and_running_exports_are_snapshots(
isolated, monkeypatch
):
from concurrent.futures import ThreadPoolExecutor
jobs = isolated.jobs(service.create_batch(payload())["id"])
barrier = threading.Barrier(2)
def generate(report, path):
barrier.wait(timeout=5)
path.write_bytes(b"generated report")
monkeypatch.setattr(reports, "word_report", generate)
with ThreadPoolExecutor(max_workers=2) as pool:
exports = [pool.submit(reports.export_session, j["session_id"], ["docx"]) for j in jobs]
assert all(result.result(timeout=10)[0].is_file() for result in exports)
count = Mock(side_effect=lambda report, path: path.write_bytes(b"snapshot"))
monkeypatch.setattr(reports, "word_report", count)
for _ in range(2):
reports.export_session(jobs[0]["session_id"], ["docx"])
assert count.call_count == 2
def test_android_artifact_export_does_not_follow_other_session_links(isolated):
root = reports.TRACES_PATH / "session" / "artifacts"
root.mkdir(parents=True)
@@ -1078,8 +1151,8 @@ def test_video_export_accepts_finalized_worker_trace_and_rejects_external_path(
with zipfile.ZipFile(archive_path) as archive:
names = archive.namelist()
assert names == [
"任务录屏/0001_web_123_" + session_id[:8] + "_PASS_2026-09-28_recording.mp4",
"任务录屏/0002_resume_0001_recording.mp4",
"任务录屏/安卓/0001_web_123_" + session_id[:8] + "_PASS_2026-09-28_recording.mp4",
"任务录屏/安卓/0002_resume_0001_recording.mp4",
"文件清单.json",
]
assert archive.read(names[0]) == b"first segment"
+325
View File
@@ -0,0 +1,325 @@
"""Favorite snapshots and transport retries never replay an old result or task."""
import asyncio
from copy import deepcopy
import json
from unittest.mock import AsyncMock
from fastapi import FastAPI
from fastapi.testclient import TestClient
from pydantic import ValidationError
import pytest
from apps.admin_console.services import automation_service
from apps.admin_console.services.automation_store import AutomationStore
from apps.admin_console.services.favorites import (
FavoriteCreate,
FavoriteRun,
FavoriteService,
FavoriteUpdate,
)
from artemis.config.exploration import ExplorationConfig
from artemis.data_engine.storage import StorageManager
from artemis.exploration.models import CaseDraft
from artemis.exploration.store import ExplorationStore
@pytest.fixture
def favorites(tmp_path, monkeypatch):
storage = AutomationStore(tmp_path / "history.db")
StorageManager(storage.path, tmp_path / "traces")
monkeypatch.setattr(automation_service, "store", storage)
return FavoriteService(storage)
def task(service, session_id="task", **options):
config = {
"profile": "pro",
"model": "original-model",
"reasoning_effort": "high",
"device_serial": "original-device",
"record_video": True,
**options,
}
job = {
"session_id": session_id,
"status": "completed",
"case": {"name": "原用例", "goal": "检查真实页面"},
"execution_config": {
"runtime_overrides": config,
"expected_output": "保留证据",
"enable_outputter": True,
},
}
service.store.save_job(job)
return service.create(FavoriteCreate(source_id=session_id))
def test_snapshot_survives_source_deletion_and_edits_do_not_modify_source(favorites):
item = task(favorites)
updated = favorites.update(
item["id"], FavoriteUpdate(name="常用检查", task_config={"model": "new-model"})
)
assert updated["snapshot"]["runtime_overrides"]["model"] == "new-model"
assert (
favorites.store.job("task")["execution_config"]["runtime_overrides"]["model"]
== "original-model"
)
favorites.store.delete_history()
assert favorites.get(item["id"])["snapshot"]["goal"] == "检查真实页面"
assert favorites.store.job("task") is None
@pytest.mark.asyncio
async def test_rerun_preserves_settings_and_is_durably_idempotent(favorites, monkeypatch):
item = task(favorites)
enqueue = AsyncMock(return_value={"tasks": [{"session_id": "new-task"}], "status": "queued"})
monkeypatch.setattr("apps.admin_console.routers.tasks.run_task", enqueue)
value = FavoriteRun(request_id="same-submission", task_config={"reasoning_effort": "max"})
first, second = await asyncio.gather(
favorites.run(item["id"], value), favorites.run(item["id"], value)
)
assert first == second
enqueue.assert_awaited_once()
request = enqueue.call_args.args[0]
assert request.session_id != "task"
assert request.runtime_overrides.model == "original-model"
assert request.runtime_overrides.reasoning_effort == "max"
assert request.runtime_overrides.record_video is True
assert request.runtime_overrides.device_serial == "original-device"
assert request.expected_output == "保留证据"
assert favorites.get(item["id"])["snapshot"]["runtime_overrides"]["reasoning_effort"] == "high"
favorites.delete(item["id"])
assert await FavoriteService(favorites.store).run(item["id"], value) == first
enqueue.assert_awaited_once()
with pytest.raises(ValueError, match="request_id"):
await favorites.run(
item["id"],
FavoriteRun(request_id="same-submission", task_config={"model": "different"}),
)
@pytest.mark.asyncio
async def test_lost_admission_reply_reuses_snapshot_and_session(favorites, monkeypatch):
item = task(favorites)
enqueue = AsyncMock(side_effect=[ConnectionError("reply lost"), {"tasks": [{}]}])
monkeypatch.setattr("apps.admin_console.routers.tasks.run_task", enqueue)
value = FavoriteRun(request_id="retry")
with pytest.raises(ConnectionError):
await favorites.run(item["id"], value)
favorites.update(item["id"], FavoriteUpdate(task_config={"model": "changed"}))
await FavoriteService(favorites.store).run(item["id"], value)
requests = [call.args[0] for call in enqueue.call_args_list]
assert requests[0].session_id == requests[1].session_id
assert requests[1].runtime_overrides.model == "original-model"
@pytest.mark.asyncio
async def test_batch_favorite_keeps_one_round_and_original_repetitions(favorites):
batch = automation_service.create_batch(
{
"name": "压测",
"repeat_count": 3,
"profile": "flash",
"device_serial": "device",
"cases": [
{"name": "甲", "goal": "检查甲", "task_config": {"model": "model-a"}},
{
"name": "乙",
"goal": "检查乙",
"task_config": {"profile": "pro", "model": "model-b"},
},
],
}
)
item = favorites.create(FavoriteCreate(source_kind="batch", source_id=batch["id"]))
assert len(item["snapshot"]["cases"]) == 2
assert item["snapshot"]["repeat_count"] == 3
result = await favorites.run(
item["id"],
FavoriteRun(
request_id="batch-rerun",
task_config={"reasoning_effort": "max"},
case_configs={1: {"model": "changed-b"}},
),
)
jobs = favorites.store.jobs(result["batch_id"])
assert len(jobs) == 6
assert [j["case"]["name"] for j in jobs] == ["甲", "乙"] * 3
assert [j["case"]["task_config"]["model"] for j in jobs] == ["model-a", "changed-b"] * 3
assert all(j["case"]["task_config"]["reasoning_effort"] == "max" for j in jobs)
assert favorites.store.get("batch", result["batch_id"])["status"] == "running"
again = await favorites.run(
item["id"],
FavoriteRun(
request_id="batch-rerun",
task_config={"reasoning_effort": "max"},
case_configs={1: {"model": "changed-b"}},
),
)
assert again == result
assert len(favorites.store.records("batch")) == 2
def test_exploration_case_pins_version_without_inheriting_expensive_model(favorites):
repo = ExplorationStore(favorites.store)
session = repo.create(
"探索目标",
ExplorationConfig(model="exploration-model", reasoning_effort="max"),
{"model": "exploration-model", "device_serial": "device"},
"create",
)
repo.save_case(
session["id"],
CaseDraft(name="完成用例", goal="验证主页", preconditions="已登录"),
0,
"case-v1",
)
item = favorites.create(FavoriteCreate(source_kind="exploration_case", source_id=session["id"]))
assert item["case_version"] == 1
assert item["snapshot"]["goal"] == "前置条件:\n已登录\n\n验证主页"
assert item["snapshot"]["runtime_overrides"]["profile"] == "flash"
assert item["snapshot"]["runtime_overrides"]["model"] != "exploration-model"
repo.save_case(session["id"], CaseDraft(name="新用例", goal="新目标"), 1, "case-v2")
assert favorites.get(item["id"])["snapshot"]["goal"].endswith("验证主页")
exploration = favorites.create(
FavoriteCreate(source_kind="exploration", source_id=session["id"])
)
assert exploration["snapshot"]["config"]["model"] == "exploration-model"
def test_old_task_recovers_real_model_and_marks_missing_settings(favorites):
with favorites.store.connection() as conn:
conn.execute(
"INSERT INTO sessions(session_id,initial_goal,status,device_info) VALUES (?,?,?,?)",
(
"legacy",
"检查浏览器",
"completed",
json.dumps({"profile": "flash", "serial": "computer:local"}),
),
)
conn.execute(
"INSERT INTO traces(trace_id,session_id,type,timestamp,payload) VALUES (?,?,?,?,?)",
(
"request",
"legacy",
"llm_diagnostic",
1,
json.dumps(
{"role": "FlashRunner", "model": "actual-model", "reasoning_effort": "max"}
),
),
)
conn.commit()
item = favorites.create(FavoriteCreate(source_id="legacy"))
options = item["snapshot"]["runtime_overrides"]
assert options["model"] == "actual-model"
assert options["reasoning_effort"] == "max"
assert options["execution_target"] == "computer"
assert "model" not in item["inherited_fields"]
assert "recording" in item["inherited_fields"]
def test_api_rejects_credentials_and_arbitrary_snapshots(favorites, monkeypatch):
from apps.admin_console.routers import favorites as api
monkeypatch.setattr(api, "favorites", favorites)
app = FastAPI()
app.include_router(api.router)
client = TestClient(app)
item = task(favorites)
assert len(client.get("/api/automation/favorites").json()) == 1
assert (
client.patch(
f"/api/automation/favorites/{item['id']}", json={"task_config": {"api_key": "never"}}
).status_code
== 422
)
assert (
client.post(
"/api/automation/favorites", json={"source_id": "task", "snapshot": {}}
).status_code
== 422
)
assert (
client.patch(f"/api/automation/favorites/{item['id']}", json={"name": " "}).status_code
== 409
)
assert client.delete(f"/api/automation/favorites/{item['id']}").status_code == 200
assert favorites.store.job("task")
with pytest.raises(ValidationError):
FavoriteRun(request_id="run", task_config={"password": "never"})
def test_collection_defaults_cannot_modify_original_snapshot(favorites):
item = task(favorites)
original = deepcopy(item)
favorites.updated(item, FavoriteUpdate(task_config={"model": "temporary"}))
assert item == original
def test_partial_recording_update_keeps_other_favorite_settings(favorites):
item = task(favorites, recording={"codec": "h264", "fps": 10, "max_long_edge": 720})
updated = favorites.updated(item, FavoriteUpdate(task_config={"recording": {"codec": "h265"}}))
assert updated["snapshot"]["runtime_overrides"]["recording"] == {
"codec": "h265",
"fps": 10,
"max_long_edge": 720,
}
assert item["snapshot"]["runtime_overrides"]["recording"]["codec"] == "h264"
def test_legacy_empty_recording_configuration_uses_current_defaults():
from apps.admin_console.services.execution_config import execution_config
from artemis.config.agent import load_agent_config
snapshot = execution_config({"profile": "flash", "runtime_overrides": {"recording": None}})
assert snapshot["runtime_overrides"]["recording"] == load_agent_config().recording.model_dump()
@pytest.mark.asyncio
@pytest.mark.parametrize("rejected", [False, True])
async def test_restart_exploration_creates_new_conversation_and_reports_rejection(
favorites, monkeypatch, rejected
):
from apps.admin_console.services.exploration_service import ExplorationService
repo = ExplorationStore(favorites.store)
original = repo.create(
"探索目标",
ExplorationConfig(model="explore-model", reasoning_effort="max"),
{"device_serial": "device", "record_video": True},
"source",
)
item = favorites.create(FavoriteCreate(source_kind="exploration", source_id=original["id"]))
monkeypatch.setattr(
"apps.admin_console.services.exploration_service.exploration_service",
ExplorationService(repo),
)
enqueue = AsyncMock(
return_value=(
{"status": "rejected", "error": "设备离线"}
if rejected
else {"status": "queued", "tasks": [{}]}
)
)
monkeypatch.setattr("apps.admin_console.routers.tasks.run_task", enqueue)
value = FavoriteRun(request_id="new-exploration", task_config={"model": "selected-model"})
if rejected:
with pytest.raises(ValueError, match="设备离线"):
await favorites.run(item["id"], value)
with pytest.raises(ValueError, match="设备离线"):
await favorites.run(item["id"], value)
else:
result = await favorites.run(item["id"], value)
assert result["exploration_id"] != original["id"]
assert repo.run(result["session_id"])["exploration_id"] == result["exploration_id"]
assert await favorites.run(item["id"], value) == result
request = enqueue.call_args.args[0]
assert request.conversation_id != original["id"]
assert request.runtime_overrides.model == "selected-model"
assert request.runtime_overrides.reasoning_effort == "max"
assert request.runtime_overrides.record_video is True
enqueue.assert_awaited_once()
assert repo.get(original["id"]) == original