380 lines
15 KiB
Python
380 lines
15 KiB
Python
"""Favorite snapshots and transport retries never replay an old result or task."""
|
|
|
|
import asyncio
|
|
from copy import deepcopy
|
|
import json
|
|
from unittest.mock import AsyncMock
|
|
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
from pydantic import ValidationError
|
|
import pytest
|
|
|
|
from apps.admin_console.services import automation_service
|
|
from apps.admin_console.services.automation_store import AutomationStore
|
|
from apps.admin_console.services.favorites import (
|
|
FavoriteCreate,
|
|
FavoriteRun,
|
|
FavoriteService,
|
|
FavoriteUpdate,
|
|
)
|
|
from artemis.config.exploration import ExplorationConfig
|
|
from artemis.data_engine.storage import StorageManager
|
|
from artemis.exploration.models import CaseDraft
|
|
from artemis.exploration.store import ExplorationStore
|
|
|
|
|
|
@pytest.fixture
|
|
def favorites(tmp_path, monkeypatch):
|
|
storage = AutomationStore(tmp_path / "history.db")
|
|
StorageManager(storage.path, tmp_path / "traces")
|
|
monkeypatch.setattr(automation_service, "store", storage)
|
|
return FavoriteService(storage)
|
|
|
|
|
|
def task(service, session_id="task", **options):
|
|
config = {
|
|
"profile": "pro",
|
|
"model": "original-model",
|
|
"reasoning_effort": "high",
|
|
"device_serial": "original-device",
|
|
"record_video": True,
|
|
**options,
|
|
}
|
|
job = {
|
|
"session_id": session_id,
|
|
"status": "completed",
|
|
"case": {"name": "原用例", "goal": "检查真实页面"},
|
|
"execution_config": {
|
|
"runtime_overrides": config,
|
|
"expected_output": "保留证据",
|
|
"enable_outputter": True,
|
|
},
|
|
}
|
|
service.store.save_job(job)
|
|
return service.create(FavoriteCreate(source_id=session_id))
|
|
|
|
|
|
def test_snapshot_survives_source_deletion_and_edits_do_not_modify_source(favorites):
|
|
item = task(favorites)
|
|
updated = favorites.update(
|
|
item["id"], FavoriteUpdate(name="常用检查", task_config={"model": "new-model"})
|
|
)
|
|
assert updated["snapshot"]["runtime_overrides"]["model"] == "new-model"
|
|
assert (
|
|
favorites.store.job("task")["execution_config"]["runtime_overrides"]["model"]
|
|
== "original-model"
|
|
)
|
|
favorites.store.delete_history()
|
|
assert favorites.get(item["id"])["snapshot"]["goal"] == "检查真实页面"
|
|
assert favorites.store.job("task") is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_rerun_preserves_settings_and_is_durably_idempotent(favorites, monkeypatch):
|
|
item = task(favorites)
|
|
enqueue = AsyncMock(return_value={"tasks": [{"session_id": "new-task"}], "status": "queued"})
|
|
monkeypatch.setattr("apps.admin_console.routers.tasks.run_task", enqueue)
|
|
value = FavoriteRun(request_id="same-submission", task_config={"reasoning_effort": "max"})
|
|
first, second = await asyncio.gather(
|
|
favorites.run(item["id"], value), favorites.run(item["id"], value)
|
|
)
|
|
assert first == second
|
|
enqueue.assert_awaited_once()
|
|
request = enqueue.call_args.args[0]
|
|
assert request.session_id != "task"
|
|
assert request.runtime_overrides.model == "original-model"
|
|
assert request.runtime_overrides.reasoning_effort == "max"
|
|
assert request.runtime_overrides.record_video is True
|
|
assert request.runtime_overrides.device_serial == "original-device"
|
|
assert request.expected_output == "保留证据"
|
|
assert favorites.get(item["id"])["snapshot"]["runtime_overrides"]["reasoning_effort"] == "high"
|
|
favorites.delete(item["id"])
|
|
assert await FavoriteService(favorites.store).run(item["id"], value) == first
|
|
enqueue.assert_awaited_once()
|
|
with pytest.raises(ValueError, match="request_id"):
|
|
await favorites.run(
|
|
item["id"],
|
|
FavoriteRun(request_id="same-submission", task_config={"model": "different"}),
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_lost_admission_reply_reuses_snapshot_and_session(favorites, monkeypatch):
|
|
item = task(favorites)
|
|
enqueue = AsyncMock(side_effect=[ConnectionError("reply lost"), {"tasks": [{}]}])
|
|
monkeypatch.setattr("apps.admin_console.routers.tasks.run_task", enqueue)
|
|
value = FavoriteRun(request_id="retry")
|
|
with pytest.raises(ConnectionError):
|
|
await favorites.run(item["id"], value)
|
|
favorites.update(item["id"], FavoriteUpdate(task_config={"model": "changed"}))
|
|
await FavoriteService(favorites.store).run(item["id"], value)
|
|
requests = [call.args[0] for call in enqueue.call_args_list]
|
|
assert requests[0].session_id == requests[1].session_id
|
|
assert requests[1].runtime_overrides.model == "original-model"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_batch_favorite_keeps_one_round_and_original_repetitions(favorites):
|
|
batch = automation_service.create_batch(
|
|
{
|
|
"name": "压测",
|
|
"repeat_count": 3,
|
|
"profile": "flash",
|
|
"device_serial": "device",
|
|
"cases": [
|
|
{"name": "甲", "goal": "检查甲", "task_config": {"model": "model-a"}},
|
|
{
|
|
"name": "乙",
|
|
"goal": "检查乙",
|
|
"task_config": {"profile": "pro", "model": "model-b"},
|
|
},
|
|
],
|
|
}
|
|
)
|
|
item = favorites.create(FavoriteCreate(source_kind="batch", source_id=batch["id"]))
|
|
assert len(item["snapshot"]["cases"]) == 2
|
|
assert item["snapshot"]["repeat_count"] == 3
|
|
result = await favorites.run(
|
|
item["id"],
|
|
FavoriteRun(
|
|
request_id="batch-rerun",
|
|
task_config={"reasoning_effort": "max"},
|
|
case_configs={1: {"model": "changed-b"}},
|
|
),
|
|
)
|
|
jobs = favorites.store.jobs(result["batch_id"])
|
|
assert len(jobs) == 6
|
|
assert [j["case"]["name"] for j in jobs] == ["甲", "乙"] * 3
|
|
assert [j["case"]["task_config"]["model"] for j in jobs] == ["model-a", "changed-b"] * 3
|
|
assert all(j["case"]["task_config"]["reasoning_effort"] == "max" for j in jobs)
|
|
assert favorites.store.get("batch", result["batch_id"])["status"] == "running"
|
|
again = await favorites.run(
|
|
item["id"],
|
|
FavoriteRun(
|
|
request_id="batch-rerun",
|
|
task_config={"reasoning_effort": "max"},
|
|
case_configs={1: {"model": "changed-b"}},
|
|
),
|
|
)
|
|
assert again == result
|
|
assert len(favorites.store.records("batch")) == 2
|
|
|
|
|
|
def test_exploration_case_pins_version_without_inheriting_expensive_model(favorites):
|
|
repo = ExplorationStore(favorites.store)
|
|
session = repo.create(
|
|
"探索目标",
|
|
ExplorationConfig(model="exploration-model", reasoning_effort="max"),
|
|
{"model": "exploration-model", "device_serial": "device"},
|
|
"create",
|
|
)
|
|
repo.save_case(
|
|
session["id"],
|
|
CaseDraft(name="完成用例", goal="验证主页", preconditions="已登录"),
|
|
0,
|
|
"case-v1",
|
|
)
|
|
item = favorites.create(FavoriteCreate(source_kind="exploration_case", source_id=session["id"]))
|
|
assert item["case_version"] == 1
|
|
assert item["snapshot"]["goal"] == "前置条件：\n已登录\n\n验证主页"
|
|
assert item["snapshot"]["runtime_overrides"]["profile"] == "flash"
|
|
assert item["snapshot"]["runtime_overrides"]["model"] != "exploration-model"
|
|
repo.save_case(session["id"], CaseDraft(name="新用例", goal="新目标"), 1, "case-v2")
|
|
assert favorites.get(item["id"])["snapshot"]["goal"].endswith("验证主页")
|
|
exploration = favorites.create(
|
|
FavoriteCreate(source_kind="exploration", source_id=session["id"])
|
|
)
|
|
assert exploration["snapshot"]["config"]["model"] == "exploration-model"
|
|
|
|
|
|
def test_old_task_recovers_real_model_and_marks_missing_settings(favorites):
|
|
with favorites.store.connection() as conn:
|
|
conn.execute(
|
|
"INSERT INTO sessions(session_id,initial_goal,status,device_info) VALUES (?,?,?,?)",
|
|
(
|
|
"legacy",
|
|
"检查浏览器",
|
|
"completed",
|
|
json.dumps({"profile": "flash", "serial": "computer:local"}),
|
|
),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO traces(trace_id,session_id,type,timestamp,payload) VALUES (?,?,?,?,?)",
|
|
(
|
|
"request",
|
|
"legacy",
|
|
"llm_diagnostic",
|
|
1,
|
|
json.dumps(
|
|
{"role": "FlashRunner", "model": "actual-model", "reasoning_effort": "max"}
|
|
),
|
|
),
|
|
)
|
|
conn.commit()
|
|
item = favorites.create(FavoriteCreate(source_id="legacy"))
|
|
options = item["snapshot"]["runtime_overrides"]
|
|
assert options["model"] == "actual-model"
|
|
assert options["reasoning_effort"] == "max"
|
|
assert options["execution_target"] == "computer"
|
|
assert "model" not in item["inherited_fields"]
|
|
assert "recording" in item["inherited_fields"]
|
|
|
|
|
|
def test_api_rejects_credentials_and_arbitrary_snapshots(favorites, monkeypatch):
|
|
from apps.admin_console.routers import favorites as api
|
|
|
|
monkeypatch.setattr(api, "favorites", favorites)
|
|
app = FastAPI()
|
|
app.include_router(api.router)
|
|
client = TestClient(app)
|
|
item = task(favorites)
|
|
assert len(client.get("/api/automation/favorites").json()) == 1
|
|
assert (
|
|
client.patch(
|
|
f"/api/automation/favorites/{item['id']}", json={"task_config": {"api_key": "never"}}
|
|
).status_code
|
|
== 422
|
|
)
|
|
assert (
|
|
client.post(
|
|
"/api/automation/favorites", json={"source_id": "task", "snapshot": {}}
|
|
).status_code
|
|
== 422
|
|
)
|
|
assert (
|
|
client.patch(f"/api/automation/favorites/{item['id']}", json={"name": " "}).status_code
|
|
== 409
|
|
)
|
|
assert client.delete(f"/api/automation/favorites/{item['id']}").status_code == 200
|
|
assert favorites.store.job("task")
|
|
with pytest.raises(ValidationError):
|
|
FavoriteRun(request_id="run", task_config={"password": "never"})
|
|
|
|
|
|
def test_collection_defaults_cannot_modify_original_snapshot(favorites):
|
|
item = task(favorites)
|
|
original = deepcopy(item)
|
|
favorites.updated(item, FavoriteUpdate(task_config={"model": "temporary"}))
|
|
assert item == original
|
|
|
|
|
|
def test_partial_recording_update_keeps_other_favorite_settings(favorites):
|
|
item = task(favorites, recording={"codec": "h264", "fps": 10, "max_long_edge": 720})
|
|
updated = favorites.updated(item, FavoriteUpdate(task_config={"recording": {"codec": "h265"}}))
|
|
assert updated["snapshot"]["runtime_overrides"]["recording"] == {
|
|
**item["snapshot"]["runtime_overrides"]["recording"],
|
|
"codec": "h265",
|
|
}
|
|
assert item["snapshot"]["runtime_overrides"]["recording"]["codec"] == "h264"
|
|
|
|
|
|
def test_legacy_empty_recording_configuration_uses_current_defaults():
|
|
from apps.admin_console.services.execution_config import execution_config
|
|
from artemis.config.agent import load_agent_config
|
|
|
|
snapshot = execution_config({"profile": "flash", "runtime_overrides": {"recording": None}})
|
|
assert snapshot["runtime_overrides"]["recording"] == load_agent_config().recording.model_dump()
|
|
|
|
|
|
def test_legacy_favorite_is_completed_once_without_changing_actual_settings(favorites, monkeypatch):
|
|
from artemis.config.settings import settings
|
|
|
|
item = task(favorites, max_retries=0, record_video=False, recording={"fps": 10})
|
|
item.pop("config_version")
|
|
original = deepcopy(item)
|
|
favorites.store.put("favorite", item["id"], item)
|
|
completed = favorites.list()[0]
|
|
options = completed["snapshot"]["runtime_overrides"]
|
|
assert options["model"] == "original-model"
|
|
assert options["max_retries"] == 0
|
|
assert options["record_video"] is False
|
|
assert options["recording"]["fps"] == 10
|
|
assert options["recording"]["max_long_edge"] > 0
|
|
assert completed["created_at"] == original["created_at"]
|
|
assert completed["updated_at"] == original["updated_at"]
|
|
assert favorites.store.get("favorite", item["id"]) == completed
|
|
monkeypatch.setattr(settings, "OPENAI_MODEL", "new-global-model")
|
|
assert FavoriteService(favorites.store).get(item["id"]) == completed
|
|
|
|
|
|
def test_partial_legacy_snapshot_and_batch_complete_missing_fields(favorites):
|
|
from apps.admin_console.services.execution_config import execution_config
|
|
|
|
expected = execution_config({})["runtime_overrides"]
|
|
item = task(favorites)
|
|
item.pop("config_version")
|
|
item["snapshot"]["runtime_overrides"] = {"profile": "pro", "model": "kept-model"}
|
|
favorites.store.put("favorite", item["id"], item)
|
|
options = favorites.get(item["id"])["snapshot"]["runtime_overrides"]
|
|
assert options["model"] == "kept-model"
|
|
assert options["reasoning_effort"] == expected["reasoning_effort"]
|
|
assert options["recording"] == expected["recording"]
|
|
item["id"] = "legacy-batch"
|
|
item["kind"] = "batch"
|
|
item["snapshot"] = {
|
|
"profile": "pro",
|
|
"device_serial": "device",
|
|
"cases": [
|
|
{
|
|
"name": "一",
|
|
"goal": "检查",
|
|
"task_config": {"model": "case-model", "record_video": False},
|
|
}
|
|
],
|
|
}
|
|
favorites.store.put("favorite", item["id"], item)
|
|
options = favorites.get(item["id"])["snapshot"]["cases"][0]["task_config"]
|
|
assert options["model"] == "case-model"
|
|
assert options["profile"] == "pro"
|
|
assert options["device_serial"] == "device"
|
|
assert options["record_video"] is False
|
|
assert options["request_timeout_seconds"] == expected["request_timeout_seconds"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("rejected", [False, True])
|
|
async def test_restart_exploration_creates_new_conversation_and_reports_rejection(
|
|
favorites, monkeypatch, rejected
|
|
):
|
|
from apps.admin_console.services.exploration_service import ExplorationService
|
|
|
|
repo = ExplorationStore(favorites.store)
|
|
original = repo.create(
|
|
"探索目标",
|
|
ExplorationConfig(model="explore-model", reasoning_effort="max"),
|
|
{"device_serial": "device", "record_video": True},
|
|
"source",
|
|
)
|
|
item = favorites.create(FavoriteCreate(source_kind="exploration", source_id=original["id"]))
|
|
monkeypatch.setattr(
|
|
"apps.admin_console.services.exploration_service.exploration_service",
|
|
ExplorationService(repo),
|
|
)
|
|
enqueue = AsyncMock(
|
|
return_value=(
|
|
{"status": "rejected", "error": "设备离线"}
|
|
if rejected
|
|
else {"status": "queued", "tasks": [{}]}
|
|
)
|
|
)
|
|
monkeypatch.setattr("apps.admin_console.routers.tasks.run_task", enqueue)
|
|
value = FavoriteRun(request_id="new-exploration", task_config={"model": "selected-model"})
|
|
if rejected:
|
|
with pytest.raises(ValueError, match="设备离线"):
|
|
await favorites.run(item["id"], value)
|
|
with pytest.raises(ValueError, match="设备离线"):
|
|
await favorites.run(item["id"], value)
|
|
else:
|
|
result = await favorites.run(item["id"], value)
|
|
assert result["exploration_id"] != original["id"]
|
|
assert repo.run(result["session_id"])["exploration_id"] == result["exploration_id"]
|
|
assert await favorites.run(item["id"], value) == result
|
|
request = enqueue.call_args.args[0]
|
|
assert request.conversation_id != original["id"]
|
|
assert request.runtime_overrides.model == "selected-model"
|
|
assert request.runtime_overrides.reasoning_effort == "max"
|
|
assert request.runtime_overrides.record_video is True
|
|
enqueue.assert_awaited_once()
|
|
assert repo.get(original["id"]) == original
|