"""Favorite snapshots and transport retries never replay an old result or task.""" import asyncio from copy import deepcopy import json from unittest.mock import AsyncMock from fastapi import FastAPI from fastapi.testclient import TestClient from pydantic import ValidationError import pytest from apps.admin_console.services import automation_service from apps.admin_console.services.automation_store import AutomationStore from apps.admin_console.services.favorites import ( FavoriteCreate, FavoriteRun, FavoriteService, FavoriteUpdate, ) from artemis.config.exploration import ExplorationConfig from artemis.data_engine.storage import StorageManager from artemis.exploration.models import CaseDraft from artemis.exploration.store import ExplorationStore @pytest.fixture def favorites(tmp_path, monkeypatch): storage = AutomationStore(tmp_path / "history.db") StorageManager(storage.path, tmp_path / "traces") monkeypatch.setattr(automation_service, "store", storage) return FavoriteService(storage) def task(service, session_id="task", **options): config = { "profile": "pro", "model": "original-model", "reasoning_effort": "high", "device_serial": "original-device", "record_video": True, **options, } job = { "session_id": session_id, "status": "completed", "case": {"name": "原用例", "goal": "检查真实页面"}, "execution_config": { "runtime_overrides": config, "expected_output": "保留证据", "enable_outputter": True, }, } service.store.save_job(job) return service.create(FavoriteCreate(source_id=session_id)) def test_snapshot_survives_source_deletion_and_edits_do_not_modify_source(favorites): item = task(favorites) updated = favorites.update( item["id"], FavoriteUpdate(name="常用检查", task_config={"model": "new-model"}) ) assert updated["snapshot"]["runtime_overrides"]["model"] == "new-model" assert ( favorites.store.job("task")["execution_config"]["runtime_overrides"]["model"] == "original-model" ) favorites.store.delete_history() assert favorites.get(item["id"])["snapshot"]["goal"] == "检查真实页面" assert favorites.store.job("task") is None @pytest.mark.asyncio async def test_rerun_preserves_settings_and_is_durably_idempotent(favorites, monkeypatch): item = task(favorites) enqueue = AsyncMock(return_value={"tasks": [{"session_id": "new-task"}], "status": "queued"}) monkeypatch.setattr("apps.admin_console.routers.tasks.run_task", enqueue) value = FavoriteRun(request_id="same-submission", task_config={"reasoning_effort": "max"}) first, second = await asyncio.gather( favorites.run(item["id"], value), favorites.run(item["id"], value) ) assert first == second enqueue.assert_awaited_once() request = enqueue.call_args.args[0] assert request.session_id != "task" assert request.runtime_overrides.model == "original-model" assert request.runtime_overrides.reasoning_effort == "max" assert request.runtime_overrides.record_video is True assert request.runtime_overrides.device_serial == "original-device" assert request.expected_output == "保留证据" assert favorites.get(item["id"])["snapshot"]["runtime_overrides"]["reasoning_effort"] == "high" favorites.delete(item["id"]) assert await FavoriteService(favorites.store).run(item["id"], value) == first enqueue.assert_awaited_once() with pytest.raises(ValueError, match="request_id"): await favorites.run( item["id"], FavoriteRun(request_id="same-submission", task_config={"model": "different"}), ) @pytest.mark.asyncio async def test_lost_admission_reply_reuses_snapshot_and_session(favorites, monkeypatch): item = task(favorites) enqueue = AsyncMock(side_effect=[ConnectionError("reply lost"), {"tasks": [{}]}]) monkeypatch.setattr("apps.admin_console.routers.tasks.run_task", enqueue) value = FavoriteRun(request_id="retry") with pytest.raises(ConnectionError): await favorites.run(item["id"], value) favorites.update(item["id"], FavoriteUpdate(task_config={"model": "changed"})) await FavoriteService(favorites.store).run(item["id"], value) requests = [call.args[0] for call in enqueue.call_args_list] assert requests[0].session_id == requests[1].session_id assert requests[1].runtime_overrides.model == "original-model" @pytest.mark.asyncio async def test_batch_favorite_keeps_one_round_and_original_repetitions(favorites): batch = automation_service.create_batch( { "name": "压测", "repeat_count": 3, "profile": "flash", "device_serial": "device", "cases": [ {"name": "甲", "goal": "检查甲", "task_config": {"model": "model-a"}}, { "name": "乙", "goal": "检查乙", "task_config": {"profile": "pro", "model": "model-b"}, }, ], } ) item = favorites.create(FavoriteCreate(source_kind="batch", source_id=batch["id"])) assert len(item["snapshot"]["cases"]) == 2 assert item["snapshot"]["repeat_count"] == 3 result = await favorites.run( item["id"], FavoriteRun( request_id="batch-rerun", task_config={"reasoning_effort": "max"}, case_configs={1: {"model": "changed-b"}}, ), ) jobs = favorites.store.jobs(result["batch_id"]) assert len(jobs) == 6 assert [j["case"]["name"] for j in jobs] == ["甲", "乙"] * 3 assert [j["case"]["task_config"]["model"] for j in jobs] == ["model-a", "changed-b"] * 3 assert all(j["case"]["task_config"]["reasoning_effort"] == "max" for j in jobs) assert favorites.store.get("batch", result["batch_id"])["status"] == "running" again = await favorites.run( item["id"], FavoriteRun( request_id="batch-rerun", task_config={"reasoning_effort": "max"}, case_configs={1: {"model": "changed-b"}}, ), ) assert again == result assert len(favorites.store.records("batch")) == 2 def test_exploration_case_pins_version_without_inheriting_expensive_model(favorites): repo = ExplorationStore(favorites.store) session = repo.create( "探索目标", ExplorationConfig(model="exploration-model", reasoning_effort="max"), {"model": "exploration-model", "device_serial": "device"}, "create", ) repo.save_case( session["id"], CaseDraft(name="完成用例", goal="验证主页", preconditions="已登录"), 0, "case-v1", ) item = favorites.create(FavoriteCreate(source_kind="exploration_case", source_id=session["id"])) assert item["case_version"] == 1 assert item["snapshot"]["goal"] == "前置条件：\n已登录\n\n验证主页" assert item["snapshot"]["runtime_overrides"]["profile"] == "flash" assert item["snapshot"]["runtime_overrides"]["model"] != "exploration-model" repo.save_case(session["id"], CaseDraft(name="新用例", goal="新目标"), 1, "case-v2") assert favorites.get(item["id"])["snapshot"]["goal"].endswith("验证主页") exploration = favorites.create( FavoriteCreate(source_kind="exploration", source_id=session["id"]) ) assert exploration["snapshot"]["config"]["model"] == "exploration-model" def test_old_task_recovers_real_model_and_marks_missing_settings(favorites): with favorites.store.connection() as conn: conn.execute( "INSERT INTO sessions(session_id,initial_goal,status,device_info) VALUES (?,?,?,?)", ( "legacy", "检查浏览器", "completed", json.dumps({"profile": "flash", "serial": "computer:local"}), ), ) conn.execute( "INSERT INTO traces(trace_id,session_id,type,timestamp,payload) VALUES (?,?,?,?,?)", ( "request", "legacy", "llm_diagnostic", 1, json.dumps( {"role": "FlashRunner", "model": "actual-model", "reasoning_effort": "max"} ), ), ) conn.commit() item = favorites.create(FavoriteCreate(source_id="legacy")) options = item["snapshot"]["runtime_overrides"] assert options["model"] == "actual-model" assert options["reasoning_effort"] == "max" assert options["execution_target"] == "computer" assert "model" not in item["inherited_fields"] assert "recording" in item["inherited_fields"] def test_api_rejects_credentials_and_arbitrary_snapshots(favorites, monkeypatch): from apps.admin_console.routers import favorites as api monkeypatch.setattr(api, "favorites", favorites) app = FastAPI() app.include_router(api.router) client = TestClient(app) item = task(favorites) assert len(client.get("/api/automation/favorites").json()) == 1 assert ( client.patch( f"/api/automation/favorites/{item['id']}", json={"task_config": {"api_key": "never"}} ).status_code == 422 ) assert ( client.post( "/api/automation/favorites", json={"source_id": "task", "snapshot": {}} ).status_code == 422 ) assert ( client.patch(f"/api/automation/favorites/{item['id']}", json={"name": " "}).status_code == 409 ) assert client.delete(f"/api/automation/favorites/{item['id']}").status_code == 200 assert favorites.store.job("task") with pytest.raises(ValidationError): FavoriteRun(request_id="run", task_config={"password": "never"}) def test_collection_defaults_cannot_modify_original_snapshot(favorites): item = task(favorites) original = deepcopy(item) favorites.updated(item, FavoriteUpdate(task_config={"model": "temporary"})) assert item == original def test_partial_recording_update_keeps_other_favorite_settings(favorites): item = task(favorites, recording={"codec": "h264", "fps": 10, "max_long_edge": 720}) updated = favorites.updated(item, FavoriteUpdate(task_config={"recording": {"codec": "h265"}})) assert updated["snapshot"]["runtime_overrides"]["recording"] == { **item["snapshot"]["runtime_overrides"]["recording"], "codec": "h265", } assert item["snapshot"]["runtime_overrides"]["recording"]["codec"] == "h264" def test_legacy_empty_recording_configuration_uses_current_defaults(): from apps.admin_console.services.execution_config import execution_config from artemis.config.agent import load_agent_config snapshot = execution_config({"profile": "flash", "runtime_overrides": {"recording": None}}) assert snapshot["runtime_overrides"]["recording"] == load_agent_config().recording.model_dump() def test_legacy_favorite_is_completed_once_without_changing_actual_settings(favorites, monkeypatch): from artemis.config.settings import settings item = task(favorites, max_retries=0, record_video=False, recording={"fps": 10}) item.pop("config_version") original = deepcopy(item) favorites.store.put("favorite", item["id"], item) completed = favorites.list()[0] options = completed["snapshot"]["runtime_overrides"] assert options["model"] == "original-model" assert options["max_retries"] == 0 assert options["record_video"] is False assert options["recording"]["fps"] == 10 assert options["recording"]["max_long_edge"] > 0 assert completed["created_at"] == original["created_at"] assert completed["updated_at"] == original["updated_at"] assert favorites.store.get("favorite", item["id"]) == completed monkeypatch.setattr(settings, "OPENAI_MODEL", "new-global-model") assert FavoriteService(favorites.store).get(item["id"]) == completed def test_partial_legacy_snapshot_and_batch_complete_missing_fields(favorites): from apps.admin_console.services.execution_config import execution_config expected = execution_config({})["runtime_overrides"] item = task(favorites) item.pop("config_version") item["snapshot"]["runtime_overrides"] = {"profile": "pro", "model": "kept-model"} favorites.store.put("favorite", item["id"], item) options = favorites.get(item["id"])["snapshot"]["runtime_overrides"] assert options["model"] == "kept-model" assert options["reasoning_effort"] == expected["reasoning_effort"] assert options["recording"] == expected["recording"] item["id"] = "legacy-batch" item["kind"] = "batch" item["snapshot"] = { "profile": "pro", "device_serial": "device", "cases": [ { "name": "一", "goal": "检查", "task_config": {"model": "case-model", "record_video": False}, } ], } favorites.store.put("favorite", item["id"], item) options = favorites.get(item["id"])["snapshot"]["cases"][0]["task_config"] assert options["model"] == "case-model" assert options["profile"] == "pro" assert options["device_serial"] == "device" assert options["record_video"] is False assert options["request_timeout_seconds"] == expected["request_timeout_seconds"] @pytest.mark.asyncio @pytest.mark.parametrize("rejected", [False, True]) async def test_restart_exploration_creates_new_conversation_and_reports_rejection( favorites, monkeypatch, rejected ): from apps.admin_console.services.exploration_service import ExplorationService repo = ExplorationStore(favorites.store) original = repo.create( "探索目标", ExplorationConfig(model="explore-model", reasoning_effort="max"), {"device_serial": "device", "record_video": True}, "source", ) item = favorites.create(FavoriteCreate(source_kind="exploration", source_id=original["id"])) monkeypatch.setattr( "apps.admin_console.services.exploration_service.exploration_service", ExplorationService(repo), ) enqueue = AsyncMock( return_value=( {"status": "rejected", "error": "设备离线"} if rejected else {"status": "queued", "tasks": [{}]} ) ) monkeypatch.setattr("apps.admin_console.routers.tasks.run_task", enqueue) value = FavoriteRun(request_id="new-exploration", task_config={"model": "selected-model"}) if rejected: with pytest.raises(ValueError, match="设备离线"): await favorites.run(item["id"], value) with pytest.raises(ValueError, match="设备离线"): await favorites.run(item["id"], value) else: result = await favorites.run(item["id"], value) assert result["exploration_id"] != original["id"] assert repo.run(result["session_id"])["exploration_id"] == result["exploration_id"] assert await favorites.run(item["id"], value) == result request = enqueue.call_args.args[0] assert request.conversation_id != original["id"] assert request.runtime_overrides.model == "selected-model" assert request.runtime_overrides.reasoning_effort == "max" assert request.runtime_overrides.record_video is True enqueue.assert_awaited_once() assert repo.get(original["id"]) == original