import asyncio import json from types import SimpleNamespace from unittest.mock import AsyncMock, Mock import pytest from apps.admin_console.services.automation_store import AutomationStore from artemis.config.exploration import ExplorationConfig from artemis.exploration.models import CaseDraft from artemis.exploration.store import ExplorationStore, ExplorationConflict from artemis.exploration.synthesis import generate_case from artemis.runtime import task_control @pytest.mark.asyncio async def test_editorial_never_consumes_device_guidance_or_restarts_recording(tmp_path): from artemis.exploration.synthesis import _invoke_editorial directory = tmp_path / "instructions" directory.mkdir() pending = directory / "user.json" pending.write_text('{"instruction":"return home"}', encoding="utf-8") control = SimpleNamespace( directory=tmp_path, checkpoint=AsyncMock(), run_interruptibly=AsyncMock() ) ctx = SimpleNamespace(task_control=control, on_task_resume=AsyncMock()) model = SimpleNamespace(ainvoke=AsyncMock(return_value="case")) assert await _invoke_editorial(ctx, model, []) == "case" assert pending.exists() control.checkpoint.assert_not_called() control.run_interruptibly.assert_not_called() ctx.on_task_resume.assert_not_called() @pytest.fixture def synthesis(tmp_path, monkeypatch): monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path / "traces") repository = ExplorationStore(AutomationStore(tmp_path / "history.db")) session = repository.create("打开设置并确认状态", ExplorationConfig(), {}, "create") run = repository.reserve_run( session["id"], "start", kind="exploration", goal="探索", task_config={} ) outcome = { "status": "completed", "test_result": {"status": "passed", "reason": "状态正确", "evidence": ["步骤3设置开启"]}, } repository.finish_run(run["id"], outcome) root = task_control.control_dir(run["id"]) root.mkdir(parents=True) (root / "exploration-evidence.json").write_text( json.dumps( { "plan": "已完成设置检查", "outcome": outcome, "steps": [{"step_number": 3, "summary": "设置开启"}], } ), encoding="utf-8", ) model = Mock() model.with_structured_output.return_value.ainvoke = AsyncMock( return_value=CaseDraft( name="设置检查", goal="打开设置,验证开关为开启。", evidence=["步骤3"] ) ) monkeypatch.setattr("artemis.exploration.synthesis.get_llm", Mock(return_value=model)) return repository, session, run, model @pytest.mark.asyncio async def test_generation_is_idempotent_and_preserves_rerun_independence(synthesis): repository, session, run, model = synthesis ctx = SimpleNamespace(task_control=None) result = await generate_case(repository, session["id"], "generate", run["id"], 0, ctx=ctx) duplicate = await generate_case(repository, session["id"], "generate", run["id"], 0, ctx=ctx) assert result == duplicate assert result["version"] == 1 assert result["source_run_id"] == run["id"] assert "validation_status" not in result assert len(repository.runs(session["id"])) == 1 model.with_structured_output.return_value.ainvoke.assert_awaited_once() @pytest.mark.asyncio async def test_synthesis_repairs_unbound_parameters_once_with_original_evidence(synthesis): repository, session, run, model = synthesis invoke = model.with_structured_output.return_value.ainvoke invoke.side_effect = [ CaseDraft(name="错误草稿", goal="访问 ${首页URL}", preconditions="已指定 ${报告位置}"), CaseDraft(name="修订", goal="核对设置,报告保存在本次任务 trace。", evidence=["步骤3"]), ] result = await generate_case( repository, session["id"], "generate", run["id"], 0, ctx=SimpleNamespace(task_control=None) ) assert result["version"] == 1 and "${" not in result["goal"] assert invoke.await_count == 2 messages = invoke.await_args.args[0] assert "不生成 ${名称}" in messages[0].content assert "用户明确指定的位置必须保留" in messages[0].content assert "不要让一次复跑递归地产生下一份复跑用例" in messages[0].content assert "保留该业务验收" in messages[0].content assert "设置开启" in messages[1].content assert "不得猜测未知业务输入或降低验收标准" in messages[-1].content @pytest.mark.asyncio async def test_failed_parameter_repair_never_saves_unexecutable_generated_case(synthesis): repository, session, run, model = synthesis repository.save_case(session["id"], CaseDraft(name="原用例", goal="核对设置"), 0, "save") invoke = model.with_structured_output.return_value.ainvoke invoke.return_value = CaseDraft(name="错误", goal="保存到 ${报告位置}") with pytest.raises(ValueError, match="未填写参数"): await generate_case( repository, session["id"], "generate", run["id"], 1, ctx=SimpleNamespace(task_control=None), ) assert invoke.await_count == 2 assert repository.case(session["id"])["version"] == 1 assert repository.case(session["id"])["name"] == "原用例" assert repository.database.records("exploration_generation")[0]["status"] == "failed" assert repository.run(run["id"])["outcome"]["test_result"]["status"] == "passed" @pytest.mark.asyncio async def test_model_generation_cannot_overwrite_a_concurrent_manual_revision(synthesis): repository, session, run, model = synthesis started, finish = asyncio.Event(), asyncio.Event() async def generate(_): started.set() await finish.wait() return CaseDraft(name="模型", goal="模型改动") model.with_structured_output.return_value.ainvoke = generate pending = asyncio.create_task( generate_case( repository, session["id"], "generate", run["id"], 0, ctx=SimpleNamespace(task_control=None), ) ) await started.wait() repository.save_case(session["id"], CaseDraft(name="用户", goal="用户修订"), 0, "manual") finish.set() with pytest.raises(ExplorationConflict): await pending assert repository.case(session["id"])["goal"] == "用户修订" assert repository.database.records("exploration_generation")[0]["status"] == "failed" @pytest.mark.asyncio async def test_interrupted_execution_cannot_generate_a_verified_case(synthesis): repository, session, run, model = synthesis stored = repository.run(run["id"]) stored["outcome"]["status"] = "interrupted" repository.database.put("exploration_run", run["id"], stored) with pytest.raises(ExplorationConflict, match="尚未验证通过"): await generate_case( repository, session["id"], "generate", run["id"], 0, ctx=SimpleNamespace(task_control=None), ) model.with_structured_output.return_value.ainvoke.assert_not_awaited() @pytest.mark.asyncio async def test_synthesis_failure_leaves_execution_verdict_unchanged(synthesis): repository, session, run, model = synthesis model.with_structured_output.return_value.ainvoke = AsyncMock( side_effect=ValueError("malformed response") ) with pytest.raises(ValueError): await generate_case( repository, session["id"], "generate", run["id"], 0, ctx=SimpleNamespace(task_control=None), ) assert repository.run(run["id"])["outcome"]["test_result"]["status"] == "passed" assert repository.case(session["id"]) is None @pytest.mark.asyncio async def test_context_initialization_failure_marks_generation_retryable(synthesis, monkeypatch): repository, session, run, model = synthesis monkeypatch.setattr( "artemis.exploration.synthesis.get_default_llm_config", Mock(side_effect=ValueError("bad config")), ) with pytest.raises(ValueError): await generate_case(repository, session["id"], "generate", run["id"], 0) assert repository.database.records("exploration_generation")[0]["status"] == "failed" @pytest.mark.asyncio async def test_long_path_compaction_covers_beginning_and_end(monkeypatch): from artemis.exploration.synthesis import _complete_path, PathSummary seen = [] async def invoke(messages): seen.append(messages[-1].content) return PathSummary(verified_path="保留这一段路径及证据") model = Mock() model.with_structured_output.return_value.ainvoke = invoke monkeypatch.setattr("artemis.exploration.synthesis.get_llm", Mock(return_value=model)) await _complete_path( SimpleNamespace(task_control=None), [{"step_number": i, "summary": "状态" * 200} for i in range(250)], ) assert '"step_number": 0,' in seen[0] assert '"step_number": 249,' in seen[-1] @pytest.mark.asyncio async def test_expired_generation_cannot_save_after_recovery(synthesis): repository, session, run, model = synthesis async def generate(_): job = repository.database.records("exploration_generation")[0] repository.database.put("exploration_generation", job["id"], {**job, "status": "failed"}) return CaseDraft(name="过期生成", goal="不应写入") model.with_structured_output.return_value.ainvoke = generate with pytest.raises(ExplorationConflict): await generate_case( repository, session["id"], "generate", run["id"], 0, ctx=SimpleNamespace(task_control=None), ) assert repository.case(session["id"]) is None