Artemis-Eapil/tests/unit/test_exploration_synthesis.py

import asyncio
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock

import pytest

from apps.admin_console.services.automation_store import AutomationStore
from artemis.config.exploration import ExplorationConfig
from artemis.exploration.models import CaseDraft
from artemis.exploration.store import ExplorationStore, ExplorationConflict
from artemis.exploration.synthesis import generate_case
from artemis.runtime import task_control


@pytest.mark.asyncio
async def test_editorial_never_consumes_device_guidance_or_restarts_recording(tmp_path):
    from artemis.exploration.synthesis import _invoke_editorial

    directory = tmp_path / "instructions"
    directory.mkdir()
    pending = directory / "user.json"
    pending.write_text('{"instruction":"return home"}', encoding="utf-8")
    control = SimpleNamespace(
        directory=tmp_path, checkpoint=AsyncMock(), run_interruptibly=AsyncMock()
    )
    ctx = SimpleNamespace(task_control=control, on_task_resume=AsyncMock())
    model = SimpleNamespace(ainvoke=AsyncMock(return_value="case"))
    assert await _invoke_editorial(ctx, model, []) == "case"
    assert pending.exists()
    control.checkpoint.assert_not_called()
    control.run_interruptibly.assert_not_called()
    ctx.on_task_resume.assert_not_called()


@pytest.fixture
def synthesis(tmp_path, monkeypatch):
    monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path / "traces")
    repository = ExplorationStore(AutomationStore(tmp_path / "history.db"))
    session = repository.create("打开设置并确认状态", ExplorationConfig(), {}, "create")
    run = repository.reserve_run(
        session["id"], "start", kind="exploration", goal="探索", task_config={}
    )
    outcome = {
        "status": "completed",
        "test_result": {"status": "passed", "reason": "状态正确", "evidence": ["步骤3设置开启"]},
    }
    repository.finish_run(run["id"], outcome)
    root = task_control.control_dir(run["id"])
    root.mkdir(parents=True)
    (root / "exploration-evidence.json").write_text(
        json.dumps(
            {
                "plan": "已完成设置检查",
                "outcome": outcome,
                "steps": [{"step_number": 3, "summary": "设置开启"}],
            }
        ),
        encoding="utf-8",
    )
    model = Mock()
    model.with_structured_output.return_value.ainvoke = AsyncMock(
        return_value=CaseDraft(
            name="设置检查", goal="打开设置，验证开关为开启。", evidence=["步骤3"]
        )
    )
    monkeypatch.setattr("artemis.exploration.synthesis.get_llm", Mock(return_value=model))
    return repository, session, run, model


@pytest.mark.asyncio
async def test_generation_is_idempotent_and_preserves_rerun_independence(synthesis):
    repository, session, run, model = synthesis
    ctx = SimpleNamespace(task_control=None)
    result = await generate_case(repository, session["id"], "generate", run["id"], 0, ctx=ctx)
    duplicate = await generate_case(repository, session["id"], "generate", run["id"], 0, ctx=ctx)
    assert result == duplicate
    assert result["version"] == 1
    assert result["source_run_id"] == run["id"]
    assert "validation_status" not in result
    assert len(repository.runs(session["id"])) == 1
    model.with_structured_output.return_value.ainvoke.assert_awaited_once()


@pytest.mark.asyncio
async def test_synthesis_repairs_unbound_parameters_once_with_original_evidence(synthesis):
    repository, session, run, model = synthesis
    invoke = model.with_structured_output.return_value.ainvoke
    invoke.side_effect = [
        CaseDraft(name="错误草稿", goal="访问 ${首页URL}", preconditions="已指定 ${报告位置}"),
        CaseDraft(name="修订", goal="核对设置，报告保存在本次任务 trace。", evidence=["步骤3"]),
    ]
    result = await generate_case(
        repository, session["id"], "generate", run["id"], 0, ctx=SimpleNamespace(task_control=None)
    )
    assert result["version"] == 1 and "${" not in result["goal"]
    assert invoke.await_count == 2
    messages = invoke.await_args.args[0]
    assert "不生成 ${名称}" in messages[0].content
    assert "用户明确指定的位置必须保留" in messages[0].content
    assert "不要让一次复跑递归地产生下一份复跑用例" in messages[0].content
    assert "保留该业务验收" in messages[0].content
    assert "设置开启" in messages[1].content
    assert "不得猜测未知业务输入或降低验收标准" in messages[-1].content


@pytest.mark.asyncio
async def test_failed_parameter_repair_never_saves_unexecutable_generated_case(synthesis):
    repository, session, run, model = synthesis
    repository.save_case(session["id"], CaseDraft(name="原用例", goal="核对设置"), 0, "save")
    invoke = model.with_structured_output.return_value.ainvoke
    invoke.return_value = CaseDraft(name="错误", goal="保存到 ${报告位置}")
    with pytest.raises(ValueError, match="未填写参数"):
        await generate_case(
            repository,
            session["id"],
            "generate",
            run["id"],
            1,
            ctx=SimpleNamespace(task_control=None),
        )
    assert invoke.await_count == 2
    assert repository.case(session["id"])["version"] == 1
    assert repository.case(session["id"])["name"] == "原用例"
    assert repository.database.records("exploration_generation")[0]["status"] == "failed"
    assert repository.run(run["id"])["outcome"]["test_result"]["status"] == "passed"


@pytest.mark.asyncio
async def test_model_generation_cannot_overwrite_a_concurrent_manual_revision(synthesis):
    repository, session, run, model = synthesis
    started, finish = asyncio.Event(), asyncio.Event()

    async def generate(_):
        started.set()
        await finish.wait()
        return CaseDraft(name="模型", goal="模型改动")

    model.with_structured_output.return_value.ainvoke = generate
    pending = asyncio.create_task(
        generate_case(
            repository,
            session["id"],
            "generate",
            run["id"],
            0,
            ctx=SimpleNamespace(task_control=None),
        )
    )
    await started.wait()
    repository.save_case(session["id"], CaseDraft(name="用户", goal="用户修订"), 0, "manual")
    finish.set()
    with pytest.raises(ExplorationConflict):
        await pending
    assert repository.case(session["id"])["goal"] == "用户修订"
    assert repository.database.records("exploration_generation")[0]["status"] == "failed"


@pytest.mark.asyncio
async def test_interrupted_execution_cannot_generate_a_verified_case(synthesis):
    repository, session, run, model = synthesis
    stored = repository.run(run["id"])
    stored["outcome"]["status"] = "interrupted"
    repository.database.put("exploration_run", run["id"], stored)
    with pytest.raises(ExplorationConflict, match="尚未验证通过"):
        await generate_case(
            repository,
            session["id"],
            "generate",
            run["id"],
            0,
            ctx=SimpleNamespace(task_control=None),
        )
    model.with_structured_output.return_value.ainvoke.assert_not_awaited()


@pytest.mark.asyncio
async def test_synthesis_failure_leaves_execution_verdict_unchanged(synthesis):
    repository, session, run, model = synthesis
    model.with_structured_output.return_value.ainvoke = AsyncMock(
        side_effect=ValueError("malformed response")
    )
    with pytest.raises(ValueError):
        await generate_case(
            repository,
            session["id"],
            "generate",
            run["id"],
            0,
            ctx=SimpleNamespace(task_control=None),
        )
    assert repository.run(run["id"])["outcome"]["test_result"]["status"] == "passed"
    assert repository.case(session["id"]) is None


@pytest.mark.asyncio
async def test_context_initialization_failure_marks_generation_retryable(synthesis, monkeypatch):
    repository, session, run, model = synthesis
    monkeypatch.setattr(
        "artemis.exploration.synthesis.get_default_llm_config",
        Mock(side_effect=ValueError("bad config")),
    )
    with pytest.raises(ValueError):
        await generate_case(repository, session["id"], "generate", run["id"], 0)
    assert repository.database.records("exploration_generation")[0]["status"] == "failed"


@pytest.mark.asyncio
async def test_long_path_compaction_covers_beginning_and_end(monkeypatch):
    from artemis.exploration.synthesis import _complete_path, PathSummary

    seen = []

    async def invoke(messages):
        seen.append(messages[-1].content)
        return PathSummary(verified_path="保留这一段路径及证据")

    model = Mock()
    model.with_structured_output.return_value.ainvoke = invoke
    monkeypatch.setattr("artemis.exploration.synthesis.get_llm", Mock(return_value=model))
    await _complete_path(
        SimpleNamespace(task_control=None),
        [{"step_number": i, "summary": "状态" * 200} for i in range(250)],
    )
    assert '"step_number": 0,' in seen[0]
    assert '"step_number": 249,' in seen[-1]


@pytest.mark.asyncio
async def test_expired_generation_cannot_save_after_recovery(synthesis):
    repository, session, run, model = synthesis

    async def generate(_):
        job = repository.database.records("exploration_generation")[0]
        repository.database.put("exploration_generation", job["id"], {**job, "status": "failed"})
        return CaseDraft(name="过期生成", goal="不应写入")

    model.with_structured_output.return_value.ainvoke = generate
    with pytest.raises(ExplorationConflict):
        await generate_case(
            repository,
            session["id"],
            "generate",
            run["id"],
            0,
            ctx=SimpleNamespace(task_control=None),
        )
    assert repository.case(session["id"]) is None