288 lines
11 KiB
Python
288 lines
11 KiB
Python
import asyncio
|
|
import json
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, Mock
|
|
|
|
import pytest
|
|
|
|
from apps.admin_console.services.automation_store import AutomationStore
|
|
from artemis.config.exploration import ExplorationConfig
|
|
from artemis.exploration.models import CaseDraft
|
|
from artemis.exploration.store import ExplorationStore, ExplorationConflict
|
|
from artemis.exploration.synthesis import generate_case
|
|
from artemis.runtime import task_control
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_editorial_never_consumes_device_guidance_or_restarts_recording(tmp_path):
|
|
from artemis.exploration.synthesis import _invoke_editorial
|
|
|
|
directory = tmp_path / "instructions"
|
|
directory.mkdir()
|
|
pending = directory / "user.json"
|
|
pending.write_text('{"instruction":"return home"}', encoding="utf-8")
|
|
control = SimpleNamespace(
|
|
directory=tmp_path, checkpoint=AsyncMock(), run_interruptibly=AsyncMock()
|
|
)
|
|
ctx = SimpleNamespace(task_control=control, on_task_resume=AsyncMock())
|
|
model = SimpleNamespace(ainvoke=AsyncMock(return_value="case"))
|
|
assert await _invoke_editorial(ctx, model, []) == "case"
|
|
assert pending.exists()
|
|
control.checkpoint.assert_not_called()
|
|
control.run_interruptibly.assert_not_called()
|
|
ctx.on_task_resume.assert_not_called()
|
|
|
|
|
|
@pytest.fixture
|
|
def synthesis(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path / "traces")
|
|
repository = ExplorationStore(AutomationStore(tmp_path / "history.db"))
|
|
session = repository.create("打开设置并确认状态", ExplorationConfig(), {}, "create")
|
|
run = repository.reserve_run(
|
|
session["id"], "start", kind="exploration", goal="探索", task_config={}
|
|
)
|
|
outcome = {
|
|
"status": "completed",
|
|
"test_result": {"status": "passed", "reason": "状态正确", "evidence": ["步骤3设置开启"]},
|
|
}
|
|
repository.finish_run(run["id"], outcome)
|
|
root = task_control.control_dir(run["id"])
|
|
root.mkdir(parents=True)
|
|
(root / "exploration-evidence.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"plan": "已完成设置检查",
|
|
"outcome": outcome,
|
|
"steps": [{"step_number": 3, "summary": "设置开启"}],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
model = Mock()
|
|
model.with_structured_output.return_value.ainvoke = AsyncMock(
|
|
return_value=CaseDraft(
|
|
name="设置检查", goal="打开设置，验证开关为开启。", evidence=["步骤3"]
|
|
)
|
|
)
|
|
monkeypatch.setattr("artemis.exploration.synthesis.get_llm", Mock(return_value=model))
|
|
return repository, session, run, model
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_generation_is_idempotent_and_preserves_rerun_independence(synthesis):
|
|
repository, session, run, model = synthesis
|
|
ctx = SimpleNamespace(task_control=None)
|
|
result = await generate_case(repository, session["id"], "generate", run["id"], 0, ctx=ctx)
|
|
duplicate = await generate_case(repository, session["id"], "generate", run["id"], 0, ctx=ctx)
|
|
assert result == duplicate
|
|
assert result["version"] == 1
|
|
assert result["source_run_id"] == run["id"]
|
|
assert "validation_status" not in result
|
|
assert len(repository.runs(session["id"])) == 1
|
|
model.with_structured_output.return_value.ainvoke.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_case_synthesis_receives_legacy_demonstration_platform(synthesis):
|
|
repository, session, run, model = synthesis
|
|
repository.database.put(
|
|
"exploration_demo",
|
|
"old-computer-demo",
|
|
{
|
|
"id": "old-computer-demo",
|
|
"exploration_id": session["id"],
|
|
"run_id": run["id"],
|
|
"source": "computer",
|
|
"status": "completed",
|
|
"media": [{"kind": "computer"}],
|
|
},
|
|
)
|
|
repository.add_message(
|
|
session["id"],
|
|
"浏览器中查看设置后返回首页",
|
|
"old-summary",
|
|
role="assistant",
|
|
metadata={"kind": "demonstration", "demonstration_id": "old-computer-demo"},
|
|
)
|
|
await generate_case(
|
|
repository, session["id"], "generate", run["id"], 0, ctx=SimpleNamespace(task_control=None)
|
|
)
|
|
messages = model.with_structured_output.return_value.ainvoke.await_args.args[0]
|
|
content = json.loads(messages[1].content)
|
|
assert content["demonstrations"] == [
|
|
{
|
|
"id": "old-computer-demo",
|
|
"run_id": run["id"],
|
|
"source": "computer",
|
|
"recorded_sources": ["computer"],
|
|
}
|
|
]
|
|
assert "不能用手机上的成功证明电脑步骤完成" in messages[0].content
|
|
assert repository.run(run["id"])["outcome"]["test_result"]["status"] == "passed"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_synthesis_repairs_unbound_parameters_once_with_original_evidence(synthesis):
|
|
repository, session, run, model = synthesis
|
|
invoke = model.with_structured_output.return_value.ainvoke
|
|
invoke.side_effect = [
|
|
CaseDraft(name="错误草稿", goal="访问 ${首页URL}", preconditions="已指定 ${报告位置}"),
|
|
CaseDraft(name="修订", goal="核对设置，报告保存在本次任务 trace。", evidence=["步骤3"]),
|
|
]
|
|
result = await generate_case(
|
|
repository, session["id"], "generate", run["id"], 0, ctx=SimpleNamespace(task_control=None)
|
|
)
|
|
assert result["version"] == 1 and "${" not in result["goal"]
|
|
assert invoke.await_count == 2
|
|
messages = invoke.await_args.args[0]
|
|
assert "不生成 ${名称}" in messages[0].content
|
|
assert "用户明确指定的位置必须保留" in messages[0].content
|
|
assert "不要让一次复跑递归地产生下一份复跑用例" in messages[0].content
|
|
assert "保留该业务验收" in messages[0].content
|
|
assert "设置开启" in messages[1].content
|
|
assert "不得猜测未知业务输入或降低验收标准" in messages[-1].content
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_failed_parameter_repair_never_saves_unexecutable_generated_case(synthesis):
|
|
repository, session, run, model = synthesis
|
|
repository.save_case(session["id"], CaseDraft(name="原用例", goal="核对设置"), 0, "save")
|
|
invoke = model.with_structured_output.return_value.ainvoke
|
|
invoke.return_value = CaseDraft(name="错误", goal="保存到 ${报告位置}")
|
|
with pytest.raises(ValueError, match="未填写参数"):
|
|
await generate_case(
|
|
repository,
|
|
session["id"],
|
|
"generate",
|
|
run["id"],
|
|
1,
|
|
ctx=SimpleNamespace(task_control=None),
|
|
)
|
|
assert invoke.await_count == 2
|
|
assert repository.case(session["id"])["version"] == 1
|
|
assert repository.case(session["id"])["name"] == "原用例"
|
|
assert repository.database.records("exploration_generation")[0]["status"] == "failed"
|
|
assert repository.run(run["id"])["outcome"]["test_result"]["status"] == "passed"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_model_generation_cannot_overwrite_a_concurrent_manual_revision(synthesis):
|
|
repository, session, run, model = synthesis
|
|
started, finish = asyncio.Event(), asyncio.Event()
|
|
|
|
async def generate(_):
|
|
started.set()
|
|
await finish.wait()
|
|
return CaseDraft(name="模型", goal="模型改动")
|
|
|
|
model.with_structured_output.return_value.ainvoke = generate
|
|
pending = asyncio.create_task(
|
|
generate_case(
|
|
repository,
|
|
session["id"],
|
|
"generate",
|
|
run["id"],
|
|
0,
|
|
ctx=SimpleNamespace(task_control=None),
|
|
)
|
|
)
|
|
await started.wait()
|
|
repository.save_case(session["id"], CaseDraft(name="用户", goal="用户修订"), 0, "manual")
|
|
finish.set()
|
|
with pytest.raises(ExplorationConflict):
|
|
await pending
|
|
assert repository.case(session["id"])["goal"] == "用户修订"
|
|
assert repository.database.records("exploration_generation")[0]["status"] == "failed"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_interrupted_execution_cannot_generate_a_verified_case(synthesis):
|
|
repository, session, run, model = synthesis
|
|
stored = repository.run(run["id"])
|
|
stored["outcome"]["status"] = "interrupted"
|
|
repository.database.put("exploration_run", run["id"], stored)
|
|
with pytest.raises(ExplorationConflict, match="尚未验证通过"):
|
|
await generate_case(
|
|
repository,
|
|
session["id"],
|
|
"generate",
|
|
run["id"],
|
|
0,
|
|
ctx=SimpleNamespace(task_control=None),
|
|
)
|
|
model.with_structured_output.return_value.ainvoke.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_synthesis_failure_leaves_execution_verdict_unchanged(synthesis):
|
|
repository, session, run, model = synthesis
|
|
model.with_structured_output.return_value.ainvoke = AsyncMock(
|
|
side_effect=ValueError("malformed response")
|
|
)
|
|
with pytest.raises(ValueError):
|
|
await generate_case(
|
|
repository,
|
|
session["id"],
|
|
"generate",
|
|
run["id"],
|
|
0,
|
|
ctx=SimpleNamespace(task_control=None),
|
|
)
|
|
assert repository.run(run["id"])["outcome"]["test_result"]["status"] == "passed"
|
|
assert repository.case(session["id"]) is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_context_initialization_failure_marks_generation_retryable(synthesis, monkeypatch):
|
|
repository, session, run, model = synthesis
|
|
monkeypatch.setattr(
|
|
"artemis.exploration.synthesis.get_default_llm_config",
|
|
Mock(side_effect=ValueError("bad config")),
|
|
)
|
|
with pytest.raises(ValueError):
|
|
await generate_case(repository, session["id"], "generate", run["id"], 0)
|
|
assert repository.database.records("exploration_generation")[0]["status"] == "failed"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_long_path_compaction_covers_beginning_and_end(monkeypatch):
|
|
from artemis.exploration.synthesis import _complete_path, PathSummary
|
|
|
|
seen = []
|
|
|
|
async def invoke(messages):
|
|
seen.append(messages[-1].content)
|
|
return PathSummary(verified_path="保留这一段路径及证据")
|
|
|
|
model = Mock()
|
|
model.with_structured_output.return_value.ainvoke = invoke
|
|
monkeypatch.setattr("artemis.exploration.synthesis.get_llm", Mock(return_value=model))
|
|
await _complete_path(
|
|
SimpleNamespace(task_control=None),
|
|
[{"step_number": i, "summary": "状态" * 200} for i in range(250)],
|
|
)
|
|
assert '"step_number": 0,' in seen[0]
|
|
assert '"step_number": 249,' in seen[-1]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_expired_generation_cannot_save_after_recovery(synthesis):
|
|
repository, session, run, model = synthesis
|
|
|
|
async def generate(_):
|
|
job = repository.database.records("exploration_generation")[0]
|
|
repository.database.put("exploration_generation", job["id"], {**job, "status": "failed"})
|
|
return CaseDraft(name="过期生成", goal="不应写入")
|
|
|
|
model.with_structured_output.return_value.ainvoke = generate
|
|
with pytest.raises(ExplorationConflict):
|
|
await generate_case(
|
|
repository,
|
|
session["id"],
|
|
"generate",
|
|
run["id"],
|
|
0,
|
|
ctx=SimpleNamespace(task_control=None),
|
|
)
|
|
assert repository.case(session["id"]) is None
|