201 lines
8.3 KiB
Python
201 lines
8.3 KiB
Python
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
import pytest
|
|
from pydantic import ValidationError
|
|
|
|
from apps.admin_console.services.automation_store import AutomationStore
|
|
from artemis.config.exploration import ExplorationConfig
|
|
from artemis.exploration.models import CaseDraft, CreateExploration
|
|
from artemis.exploration.store import ExplorationConflict, ExplorationStore
|
|
|
|
|
|
@pytest.fixture
|
|
def repository(tmp_path):
|
|
return ExplorationStore(AutomationStore(tmp_path / "history.db"))
|
|
|
|
|
|
def create(repository, request_id="create-one"):
|
|
return repository.create("探索相机夜视设置并验证画面", ExplorationConfig(), {}, request_id)
|
|
|
|
|
|
def test_create_retry_is_idempotent_and_conflicting_request_is_rejected(repository):
|
|
first = create(repository)
|
|
assert create(repository)["id"] == first["id"]
|
|
assert len(repository.list()) == 1
|
|
assert len(repository.messages(first["id"])) == 1
|
|
with pytest.raises(ExplorationConflict):
|
|
repository.create("另一个目标", ExplorationConfig(), {}, "create-one")
|
|
assert len(repository.list()) == 1
|
|
|
|
|
|
def test_parallel_web_mcp_messages_are_additive_and_cursor_is_stable(repository):
|
|
sid = create(repository)["id"]
|
|
with ThreadPoolExecutor(max_workers=6) as pool:
|
|
messages = list(
|
|
pool.map(
|
|
lambda i: repository.add_message(sid, f"指导 {i}", f"message-{i}", source="mcp"),
|
|
range(24),
|
|
)
|
|
)
|
|
assert len({m["sequence"] for m in messages}) == 24
|
|
assert repository.get(sid)["message_count"] == 25
|
|
first_page = repository.messages(sid, limit=10)
|
|
next_page = repository.messages(sid, after=first_page[-1]["sequence"])
|
|
assert [m["sequence"] for m in first_page + next_page] == list(range(1, 26))
|
|
previous = repository.add_message(sid, "指导 0", "message-0", source="web")
|
|
assert previous["source"] == "mcp"
|
|
assert repository.get(sid)["message_count"] == 25
|
|
|
|
|
|
def test_parallel_start_reserves_only_one_actual_run(repository):
|
|
sid = create(repository)["id"]
|
|
|
|
def reserve(_):
|
|
return repository.reserve_run(
|
|
sid, "run-one", kind="exploration", goal="探索", task_config={}
|
|
)
|
|
|
|
with ThreadPoolExecutor(max_workers=4) as pool:
|
|
runs = list(pool.map(reserve, range(8)))
|
|
assert len({r["id"] for r in runs}) == 1
|
|
assert len(repository.runs(sid)) == 1
|
|
with pytest.raises(ExplorationConflict):
|
|
repository.reserve_run(sid, "run-two", kind="exploration", goal="探索", task_config={})
|
|
|
|
|
|
def test_case_versions_are_immutable_and_stale_edits_do_not_overwrite(repository):
|
|
sid = create(repository)["id"]
|
|
first = repository.save_case(
|
|
sid, CaseDraft(name="夜视", goal="打开夜视并核对画面"), 0, "case-one"
|
|
)
|
|
assert (
|
|
repository.save_case(sid, CaseDraft(name="夜视", goal="打开夜视并核对画面"), 0, "case-one")
|
|
== first
|
|
)
|
|
second = repository.save_case(
|
|
sid, CaseDraft(name="夜视", goal="切换自动夜视并核对画面"), 1, "case-two"
|
|
)
|
|
with pytest.raises(ExplorationConflict):
|
|
repository.save_case(sid, CaseDraft(name="过期", goal="旧编辑"), 1, "stale")
|
|
assert repository.case(sid, 1) == first
|
|
assert repository.case(sid) == second
|
|
assert repository.get(sid)["case_version"] == 2
|
|
|
|
|
|
def test_validation_pins_case_version_and_actual_verdict(repository):
|
|
sid = create(repository)["id"]
|
|
repository.save_case(sid, CaseDraft(name="夜视", goal="核对夜视"), 0, "save")
|
|
with pytest.raises(ValueError, match="版本"):
|
|
repository.reserve_run(
|
|
sid, "bad-version", kind="validation", goal="复跑", task_config={}, case_version=2
|
|
)
|
|
run = repository.reserve_run(
|
|
sid,
|
|
"validation",
|
|
kind="validation",
|
|
goal="核对夜视",
|
|
task_config={"profile": "flash"},
|
|
case_version=1,
|
|
)
|
|
outcome = {
|
|
"status": "completed",
|
|
"test_result": {"status": "failed", "reason": "画面不符合预期"},
|
|
}
|
|
repository.finish_run(run["id"], outcome)
|
|
assert repository.run(run["id"])["outcome"] == outcome
|
|
assert repository.run(run["id"])["case_version"] == 1
|
|
assert repository.get(sid)["active_run_id"] is None
|
|
assert (
|
|
repository.finish_run(
|
|
run["id"], {"status": "completed", "test_result": {"status": "passed"}}
|
|
)["outcome"]
|
|
== outcome
|
|
)
|
|
|
|
|
|
def test_old_worker_cannot_change_new_run_state(repository):
|
|
sid = create(repository)["id"]
|
|
old = repository.reserve_run(sid, "old", kind="exploration", goal="探索", task_config={})
|
|
repository.finish_run(old["id"], {"status": "interrupted"})
|
|
new = repository.reserve_run(sid, "new", kind="exploration", goal="继续", task_config={})
|
|
with pytest.raises(ExplorationConflict):
|
|
repository.set_run_state(old["id"], "running")
|
|
assert repository.get(sid)["active_run_id"] == new["id"]
|
|
|
|
|
|
def test_normal_exit_preserves_checker_verdict_and_incomplete_checks(repository):
|
|
sid = create(repository)["id"]
|
|
run = repository.reserve_run(sid, "start", kind="exploration", goal="探索", task_config={})
|
|
verified = {
|
|
"task_status": "completed",
|
|
"tests": {"passed": 1, "inconclusive": 1},
|
|
"test_result": {
|
|
"status": "inconclusive",
|
|
"reason": "缺少入口点击证据",
|
|
"evidence": ["搜索页可见"],
|
|
},
|
|
}
|
|
repository.record_verdict(run["id"], verified)
|
|
result = repository.finish_run(run["id"], {"status": "completed", "test_result": None})
|
|
assert result["outcome"] == {**verified, "status": "completed"}
|
|
assert "缺少入口点击证据" in repository.messages(sid)[-1]["text"]
|
|
|
|
|
|
def test_execution_budget_pause_does_not_finish_run_or_create_failed_verdict(repository):
|
|
sid = create(repository)["id"]
|
|
run = repository.reserve_run(sid, "run", kind="exploration", goal="探索", task_config={})
|
|
repository.set_run_state(run["id"], "running", elapsed_seconds=10)
|
|
repository.set_run_state(run["id"], "paused", reason="execution_budget", elapsed_seconds=1800)
|
|
assert repository.run(run["id"])["outcome"] is None
|
|
assert repository.get(sid)["active_run_id"] == run["id"]
|
|
assert repository.get(sid)["elapsed_seconds"] == 1800
|
|
repository.set_run_state(run["id"], "running", elapsed_seconds=1800)
|
|
repository.set_run_state(run["id"], "running", elapsed_seconds=1810)
|
|
assert repository.get(sid)["elapsed_seconds"] == 1810
|
|
|
|
|
|
def test_questions_and_answers_keep_origin_and_order(repository):
|
|
sid = create(repository)["id"]
|
|
run = repository.reserve_run(sid, "run", kind="exploration", goal="探索", task_config={})
|
|
question = repository.ask(run["id"], "预期需要自动切换吗？", "建议验证自动切换", "q1")
|
|
assert repository.get(sid)["pending_question"] == question["id"]
|
|
assert repository.ask(run["id"], "预期需要自动切换吗？", "建议验证自动切换", "q1") == question
|
|
answer = repository.add_message(
|
|
sid, "是", "answer", source="mcp", metadata={"question_id": question["id"]}
|
|
)
|
|
assert answer["sequence"] == question["sequence"] + 1
|
|
assert answer["question_id"] == question["id"]
|
|
|
|
|
|
def test_generated_case_cannot_borrow_another_sessions_evidence(repository):
|
|
sid = create(repository)["id"]
|
|
other = create(repository, "other")["id"]
|
|
run = repository.reserve_run(other, "run", kind="exploration", goal="探索", task_config={})
|
|
repository.finish_run(run["id"], {"status": "completed"})
|
|
with pytest.raises(ValueError, match="不属于"):
|
|
repository.save_case(
|
|
sid,
|
|
CaseDraft(name="用例", goal="目标"),
|
|
0,
|
|
"case",
|
|
source_run_id=run["id"],
|
|
generated=True,
|
|
)
|
|
assert repository.get(sid)["case_version"] == 0
|
|
|
|
|
|
def test_exploration_config_defaults_and_rejects_credentials():
|
|
config = ExplorationConfig()
|
|
assert (config.model, config.reasoning_effort, config.timeout_seconds) == (
|
|
"gpt-6.1-sol",
|
|
"high",
|
|
1800,
|
|
)
|
|
assert config.demonstration_timeout_seconds == 900
|
|
with pytest.raises(ValidationError):
|
|
ExplorationConfig.model_validate({"api_key": "not-allowed"})
|
|
with pytest.raises(ValidationError):
|
|
ExplorationConfig(demonstration_timeout_seconds=60, demonstration_warning_seconds=60)
|
|
with pytest.raises(ValidationError):
|
|
CreateExploration(request_id="r", goal=" ")
|