feat: add tool-call fallback and resilient batch exports
This commit is contained in:
1 parent
ab066733b8
commit
d31eb2a19d
32 files changed
+538
-72
No files matched your search
@@ -318,6 +318,38 @@ async def test_fail_fast_includes_skipped_reports(isolated, monkeypatch):
|
||||
assert sum(n.endswith(".docx") for n in z.namelist()) == 3
|
||||
|
||||
|
||||
def test_batch_archive_contains_each_task_media_when_present(isolated, monkeypatch, tmp_path):
|
||||
batch = service.create_batch(payload())
|
||||
first, second = isolated.jobs(batch["id"])
|
||||
image1 = tmp_path / "first.png"
|
||||
image2 = tmp_path / "second.png"
|
||||
video = tmp_path / "recording.mp4"
|
||||
image1.write_bytes(b"first screenshot")
|
||||
image2.write_bytes(b"second screenshot")
|
||||
video.write_bytes(b"saved recording")
|
||||
images = {first["session_id"]: image1, second["session_id"]: image2}
|
||||
monkeypatch.setattr(
|
||||
reports,
|
||||
"session_screenshots",
|
||||
lambda report: [(images[report["id"]], "0001_步骤0001_操作前.png")],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
reports,
|
||||
"session_videos",
|
||||
lambda report: (
|
||||
[(video, "0001_recording.mp4")] if report["id"] == first["session_id"] else []
|
||||
),
|
||||
)
|
||||
|
||||
archive = reports.export_batch(batch["id"])
|
||||
with zipfile.ZipFile(archive) as z:
|
||||
names = z.namelist()
|
||||
assert len([name for name in names if "/步骤截图/" in name]) == 2
|
||||
assert len([name for name in names if "/任务录屏/" in name]) == 1
|
||||
assert all(name.startswith("任务报告/") for name in names if "步骤截图" in name)
|
||||
assert z.read(next(name for name in names if "/任务录屏/" in name)) == b"saved recording"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_report_failure_does_not_strand_job(isolated, monkeypatch):
|
||||
job = isolated.jobs(service.create_batch(payload())["id"])[0]
|
||||
|
||||
@@ -23,6 +23,11 @@ def isolated_settings(monkeypatch, tmp_path):
|
||||
"OPENAI_TIMEOUT_SECONDS",
|
||||
"OPENAI_MAX_RETRIES",
|
||||
"OPENAI_RETRY_INTERVAL_SECONDS",
|
||||
"OPENAI_RATE_LIMIT_RETRY_SECONDS",
|
||||
"OPENAI_TOOL_FALLBACK_MODEL",
|
||||
"OPENAI_TOOL_FALLBACK_REASONING_EFFORT",
|
||||
"OPENAI_TOOL_FALLBACK_MAX_RETRIES",
|
||||
"ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS",
|
||||
"OPENAI_REASONING_EFFORT",
|
||||
"ARTEMIS_VIDEO_ENABLED",
|
||||
"ARTEMIS_OUTPUTTER_ENABLED",
|
||||
@@ -43,6 +48,11 @@ def test_save_reload_and_blank_key_preserves_secret(isolated_settings):
|
||||
api_key="new-test-secret",
|
||||
timeout_seconds=75,
|
||||
max_retries=2,
|
||||
rate_limit_retry_seconds=90,
|
||||
tool_fallback_model="custom/fallback",
|
||||
tool_fallback_reasoning_effort="max",
|
||||
tool_fallback_max_retries=2,
|
||||
tool_fallback_no_progress_attempts=4,
|
||||
)
|
||||
result = service.save_model_settings(update)
|
||||
assert result["api_key_configured"]
|
||||
@@ -50,12 +60,16 @@ def test_save_reload_and_blank_key_preserves_secret(isolated_settings):
|
||||
saved = dotenv_values(isolated_settings)
|
||||
assert saved["OPENAI_API_KEY"] == "new-test-secret"
|
||||
assert saved["OPENAI_MODEL"] == "my-model"
|
||||
assert saved["OPENAI_RATE_LIMIT_RETRY_SECONDS"] == "90.0"
|
||||
assert saved["OPENAI_TOOL_FALLBACK_MODEL"] == "custom/fallback"
|
||||
assert saved["ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS"] == "4"
|
||||
assert saved["UNRELATED"] == "keep"
|
||||
assert "# user settings" in isolated_settings.read_text(encoding="utf-8")
|
||||
reloaded = Settings(_env_file=isolated_settings)
|
||||
assert reloaded.OPENAI_MODEL == "my-model"
|
||||
assert parse_llm_config().operator.model == "my-model"
|
||||
assert parse_llm_config().operator.timeout == 75
|
||||
assert parse_llm_config().operator.fallback.model == "custom/fallback"
|
||||
service.save_model_settings(update.model_copy(update={"api_key": SecretStr("")}))
|
||||
assert dotenv_values(isolated_settings)["OPENAI_API_KEY"] == "new-test-secret"
|
||||
|
||||
|
||||
Reference in new issue
Block a user