142 lines
5.4 KiB
Python
142 lines
5.4 KiB
Python
import os
|
|
from pathlib import Path
|
|
|
|
from dotenv import dotenv_values
|
|
import pytest
|
|
from pydantic import SecretStr, ValidationError
|
|
|
|
from apps.admin_console.services import model_settings_service as service
|
|
from artemis.config.settings import Settings, settings
|
|
from artemis.config.agent import load_agent_config
|
|
from artemis.config.llm import parse_llm_config
|
|
|
|
|
|
@pytest.fixture
|
|
def isolated_settings(monkeypatch, tmp_path):
|
|
env = tmp_path / ".env"
|
|
env.write_text("# user settings\nADB_PORT=5037\nUNRELATED=keep\n", encoding="utf-8")
|
|
monkeypatch.setattr(service, "get_env_file", lambda: env)
|
|
keys = [
|
|
"OPENAI_API_KEY",
|
|
"OPENAI_MODEL",
|
|
"OPENAI_BASE_URL",
|
|
"OPENAI_TIMEOUT_SECONDS",
|
|
"OPENAI_MAX_RETRIES",
|
|
"OPENAI_RETRY_INTERVAL_SECONDS",
|
|
"OPENAI_REASONING_EFFORT",
|
|
"ARTEMIS_VIDEO_ENABLED",
|
|
"ARTEMIS_OUTPUTTER_ENABLED",
|
|
"ARTEMIS_STEP_SUMMARIZER_ENABLED",
|
|
]
|
|
for key in keys:
|
|
monkeypatch.setattr(settings, key, getattr(settings, key))
|
|
monkeypatch.delenv(key, raising=False)
|
|
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("existing-test-secret"))
|
|
return env
|
|
|
|
|
|
def test_save_reload_and_blank_key_preserves_secret(isolated_settings):
|
|
update = service.ModelSettingsUpdate(
|
|
base_url="https://gateway.example/v1/",
|
|
model="my-model",
|
|
api_key="new-test-secret",
|
|
timeout_seconds=75,
|
|
max_retries=2,
|
|
)
|
|
result = service.save_model_settings(update)
|
|
assert result["api_key_configured"]
|
|
assert "new-test-secret" not in repr(result)
|
|
saved = dotenv_values(isolated_settings)
|
|
assert saved["OPENAI_API_KEY"] == "new-test-secret"
|
|
assert saved["OPENAI_MODEL"] == "my-model"
|
|
assert saved["UNRELATED"] == "keep"
|
|
assert "# user settings" in isolated_settings.read_text(encoding="utf-8")
|
|
reloaded = Settings(_env_file=isolated_settings)
|
|
assert reloaded.OPENAI_MODEL == "my-model"
|
|
assert parse_llm_config().operator.model == "my-model"
|
|
assert parse_llm_config().operator.timeout == 75
|
|
service.save_model_settings(update.model_copy(update={"api_key": SecretStr("")}))
|
|
assert dotenv_values(isolated_settings)["OPENAI_API_KEY"] == "new-test-secret"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"url",
|
|
[
|
|
"ftp://example.com",
|
|
"https://user:pass@example.com",
|
|
"https://example.com/v1/responses",
|
|
"https://example.com/v1?key=secret",
|
|
],
|
|
)
|
|
def test_rejects_invalid_base_url(url):
|
|
with pytest.raises(ValidationError):
|
|
service.ModelSettingsUpdate(base_url=url, model="test")
|
|
|
|
|
|
def test_capabilities_persist_and_apply_to_new_tasks(isolated_settings):
|
|
result = service.save_capability_settings(
|
|
service.CapabilitySettings(
|
|
video_enabled=False, outputter_enabled=False, step_summarizer_enabled=False
|
|
)
|
|
)
|
|
assert result.video_enabled is False
|
|
saved = dotenv_values(isolated_settings)
|
|
assert saved["ARTEMIS_VIDEO_ENABLED"] == "false"
|
|
config = load_agent_config()
|
|
assert config.video_analyzer.enabled is False
|
|
assert config.pro.video_analyzer.enabled is False
|
|
assert config.outputter.enabled is False
|
|
assert config.flash.step_summarizer.enabled is False
|
|
service.save_capability_settings(service.CapabilitySettings(video_enabled=True))
|
|
assert load_agent_config().video_analyzer.enabled is True
|
|
|
|
|
|
def test_failed_write_keeps_running_settings(monkeypatch, isolated_settings):
|
|
previous = settings.OPENAI_MODEL
|
|
monkeypatch.setattr(
|
|
service.os, "replace", lambda *args: (_ for _ in ()).throw(OSError("read only"))
|
|
)
|
|
with pytest.raises(OSError):
|
|
service.save_model_settings(service.ModelSettingsUpdate(model="not-saved"))
|
|
assert settings.OPENAI_MODEL == previous
|
|
assert "not-saved" not in isolated_settings.read_text(encoding="utf-8")
|
|
|
|
|
|
def test_key_not_in_validation_errors(isolated_settings):
|
|
with pytest.raises(ValueError) as exc:
|
|
service.save_model_settings(
|
|
service.ModelSettingsUpdate(model="test", api_key="secret\nvalue")
|
|
)
|
|
assert "secret" not in str(exc.value)
|
|
|
|
|
|
def test_reasoning_retry_settings_roundtrip(isolated_settings):
|
|
result = service.save_model_settings(service.ModelSettingsUpdate(model="gpt-5.6-luna"))
|
|
assert result["timeout_seconds"] == 300
|
|
assert result["max_retries"] == 5
|
|
assert result["retry_interval_seconds"] == 10
|
|
assert result["reasoning_effort"] == "high"
|
|
saved = Settings(_env_file=isolated_settings)
|
|
assert saved.OPENAI_REASONING_EFFORT == "high"
|
|
assert parse_llm_config().operator.reasoning_effort == "high"
|
|
service.save_model_settings(
|
|
service.ModelSettingsUpdate(model="custom", reasoning_effort="default")
|
|
)
|
|
assert parse_llm_config().operator.reasoning_effort == "default"
|
|
|
|
|
|
def test_responses_payload_has_reasoning_and_no_sdk_retry(monkeypatch):
|
|
from artemis.llm.router import ModelEndpoint, ModelFactory
|
|
from unittest.mock import Mock
|
|
|
|
factory = Mock()
|
|
monkeypatch.setattr("langchain_openai.ChatOpenAI", factory)
|
|
ModelFactory.create_model(
|
|
ModelEndpoint(api_key="test-only", reasoning_effort="high", max_retries=5)
|
|
)
|
|
assert factory.call_args.kwargs["reasoning_effort"] == "high"
|
|
assert factory.call_args.kwargs["max_retries"] == 0
|
|
assert factory.call_args.kwargs["use_responses_api"] is True
|
|
ModelFactory.create_model(ModelEndpoint(api_key="test-only", reasoning_effort="default"))
|
|
assert "reasoning_effort" not in factory.call_args.kwargs
|