205 lines
8.4 KiB
Python
205 lines
8.4 KiB
Python
import os
|
|
from pathlib import Path
|
|
|
|
from dotenv import dotenv_values
|
|
import pytest
|
|
from pydantic import SecretStr, ValidationError
|
|
|
|
from apps.admin_console.services import model_settings_service as service
|
|
from artemis.config.settings import Settings, settings
|
|
from artemis.config.agent import load_agent_config
|
|
from artemis.config.llm import parse_llm_config
|
|
|
|
|
|
@pytest.fixture
|
|
def isolated_settings(monkeypatch, tmp_path):
|
|
env = tmp_path / ".env"
|
|
env.write_text("# user settings\nADB_PORT=5037\nUNRELATED=keep\n", encoding="utf-8")
|
|
monkeypatch.setattr(service, "get_env_file", lambda: env)
|
|
keys = [
|
|
"OPENAI_API_KEY",
|
|
"OPENAI_MODEL",
|
|
"OPENAI_BASE_URL",
|
|
"OPENAI_TIMEOUT_SECONDS",
|
|
"OPENAI_MAX_RETRIES",
|
|
"OPENAI_RETRY_INTERVAL_SECONDS",
|
|
"OPENAI_RATE_LIMIT_RETRY_SECONDS",
|
|
"OPENAI_TOOL_FALLBACK_MODEL",
|
|
"OPENAI_TOOL_FALLBACK_REASONING_EFFORT",
|
|
"OPENAI_TOOL_FALLBACK_MAX_RETRIES",
|
|
"ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS",
|
|
"OPENAI_REASONING_EFFORT",
|
|
"ARTEMIS_VIDEO_ENABLED",
|
|
"ARTEMIS_OUTPUTTER_ENABLED",
|
|
"ARTEMIS_STEP_SUMMARIZER_ENABLED",
|
|
"ARTEMIS_SCREENSHOT_LIMIT_ENABLED",
|
|
]
|
|
for key in keys:
|
|
monkeypatch.setattr(settings, key, getattr(settings, key))
|
|
monkeypatch.delenv(key, raising=False)
|
|
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("existing-test-secret"))
|
|
return env
|
|
|
|
|
|
def test_save_reload_and_blank_key_preserves_secret(isolated_settings):
|
|
update = service.ModelSettingsUpdate(
|
|
base_url="https://gateway.example/v1/",
|
|
model="my-model",
|
|
api_key="new-test-secret",
|
|
timeout_seconds=75,
|
|
max_retries=2,
|
|
rate_limit_retry_seconds=90,
|
|
tool_fallback_model="custom/fallback",
|
|
tool_fallback_reasoning_effort="max",
|
|
tool_fallback_max_retries=2,
|
|
tool_fallback_no_progress_attempts=4,
|
|
)
|
|
result = service.save_model_settings(update)
|
|
assert result["api_key_configured"]
|
|
assert "new-test-secret" not in repr(result)
|
|
saved = dotenv_values(isolated_settings)
|
|
assert saved["OPENAI_API_KEY"] == "new-test-secret"
|
|
assert saved["OPENAI_MODEL"] == "my-model"
|
|
assert saved["OPENAI_RATE_LIMIT_RETRY_SECONDS"] == "90.0"
|
|
assert saved["OPENAI_TOOL_FALLBACK_MODEL"] == "custom/fallback"
|
|
assert saved["ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS"] == "4"
|
|
assert saved["UNRELATED"] == "keep"
|
|
assert "# user settings" in isolated_settings.read_text(encoding="utf-8")
|
|
reloaded = Settings(_env_file=isolated_settings)
|
|
assert reloaded.OPENAI_MODEL == "my-model"
|
|
assert parse_llm_config().operator.model == "my-model"
|
|
assert parse_llm_config().operator.timeout == 75
|
|
assert parse_llm_config().operator.fallback.model == "custom/fallback"
|
|
service.save_model_settings(update.model_copy(update={"api_key": SecretStr("")}))
|
|
assert dotenv_values(isolated_settings)["OPENAI_API_KEY"] == "new-test-secret"
|
|
|
|
|
|
def test_new_install_uses_playones_endpoint_without_overriding_saved_url(monkeypatch):
|
|
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
|
expected = "https://api-n-cd.playones.com/v1"
|
|
assert Settings(_env_file=None).OPENAI_BASE_URL == expected
|
|
assert service.ModelSettingsUpdate(model="test").base_url == expected
|
|
monkeypatch.setattr(settings, "OPENAI_BASE_URL", None)
|
|
assert service.get_model_settings()["base_url"] == expected
|
|
monkeypatch.setattr(settings, "OPENAI_BASE_URL", "https://saved.example/v1")
|
|
assert service.get_model_settings()["base_url"] == "https://saved.example/v1"
|
|
|
|
|
|
def test_model_presets_include_flash_pro_and_luna_without_restricting_custom_names():
|
|
presets = service.get_model_settings()["model_presets"]
|
|
assert presets == ["gpt-5.6-luna", "gpt-6-luna", "mimo-v2.6-flash", "mimo-v2.6-pro"]
|
|
assert service.ModelSettingsUpdate(model="custom/model-v2").model == "custom/model-v2"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"url",
|
|
[
|
|
"ftp://example.com",
|
|
"https://user:pass@example.com",
|
|
"https://example.com/v1/responses",
|
|
"https://example.com/v1?key=secret",
|
|
],
|
|
)
|
|
def test_rejects_invalid_base_url(url):
|
|
with pytest.raises(ValidationError):
|
|
service.ModelSettingsUpdate(base_url=url, model="test")
|
|
|
|
|
|
def test_capabilities_persist_and_apply_to_new_tasks(isolated_settings):
|
|
result = service.save_capability_settings(
|
|
service.CapabilitySettings(
|
|
video_enabled=False,
|
|
outputter_enabled=False,
|
|
step_summarizer_enabled=False,
|
|
screenshot_limit_enabled=False,
|
|
)
|
|
)
|
|
assert result.video_enabled is False
|
|
saved = dotenv_values(isolated_settings)
|
|
assert saved["ARTEMIS_VIDEO_ENABLED"] == "false"
|
|
assert saved["ARTEMIS_SCREENSHOT_LIMIT_ENABLED"] == "false"
|
|
config = load_agent_config()
|
|
assert config.video_analyzer.enabled is False
|
|
assert config.pro.video_analyzer.enabled is False
|
|
assert config.outputter.enabled is False
|
|
assert config.flash.step_summarizer.enabled is False
|
|
assert config.screenshot.enabled is False
|
|
service.save_capability_settings(service.CapabilitySettings(video_enabled=True))
|
|
assert load_agent_config().video_analyzer.enabled is True
|
|
|
|
|
|
def test_screenshot_json_can_override_environment(monkeypatch, tmp_path):
|
|
config_path = tmp_path / "artemis.jsonc"
|
|
config_path.write_text(
|
|
'{"agent": {"screenshot": {"enabled": true, "max_long_edge": 1024}}}',
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setenv("ARTEMIS_SCREENSHOT_LIMIT_ENABLED", "false")
|
|
screenshot = load_agent_config(config_path).screenshot
|
|
assert screenshot.enabled is True
|
|
assert screenshot.max_long_edge == 1024
|
|
|
|
|
|
def test_failed_write_keeps_running_settings(monkeypatch, isolated_settings):
|
|
previous = settings.OPENAI_MODEL
|
|
monkeypatch.setattr(
|
|
service.os, "replace", lambda *args: (_ for _ in ()).throw(OSError("read only"))
|
|
)
|
|
with pytest.raises(OSError):
|
|
service.save_model_settings(service.ModelSettingsUpdate(model="not-saved"))
|
|
assert settings.OPENAI_MODEL == previous
|
|
assert "not-saved" not in isolated_settings.read_text(encoding="utf-8")
|
|
|
|
|
|
def test_key_not_in_validation_errors(isolated_settings):
|
|
with pytest.raises(ValueError) as exc:
|
|
service.save_model_settings(
|
|
service.ModelSettingsUpdate(model="test", api_key="secret\nvalue")
|
|
)
|
|
assert "secret" not in str(exc.value)
|
|
|
|
|
|
def test_reasoning_retry_settings_roundtrip(isolated_settings):
|
|
result = service.save_model_settings(service.ModelSettingsUpdate(model="gpt-5.6-luna"))
|
|
assert result["timeout_seconds"] == 300
|
|
assert result["max_retries"] == 5
|
|
assert result["retry_interval_seconds"] == 10
|
|
assert result["reasoning_effort"] == "high"
|
|
saved = Settings(_env_file=isolated_settings)
|
|
assert saved.OPENAI_REASONING_EFFORT == "high"
|
|
assert parse_llm_config().operator.reasoning_effort == "high"
|
|
service.save_model_settings(
|
|
service.ModelSettingsUpdate(model="custom", reasoning_effort="default")
|
|
)
|
|
assert parse_llm_config().operator.reasoning_effort == "default"
|
|
|
|
|
|
def test_max_reasoning_and_custom_model_survive_reload(isolated_settings):
|
|
result = service.save_model_settings(
|
|
service.ModelSettingsUpdate(model="custom/model-v2", reasoning_effort="max")
|
|
)
|
|
assert result["model"] == "custom/model-v2"
|
|
assert result["reasoning_effort"] == "max"
|
|
assert dotenv_values(isolated_settings)["OPENAI_REASONING_EFFORT"] == "max"
|
|
reloaded = Settings(_env_file=isolated_settings)
|
|
assert reloaded.OPENAI_REASONING_EFFORT == "max"
|
|
assert reloaded.OPENAI_MODEL == "custom/model-v2"
|
|
assert parse_llm_config().operator.reasoning_effort == "max"
|
|
|
|
|
|
@pytest.mark.parametrize("effort", ["high", "max"])
|
|
def test_responses_payload_has_reasoning_and_no_sdk_retry(monkeypatch, effort):
|
|
from artemis.llm.router import ModelEndpoint, ModelFactory
|
|
from unittest.mock import Mock
|
|
|
|
factory = Mock()
|
|
monkeypatch.setattr("langchain_openai.ChatOpenAI", factory)
|
|
ModelFactory.create_model(
|
|
ModelEndpoint(api_key="test-only", reasoning_effort=effort, max_retries=5)
|
|
)
|
|
assert factory.call_args.kwargs["reasoning_effort"] == effort
|
|
assert factory.call_args.kwargs["max_retries"] == 0
|
|
assert factory.call_args.kwargs["use_responses_api"] is True
|
|
ModelFactory.create_model(ModelEndpoint(api_key="test-only", reasoning_effort="default"))
|
|
assert "reasoning_effort" not in factory.call_args.kwargs
|