327 lines
14 KiB
Python
327 lines
14 KiB
Python
import os
|
|
from pathlib import Path
|
|
|
|
from dotenv import dotenv_values
|
|
import pytest
|
|
from pydantic import SecretStr, ValidationError
|
|
|
|
from apps.admin_console.services import model_settings_service as service
|
|
from artemis.config.settings import Settings, settings
|
|
from artemis.config.agent import load_agent_config
|
|
from artemis.config.llm import parse_llm_config
|
|
from artemis.config.environment import migrate_transport_default, TRANSPORT_DEFAULT_VERSION_KEY
|
|
|
|
|
|
@pytest.fixture
|
|
def isolated_settings(monkeypatch, tmp_path):
|
|
env = tmp_path / ".env"
|
|
env.write_text("# user settings\nADB_PORT=5037\nUNRELATED=keep\n", encoding="utf-8")
|
|
monkeypatch.setattr(service, "get_env_file", lambda: env)
|
|
keys = [
|
|
"OPENAI_API_KEY",
|
|
"OPENAI_MODEL",
|
|
"OPENAI_BASE_URL",
|
|
"OPENAI_TIMEOUT_SECONDS",
|
|
"OPENAI_TRANSPORT",
|
|
"OPENAI_MAX_RETRIES",
|
|
"OPENAI_RETRY_INTERVAL_SECONDS",
|
|
"OPENAI_RATE_LIMIT_RETRY_SECONDS",
|
|
"OPENAI_TOOL_FALLBACK_MODEL",
|
|
"OPENAI_TOOL_FALLBACK_REASONING_EFFORT",
|
|
"OPENAI_TOOL_FALLBACK_MAX_RETRIES",
|
|
"ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS",
|
|
"OPENAI_REASONING_EFFORT",
|
|
"ARTEMIS_VIDEO_ENABLED",
|
|
"ARTEMIS_OUTPUTTER_ENABLED",
|
|
"ARTEMIS_FLASH_FINAL_CHECK_ENABLED",
|
|
"ARTEMIS_STEP_SUMMARIZER_ENABLED",
|
|
"ARTEMIS_SCREENSHOT_LIMIT_ENABLED",
|
|
"ARTEMIS_TASK_TIMEOUT_SECONDS",
|
|
"ARTEMIS_OBSERVATION_TIMEOUT_SECONDS",
|
|
"ARTEMIS_OBSERVATION_MAX_ATTEMPTS",
|
|
"ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS",
|
|
"ARTEMIS_ACTION_FAILURE_ATTEMPTS",
|
|
"ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS",
|
|
]
|
|
for key in keys:
|
|
monkeypatch.setattr(settings, key, getattr(settings, key))
|
|
monkeypatch.delenv(key, raising=False)
|
|
monkeypatch.delenv(TRANSPORT_DEFAULT_VERSION_KEY, raising=False)
|
|
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("existing-test-secret"))
|
|
return env
|
|
|
|
|
|
def test_save_reload_and_blank_key_preserves_secret(isolated_settings):
|
|
update = service.ModelSettingsUpdate(
|
|
transport="websocket",
|
|
base_url="https://gateway.example/v1/",
|
|
model="my-model",
|
|
api_key="new-test-secret",
|
|
timeout_seconds=75,
|
|
max_retries=2,
|
|
rate_limit_retry_seconds=90,
|
|
tool_fallback_model="custom/fallback",
|
|
tool_fallback_reasoning_effort="max",
|
|
tool_fallback_max_retries=2,
|
|
tool_fallback_no_progress_attempts=4,
|
|
)
|
|
result = service.save_model_settings(update)
|
|
assert result["api_key_configured"]
|
|
assert result["transport"] == "websocket"
|
|
assert dotenv_values(isolated_settings)["OPENAI_TRANSPORT"] == "websocket"
|
|
assert service.ModelSettingsUpdate(model="test").transport is None
|
|
assert "new-test-secret" not in repr(result)
|
|
saved = dotenv_values(isolated_settings)
|
|
assert saved["OPENAI_API_KEY"] == "new-test-secret"
|
|
assert saved["OPENAI_MODEL"] == "my-model"
|
|
assert saved["OPENAI_RATE_LIMIT_RETRY_SECONDS"] == "90.0"
|
|
assert saved["OPENAI_TOOL_FALLBACK_MODEL"] == "custom/fallback"
|
|
assert saved["ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS"] == "4"
|
|
assert saved["UNRELATED"] == "keep"
|
|
assert "# user settings" in isolated_settings.read_text(encoding="utf-8")
|
|
reloaded = Settings(_env_file=isolated_settings)
|
|
assert reloaded.OPENAI_MODEL == "my-model"
|
|
assert parse_llm_config().operator.model == "my-model"
|
|
assert parse_llm_config().operator.timeout == 75
|
|
assert parse_llm_config().operator.fallback.model == "custom/fallback"
|
|
service.save_model_settings(update.model_copy(update={"api_key": SecretStr("")}))
|
|
assert dotenv_values(isolated_settings)["OPENAI_API_KEY"] == "new-test-secret"
|
|
|
|
|
|
def test_saved_http_opt_out_survives_restart_migration(isolated_settings):
|
|
result = service.save_model_settings(
|
|
service.ModelSettingsUpdate(model="test", transport="http")
|
|
)
|
|
assert result["transport"] == "http"
|
|
assert dotenv_values(isolated_settings)[TRANSPORT_DEFAULT_VERSION_KEY] == "1"
|
|
assert migrate_transport_default(isolated_settings) is False
|
|
assert Settings(_env_file=isolated_settings).OPENAI_TRANSPORT == "http"
|
|
|
|
|
|
def test_new_install_uses_playones_endpoint_without_overriding_saved_url(monkeypatch):
|
|
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
|
expected = "https://api-n-cd.playones.com/v1"
|
|
assert Settings(_env_file=None).OPENAI_BASE_URL == expected
|
|
assert service.ModelSettingsUpdate(model="test").base_url == expected
|
|
monkeypatch.setattr(settings, "OPENAI_BASE_URL", None)
|
|
assert service.get_model_settings()["base_url"] == expected
|
|
monkeypatch.setattr(settings, "OPENAI_BASE_URL", "https://saved.example/v1")
|
|
assert service.get_model_settings()["base_url"] == "https://saved.example/v1"
|
|
|
|
|
|
def test_model_presets_keep_supported_choices_without_restricting_custom_names():
|
|
presets = service.get_model_settings()["model_presets"]
|
|
assert presets == [
|
|
"gpt-6-luna",
|
|
"gpt-6.1-terra",
|
|
"gpt-6.1-sol",
|
|
"gpt-6-astra",
|
|
"mimo-v2.6-pro",
|
|
]
|
|
assert service.ModelSettingsUpdate(model="custom/model-v2").model == "custom/model-v2"
|
|
|
|
|
|
def test_new_install_defaults_to_luna_high_and_preserves_saved_model(isolated_settings):
|
|
from artemis.config.constants import DEFAULT_MODEL
|
|
|
|
fresh = Settings(_env_file=None)
|
|
assert DEFAULT_MODEL == fresh.OPENAI_MODEL == "gpt-6-luna"
|
|
assert fresh.OPENAI_REASONING_EFFORT == "high"
|
|
isolated_settings.write_text("OPENAI_MODEL=custom/existing\n", encoding="utf-8")
|
|
assert Settings(_env_file=isolated_settings).OPENAI_MODEL == "custom/existing"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"url",
|
|
[
|
|
"ftp://example.com",
|
|
"https://user:pass@example.com",
|
|
"https://example.com/v1/responses",
|
|
"https://example.com/v1?key=secret",
|
|
],
|
|
)
|
|
def test_rejects_invalid_base_url(url):
|
|
with pytest.raises(ValidationError):
|
|
service.ModelSettingsUpdate(base_url=url, model="test")
|
|
|
|
|
|
def test_capabilities_persist_and_apply_to_new_tasks(isolated_settings):
|
|
result = service.save_capability_settings(
|
|
service.CapabilitySettings(
|
|
video_enabled=False,
|
|
outputter_enabled=False,
|
|
step_summarizer_enabled=False,
|
|
screenshot_limit_enabled=False,
|
|
)
|
|
)
|
|
assert result.video_enabled is False
|
|
saved = dotenv_values(isolated_settings)
|
|
assert saved["ARTEMIS_VIDEO_ENABLED"] == "false"
|
|
assert saved["ARTEMIS_SCREENSHOT_LIMIT_ENABLED"] == "false"
|
|
config = load_agent_config()
|
|
assert config.video_analyzer.enabled is False
|
|
assert config.pro.video_analyzer.enabled is False
|
|
assert config.outputter.enabled is False
|
|
assert config.flash.step_summarizer.enabled is False
|
|
assert config.screenshot.enabled is False
|
|
service.save_capability_settings(service.CapabilitySettings(video_enabled=True))
|
|
assert load_agent_config().video_analyzer.enabled is True
|
|
|
|
|
|
def test_flash_review_defaults_off_and_persists_without_changing_pro(isolated_settings):
|
|
assert Settings(_env_file=None).ARTEMIS_FLASH_FINAL_CHECK_ENABLED is False
|
|
assert service.get_capability_settings().flash_final_check_enabled is False
|
|
assert load_agent_config().checker.final_check is True
|
|
for enabled in (True, False):
|
|
saved = service.save_capability_settings(
|
|
service.CapabilitySettings(flash_final_check_enabled=enabled)
|
|
)
|
|
assert saved.flash_final_check_enabled is enabled
|
|
assert Settings(_env_file=isolated_settings).ARTEMIS_FLASH_FINAL_CHECK_ENABLED is enabled
|
|
config = load_agent_config()
|
|
assert config.flash.final_check_enabled is enabled
|
|
assert config.checker.final_check is True
|
|
assert config.pro.checker.final_check is True
|
|
# Older clients omit the new switch; saving another setting keeps it.
|
|
assert (
|
|
service.save_capability_settings(
|
|
service.CapabilitySettings(recording_enabled=False)
|
|
).flash_final_check_enabled
|
|
is enabled
|
|
)
|
|
|
|
|
|
def test_old_checker_config_does_not_enable_flash_review(monkeypatch, tmp_path):
|
|
monkeypatch.delenv("ARTEMIS_FLASH_FINAL_CHECK_ENABLED", raising=False)
|
|
path = tmp_path / "artemis.jsonc"
|
|
path.write_text('{"agent":{"checker":{"enabled":true,"final_check":true}}}', encoding="utf-8")
|
|
assert load_agent_config(path).flash.final_check_enabled is False
|
|
|
|
|
|
@pytest.mark.parametrize("enabled", [True, False])
|
|
def test_saved_flash_switch_reaches_next_worker_without_restarting(
|
|
isolated_settings, monkeypatch, enabled
|
|
):
|
|
from apps.admin_console.services.execution_config import execution_config
|
|
from apps.admin_console.services.task_queue_service import TaskQueueService
|
|
|
|
monkeypatch.setattr("artemis.exploration.store.explorations.run", lambda _sid: None)
|
|
monkeypatch.setattr("apps.admin_console.services.automation_store.store.job", lambda _sid: None)
|
|
monkeypatch.setenv("ARTEMIS_FLASH_FINAL_CHECK_ENABLED", str(not enabled).lower())
|
|
running = {"profile": "flash", "goal": "already running", "session_id": "running-review"}
|
|
running["execution_config"] = execution_config(running)
|
|
queued = {"profile": "flash", "goal": "queued", "session_id": "queued-review"}
|
|
queued["execution_config"] = execution_config(queued)
|
|
|
|
service.save_capability_settings(service.CapabilitySettings(flash_final_check_enabled=enabled))
|
|
# The real queue refreshes pending snapshots just before starting their worker.
|
|
queued["execution_config"] = execution_config(queued)
|
|
for item, expected in ((running, not enabled), (queued, enabled)):
|
|
_cmd, env = TaskQueueService._build_worker_invocation(
|
|
item,
|
|
item["session_id"],
|
|
item["session_id"],
|
|
item["goal"],
|
|
"flash",
|
|
TaskQueueService._task_target(item),
|
|
)
|
|
assert env["ARTEMIS_FLASH_FINAL_CHECK_ENABLED"] == str(expected).lower()
|
|
assert load_agent_config().pro.checker.final_check is True
|
|
|
|
|
|
def test_screenshot_json_can_override_environment(monkeypatch, tmp_path):
|
|
config_path = tmp_path / "artemis.jsonc"
|
|
config_path.write_text(
|
|
'{"agent": {"screenshot": {"enabled": true, "max_long_edge": 1024}}}',
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setenv("ARTEMIS_SCREENSHOT_LIMIT_ENABLED", "false")
|
|
screenshot = load_agent_config(config_path).screenshot
|
|
assert screenshot.enabled is True
|
|
assert screenshot.max_long_edge == 1024
|
|
|
|
|
|
def test_failed_write_keeps_running_settings(monkeypatch, isolated_settings):
|
|
previous = settings.OPENAI_MODEL
|
|
monkeypatch.setattr(os, "replace", lambda *args: (_ for _ in ()).throw(OSError("read only")))
|
|
with pytest.raises(OSError):
|
|
service.save_model_settings(service.ModelSettingsUpdate(model="not-saved"))
|
|
assert settings.OPENAI_MODEL == previous
|
|
assert "not-saved" not in isolated_settings.read_text(encoding="utf-8")
|
|
|
|
|
|
def test_key_not_in_validation_errors(isolated_settings):
|
|
with pytest.raises(ValueError) as exc:
|
|
service.save_model_settings(
|
|
service.ModelSettingsUpdate(model="test", api_key="secret\nvalue")
|
|
)
|
|
assert "secret" not in str(exc.value)
|
|
|
|
|
|
def test_reasoning_retry_settings_roundtrip(isolated_settings):
|
|
result = service.save_model_settings(service.ModelSettingsUpdate(model="gpt-5.6-luna"))
|
|
assert result["timeout_seconds"] == 300
|
|
assert result["max_retries"] == 5
|
|
assert result["retry_interval_seconds"] == 10
|
|
assert result["reasoning_effort"] == "high"
|
|
saved = Settings(_env_file=isolated_settings)
|
|
assert saved.OPENAI_REASONING_EFFORT == "high"
|
|
assert parse_llm_config().operator.reasoning_effort == "high"
|
|
service.save_model_settings(
|
|
service.ModelSettingsUpdate(model="custom", reasoning_effort="default")
|
|
)
|
|
assert parse_llm_config().operator.reasoning_effort == "default"
|
|
|
|
|
|
def test_max_reasoning_and_custom_model_survive_reload(isolated_settings):
|
|
result = service.save_model_settings(
|
|
service.ModelSettingsUpdate(model="custom/model-v2", reasoning_effort="max")
|
|
)
|
|
assert result["model"] == "custom/model-v2"
|
|
assert result["reasoning_effort"] == "max"
|
|
assert dotenv_values(isolated_settings)["OPENAI_REASONING_EFFORT"] == "max"
|
|
reloaded = Settings(_env_file=isolated_settings)
|
|
assert reloaded.OPENAI_REASONING_EFFORT == "max"
|
|
assert reloaded.OPENAI_MODEL == "custom/model-v2"
|
|
assert parse_llm_config().operator.reasoning_effort == "max"
|
|
|
|
|
|
@pytest.mark.parametrize("effort", ["high", "max"])
|
|
def test_responses_payload_has_reasoning_and_no_sdk_retry(monkeypatch, effort):
|
|
from artemis.llm.router import ModelEndpoint, ModelFactory
|
|
from unittest.mock import Mock
|
|
|
|
factory = Mock()
|
|
monkeypatch.setattr("langchain_openai.ChatOpenAI", factory)
|
|
ModelFactory.create_model(
|
|
ModelEndpoint(api_key="test-only", reasoning_effort=effort, max_retries=5)
|
|
)
|
|
assert factory.call_args.kwargs["reasoning_effort"] == effort
|
|
assert factory.call_args.kwargs["max_retries"] == 0
|
|
assert factory.call_args.kwargs["use_responses_api"] is True
|
|
ModelFactory.create_model(ModelEndpoint(api_key="test-only", reasoning_effort="default"))
|
|
assert "reasoning_effort" not in factory.call_args.kwargs
|
|
|
|
|
|
def test_execution_protection_saves_reload_without_touching_secret(isolated_settings):
|
|
result = service.save_capability_settings(
|
|
service.CapabilitySettings(
|
|
task_timeout_seconds=3600,
|
|
observation_timeout_seconds=45,
|
|
observation_max_attempts=2,
|
|
device_command_timeout_seconds=8,
|
|
action_failure_attempts=4,
|
|
worker_cleanup_timeout_seconds=15,
|
|
)
|
|
)
|
|
reloaded = Settings(_env_file=isolated_settings)
|
|
assert reloaded.ARTEMIS_TASK_TIMEOUT_SECONDS == result.task_timeout_seconds == 3600
|
|
assert reloaded.ARTEMIS_OBSERVATION_TIMEOUT_SECONDS == 45
|
|
assert reloaded.ARTEMIS_OBSERVATION_MAX_ATTEMPTS == 2
|
|
assert reloaded.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS == 8
|
|
assert reloaded.ARTEMIS_ACTION_FAILURE_ATTEMPTS == 4
|
|
assert reloaded.ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS == 15
|
|
assert reloaded.OPENAI_API_KEY is None
|
|
assert settings.OPENAI_API_KEY.get_secret_value() == "existing-test-secret"
|
|
assert dotenv_values(isolated_settings)["UNRELATED"] == "keep"
|