import os from pathlib import Path from dotenv import dotenv_values import pytest from pydantic import SecretStr, ValidationError from apps.admin_console.services import model_settings_service as service from artemis.config.settings import Settings, settings from artemis.config.agent import load_agent_config from artemis.config.llm import parse_llm_config @pytest.fixture def isolated_settings(monkeypatch, tmp_path): env = tmp_path / ".env" env.write_text("# user settings\nADB_PORT=5037\nUNRELATED=keep\n", encoding="utf-8") monkeypatch.setattr(service, "get_env_file", lambda: env) keys = [ "OPENAI_API_KEY", "OPENAI_MODEL", "OPENAI_BASE_URL", "OPENAI_TIMEOUT_SECONDS", "OPENAI_MAX_RETRIES", "OPENAI_RETRY_INTERVAL_SECONDS", "OPENAI_RATE_LIMIT_RETRY_SECONDS", "OPENAI_TOOL_FALLBACK_MODEL", "OPENAI_TOOL_FALLBACK_REASONING_EFFORT", "OPENAI_TOOL_FALLBACK_MAX_RETRIES", "ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS", "OPENAI_REASONING_EFFORT", "ARTEMIS_VIDEO_ENABLED", "ARTEMIS_OUTPUTTER_ENABLED", "ARTEMIS_FLASH_FINAL_CHECK_ENABLED", "ARTEMIS_STEP_SUMMARIZER_ENABLED", "ARTEMIS_SCREENSHOT_LIMIT_ENABLED", "ARTEMIS_TASK_TIMEOUT_SECONDS", "ARTEMIS_OBSERVATION_TIMEOUT_SECONDS", "ARTEMIS_OBSERVATION_MAX_ATTEMPTS", "ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS", "ARTEMIS_ACTION_FAILURE_ATTEMPTS", "ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS", ] for key in keys: monkeypatch.setattr(settings, key, getattr(settings, key)) monkeypatch.delenv(key, raising=False) monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("existing-test-secret")) return env def test_save_reload_and_blank_key_preserves_secret(isolated_settings): update = service.ModelSettingsUpdate( base_url="https://gateway.example/v1/", model="my-model", api_key="new-test-secret", timeout_seconds=75, max_retries=2, rate_limit_retry_seconds=90, tool_fallback_model="custom/fallback", tool_fallback_reasoning_effort="max", tool_fallback_max_retries=2, tool_fallback_no_progress_attempts=4, ) result = service.save_model_settings(update) assert result["api_key_configured"] assert "new-test-secret" not in repr(result) saved = dotenv_values(isolated_settings) assert saved["OPENAI_API_KEY"] == "new-test-secret" assert saved["OPENAI_MODEL"] == "my-model" assert saved["OPENAI_RATE_LIMIT_RETRY_SECONDS"] == "90.0" assert saved["OPENAI_TOOL_FALLBACK_MODEL"] == "custom/fallback" assert saved["ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS"] == "4" assert saved["UNRELATED"] == "keep" assert "# user settings" in isolated_settings.read_text(encoding="utf-8") reloaded = Settings(_env_file=isolated_settings) assert reloaded.OPENAI_MODEL == "my-model" assert parse_llm_config().operator.model == "my-model" assert parse_llm_config().operator.timeout == 75 assert parse_llm_config().operator.fallback.model == "custom/fallback" service.save_model_settings(update.model_copy(update={"api_key": SecretStr("")})) assert dotenv_values(isolated_settings)["OPENAI_API_KEY"] == "new-test-secret" def test_new_install_uses_playones_endpoint_without_overriding_saved_url(monkeypatch): monkeypatch.delenv("OPENAI_BASE_URL", raising=False) expected = "https://api-n-cd.playones.com/v1" assert Settings(_env_file=None).OPENAI_BASE_URL == expected assert service.ModelSettingsUpdate(model="test").base_url == expected monkeypatch.setattr(settings, "OPENAI_BASE_URL", None) assert service.get_model_settings()["base_url"] == expected monkeypatch.setattr(settings, "OPENAI_BASE_URL", "https://saved.example/v1") assert service.get_model_settings()["base_url"] == "https://saved.example/v1" def test_model_presets_keep_supported_choices_without_restricting_custom_names(): presets = service.get_model_settings()["model_presets"] assert presets == [ "gpt-6-luna", "gpt-6.1-sol", "gpt-6-astra", "mimo-v2.6-pro", ] assert service.ModelSettingsUpdate(model="custom/model-v2").model == "custom/model-v2" def test_new_install_defaults_to_luna_high_and_preserves_saved_model(isolated_settings): from artemis.config.constants import DEFAULT_MODEL fresh = Settings(_env_file=None) assert DEFAULT_MODEL == fresh.OPENAI_MODEL == "gpt-6-luna" assert fresh.OPENAI_REASONING_EFFORT == "high" isolated_settings.write_text("OPENAI_MODEL=custom/existing\n", encoding="utf-8") assert Settings(_env_file=isolated_settings).OPENAI_MODEL == "custom/existing" @pytest.mark.parametrize( "url", [ "ftp://example.com", "https://user:pass@example.com", "https://example.com/v1/responses", "https://example.com/v1?key=secret", ], ) def test_rejects_invalid_base_url(url): with pytest.raises(ValidationError): service.ModelSettingsUpdate(base_url=url, model="test") def test_capabilities_persist_and_apply_to_new_tasks(isolated_settings): result = service.save_capability_settings( service.CapabilitySettings( video_enabled=False, outputter_enabled=False, step_summarizer_enabled=False, screenshot_limit_enabled=False, ) ) assert result.video_enabled is False saved = dotenv_values(isolated_settings) assert saved["ARTEMIS_VIDEO_ENABLED"] == "false" assert saved["ARTEMIS_SCREENSHOT_LIMIT_ENABLED"] == "false" config = load_agent_config() assert config.video_analyzer.enabled is False assert config.pro.video_analyzer.enabled is False assert config.outputter.enabled is False assert config.flash.step_summarizer.enabled is False assert config.screenshot.enabled is False service.save_capability_settings(service.CapabilitySettings(video_enabled=True)) assert load_agent_config().video_analyzer.enabled is True def test_flash_review_defaults_off_and_persists_without_changing_pro(isolated_settings): assert Settings(_env_file=None).ARTEMIS_FLASH_FINAL_CHECK_ENABLED is False assert service.get_capability_settings().flash_final_check_enabled is False assert load_agent_config().checker.final_check is True for enabled in (True, False): saved = service.save_capability_settings( service.CapabilitySettings(flash_final_check_enabled=enabled) ) assert saved.flash_final_check_enabled is enabled assert Settings(_env_file=isolated_settings).ARTEMIS_FLASH_FINAL_CHECK_ENABLED is enabled config = load_agent_config() assert config.flash.final_check_enabled is enabled assert config.checker.final_check is True assert config.pro.checker.final_check is True # Older clients omit the new switch; saving another setting keeps it. assert ( service.save_capability_settings( service.CapabilitySettings(recording_enabled=False) ).flash_final_check_enabled is enabled ) def test_old_checker_config_does_not_enable_flash_review(monkeypatch, tmp_path): monkeypatch.delenv("ARTEMIS_FLASH_FINAL_CHECK_ENABLED", raising=False) path = tmp_path / "artemis.jsonc" path.write_text('{"agent":{"checker":{"enabled":true,"final_check":true}}}', encoding="utf-8") assert load_agent_config(path).flash.final_check_enabled is False @pytest.mark.parametrize("enabled", [True, False]) def test_saved_flash_switch_reaches_next_worker_without_restarting( isolated_settings, monkeypatch, enabled ): from apps.admin_console.services.execution_config import execution_config from apps.admin_console.services.task_queue_service import TaskQueueService monkeypatch.setattr("artemis.exploration.store.explorations.run", lambda _sid: None) monkeypatch.setattr("apps.admin_console.services.automation_store.store.job", lambda _sid: None) monkeypatch.setenv("ARTEMIS_FLASH_FINAL_CHECK_ENABLED", str(not enabled).lower()) running = {"profile": "flash", "goal": "already running", "session_id": "running-review"} running["execution_config"] = execution_config(running) queued = {"profile": "flash", "goal": "queued", "session_id": "queued-review"} queued["execution_config"] = execution_config(queued) service.save_capability_settings(service.CapabilitySettings(flash_final_check_enabled=enabled)) # The real queue refreshes pending snapshots just before starting their worker. queued["execution_config"] = execution_config(queued) for item, expected in ((running, not enabled), (queued, enabled)): _cmd, env = TaskQueueService._build_worker_invocation( item, item["session_id"], item["session_id"], item["goal"], "flash", TaskQueueService._task_target(item), ) assert env["ARTEMIS_FLASH_FINAL_CHECK_ENABLED"] == str(expected).lower() assert load_agent_config().pro.checker.final_check is True def test_screenshot_json_can_override_environment(monkeypatch, tmp_path): config_path = tmp_path / "artemis.jsonc" config_path.write_text( '{"agent": {"screenshot": {"enabled": true, "max_long_edge": 1024}}}', encoding="utf-8", ) monkeypatch.setenv("ARTEMIS_SCREENSHOT_LIMIT_ENABLED", "false") screenshot = load_agent_config(config_path).screenshot assert screenshot.enabled is True assert screenshot.max_long_edge == 1024 def test_failed_write_keeps_running_settings(monkeypatch, isolated_settings): previous = settings.OPENAI_MODEL monkeypatch.setattr( service.os, "replace", lambda *args: (_ for _ in ()).throw(OSError("read only")) ) with pytest.raises(OSError): service.save_model_settings(service.ModelSettingsUpdate(model="not-saved")) assert settings.OPENAI_MODEL == previous assert "not-saved" not in isolated_settings.read_text(encoding="utf-8") def test_key_not_in_validation_errors(isolated_settings): with pytest.raises(ValueError) as exc: service.save_model_settings( service.ModelSettingsUpdate(model="test", api_key="secret\nvalue") ) assert "secret" not in str(exc.value) def test_reasoning_retry_settings_roundtrip(isolated_settings): result = service.save_model_settings(service.ModelSettingsUpdate(model="gpt-5.6-luna")) assert result["timeout_seconds"] == 300 assert result["max_retries"] == 5 assert result["retry_interval_seconds"] == 10 assert result["reasoning_effort"] == "high" saved = Settings(_env_file=isolated_settings) assert saved.OPENAI_REASONING_EFFORT == "high" assert parse_llm_config().operator.reasoning_effort == "high" service.save_model_settings( service.ModelSettingsUpdate(model="custom", reasoning_effort="default") ) assert parse_llm_config().operator.reasoning_effort == "default" def test_max_reasoning_and_custom_model_survive_reload(isolated_settings): result = service.save_model_settings( service.ModelSettingsUpdate(model="custom/model-v2", reasoning_effort="max") ) assert result["model"] == "custom/model-v2" assert result["reasoning_effort"] == "max" assert dotenv_values(isolated_settings)["OPENAI_REASONING_EFFORT"] == "max" reloaded = Settings(_env_file=isolated_settings) assert reloaded.OPENAI_REASONING_EFFORT == "max" assert reloaded.OPENAI_MODEL == "custom/model-v2" assert parse_llm_config().operator.reasoning_effort == "max" @pytest.mark.parametrize("effort", ["high", "max"]) def test_responses_payload_has_reasoning_and_no_sdk_retry(monkeypatch, effort): from artemis.llm.router import ModelEndpoint, ModelFactory from unittest.mock import Mock factory = Mock() monkeypatch.setattr("langchain_openai.ChatOpenAI", factory) ModelFactory.create_model( ModelEndpoint(api_key="test-only", reasoning_effort=effort, max_retries=5) ) assert factory.call_args.kwargs["reasoning_effort"] == effort assert factory.call_args.kwargs["max_retries"] == 0 assert factory.call_args.kwargs["use_responses_api"] is True ModelFactory.create_model(ModelEndpoint(api_key="test-only", reasoning_effort="default")) assert "reasoning_effort" not in factory.call_args.kwargs def test_execution_protection_saves_reload_without_touching_secret(isolated_settings): result = service.save_capability_settings( service.CapabilitySettings( task_timeout_seconds=3600, observation_timeout_seconds=45, observation_max_attempts=2, device_command_timeout_seconds=8, action_failure_attempts=4, worker_cleanup_timeout_seconds=15, ) ) reloaded = Settings(_env_file=isolated_settings) assert reloaded.ARTEMIS_TASK_TIMEOUT_SECONDS == result.task_timeout_seconds == 3600 assert reloaded.ARTEMIS_OBSERVATION_TIMEOUT_SECONDS == 45 assert reloaded.ARTEMIS_OBSERVATION_MAX_ATTEMPTS == 2 assert reloaded.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS == 8 assert reloaded.ARTEMIS_ACTION_FAILURE_ATTEMPTS == 4 assert reloaded.ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS == 15 assert reloaded.OPENAI_API_KEY is None assert settings.OPENAI_API_KEY.get_secret_value() == "existing-test-secret" assert dotenv_values(isolated_settings)["UNRELATED"] == "keep"