chenmingxuan
83bcfd54d5
Default Flash final review to off and persist the setting across the UI, MCP, SDK and queued tasks. Keep execution separate from test verdicts. Bundle UI fonts and licenses to remove Google Fonts from startup builds. Align backend/frontend versions and document installation and upgrades.
280 lines
12 KiB
Python
280 lines
12 KiB
Python
import os
|
|
from pathlib import Path
|
|
|
|
from dotenv import dotenv_values
|
|
import pytest
|
|
from pydantic import SecretStr, ValidationError
|
|
|
|
from apps.admin_console.services import model_settings_service as service
|
|
from artemis.config.settings import Settings, settings
|
|
from artemis.config.agent import load_agent_config
|
|
from artemis.config.llm import parse_llm_config
|
|
|
|
|
|
@pytest.fixture
|
|
def isolated_settings(monkeypatch, tmp_path):
|
|
env = tmp_path / ".env"
|
|
env.write_text("# user settings\nADB_PORT=5037\nUNRELATED=keep\n", encoding="utf-8")
|
|
monkeypatch.setattr(service, "get_env_file", lambda: env)
|
|
keys = [
|
|
"OPENAI_API_KEY",
|
|
"OPENAI_MODEL",
|
|
"OPENAI_BASE_URL",
|
|
"OPENAI_TIMEOUT_SECONDS",
|
|
"OPENAI_MAX_RETRIES",
|
|
"OPENAI_RETRY_INTERVAL_SECONDS",
|
|
"OPENAI_RATE_LIMIT_RETRY_SECONDS",
|
|
"OPENAI_TOOL_FALLBACK_MODEL",
|
|
"OPENAI_TOOL_FALLBACK_REASONING_EFFORT",
|
|
"OPENAI_TOOL_FALLBACK_MAX_RETRIES",
|
|
"ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS",
|
|
"OPENAI_REASONING_EFFORT",
|
|
"ARTEMIS_VIDEO_ENABLED",
|
|
"ARTEMIS_OUTPUTTER_ENABLED",
|
|
"ARTEMIS_FLASH_FINAL_CHECK_ENABLED",
|
|
"ARTEMIS_STEP_SUMMARIZER_ENABLED",
|
|
"ARTEMIS_SCREENSHOT_LIMIT_ENABLED",
|
|
"ARTEMIS_TASK_TIMEOUT_SECONDS",
|
|
"ARTEMIS_OBSERVATION_TIMEOUT_SECONDS",
|
|
"ARTEMIS_OBSERVATION_MAX_ATTEMPTS",
|
|
"ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS",
|
|
"ARTEMIS_ACTION_FAILURE_ATTEMPTS",
|
|
"ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS",
|
|
]
|
|
for key in keys:
|
|
monkeypatch.setattr(settings, key, getattr(settings, key))
|
|
monkeypatch.delenv(key, raising=False)
|
|
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("existing-test-secret"))
|
|
return env
|
|
|
|
|
|
def test_save_reload_and_blank_key_preserves_secret(isolated_settings):
|
|
update = service.ModelSettingsUpdate(
|
|
base_url="https://gateway.example/v1/",
|
|
model="my-model",
|
|
api_key="new-test-secret",
|
|
timeout_seconds=75,
|
|
max_retries=2,
|
|
rate_limit_retry_seconds=90,
|
|
tool_fallback_model="custom/fallback",
|
|
tool_fallback_reasoning_effort="max",
|
|
tool_fallback_max_retries=2,
|
|
tool_fallback_no_progress_attempts=4,
|
|
)
|
|
result = service.save_model_settings(update)
|
|
assert result["api_key_configured"]
|
|
assert "new-test-secret" not in repr(result)
|
|
saved = dotenv_values(isolated_settings)
|
|
assert saved["OPENAI_API_KEY"] == "new-test-secret"
|
|
assert saved["OPENAI_MODEL"] == "my-model"
|
|
assert saved["OPENAI_RATE_LIMIT_RETRY_SECONDS"] == "90.0"
|
|
assert saved["OPENAI_TOOL_FALLBACK_MODEL"] == "custom/fallback"
|
|
assert saved["ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS"] == "4"
|
|
assert saved["UNRELATED"] == "keep"
|
|
assert "# user settings" in isolated_settings.read_text(encoding="utf-8")
|
|
reloaded = Settings(_env_file=isolated_settings)
|
|
assert reloaded.OPENAI_MODEL == "my-model"
|
|
assert parse_llm_config().operator.model == "my-model"
|
|
assert parse_llm_config().operator.timeout == 75
|
|
assert parse_llm_config().operator.fallback.model == "custom/fallback"
|
|
service.save_model_settings(update.model_copy(update={"api_key": SecretStr("")}))
|
|
assert dotenv_values(isolated_settings)["OPENAI_API_KEY"] == "new-test-secret"
|
|
|
|
|
|
def test_new_install_uses_playones_endpoint_without_overriding_saved_url(monkeypatch):
|
|
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
|
expected = "https://api-n-cd.playones.com/v1"
|
|
assert Settings(_env_file=None).OPENAI_BASE_URL == expected
|
|
assert service.ModelSettingsUpdate(model="test").base_url == expected
|
|
monkeypatch.setattr(settings, "OPENAI_BASE_URL", None)
|
|
assert service.get_model_settings()["base_url"] == expected
|
|
monkeypatch.setattr(settings, "OPENAI_BASE_URL", "https://saved.example/v1")
|
|
assert service.get_model_settings()["base_url"] == "https://saved.example/v1"
|
|
|
|
|
|
def test_model_presets_keep_supported_choices_without_restricting_custom_names():
|
|
presets = service.get_model_settings()["model_presets"]
|
|
assert presets == [
|
|
"gpt-6-luna",
|
|
"gpt-6.1-sol",
|
|
"gpt-6-astra",
|
|
"mimo-v2.6-pro",
|
|
]
|
|
assert service.ModelSettingsUpdate(model="custom/model-v2").model == "custom/model-v2"
|
|
|
|
|
|
def test_new_install_defaults_to_luna_high_and_preserves_saved_model(isolated_settings):
|
|
from artemis.config.constants import DEFAULT_MODEL
|
|
|
|
fresh = Settings(_env_file=None)
|
|
assert DEFAULT_MODEL == fresh.OPENAI_MODEL == "gpt-6-luna"
|
|
assert fresh.OPENAI_REASONING_EFFORT == "high"
|
|
isolated_settings.write_text("OPENAI_MODEL=custom/existing\n", encoding="utf-8")
|
|
assert Settings(_env_file=isolated_settings).OPENAI_MODEL == "custom/existing"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"url",
|
|
[
|
|
"ftp://example.com",
|
|
"https://user:pass@example.com",
|
|
"https://example.com/v1/responses",
|
|
"https://example.com/v1?key=secret",
|
|
],
|
|
)
|
|
def test_rejects_invalid_base_url(url):
|
|
with pytest.raises(ValidationError):
|
|
service.ModelSettingsUpdate(base_url=url, model="test")
|
|
|
|
|
|
def test_capabilities_persist_and_apply_to_new_tasks(isolated_settings):
|
|
result = service.save_capability_settings(
|
|
service.CapabilitySettings(
|
|
video_enabled=False,
|
|
outputter_enabled=False,
|
|
step_summarizer_enabled=False,
|
|
screenshot_limit_enabled=False,
|
|
)
|
|
)
|
|
assert result.video_enabled is False
|
|
saved = dotenv_values(isolated_settings)
|
|
assert saved["ARTEMIS_VIDEO_ENABLED"] == "false"
|
|
assert saved["ARTEMIS_SCREENSHOT_LIMIT_ENABLED"] == "false"
|
|
config = load_agent_config()
|
|
assert config.video_analyzer.enabled is False
|
|
assert config.pro.video_analyzer.enabled is False
|
|
assert config.outputter.enabled is False
|
|
assert config.flash.step_summarizer.enabled is False
|
|
assert config.screenshot.enabled is False
|
|
service.save_capability_settings(service.CapabilitySettings(video_enabled=True))
|
|
assert load_agent_config().video_analyzer.enabled is True
|
|
|
|
|
|
def test_flash_review_defaults_off_and_persists_without_changing_pro(isolated_settings):
|
|
assert Settings(_env_file=None).ARTEMIS_FLASH_FINAL_CHECK_ENABLED is False
|
|
assert service.get_capability_settings().flash_final_check_enabled is False
|
|
assert load_agent_config().checker.final_check is True
|
|
for enabled in (True, False):
|
|
saved = service.save_capability_settings(
|
|
service.CapabilitySettings(flash_final_check_enabled=enabled)
|
|
)
|
|
assert saved.flash_final_check_enabled is enabled
|
|
assert Settings(_env_file=isolated_settings).ARTEMIS_FLASH_FINAL_CHECK_ENABLED is enabled
|
|
config = load_agent_config()
|
|
assert config.flash.final_check_enabled is enabled
|
|
assert config.checker.final_check is True
|
|
assert config.pro.checker.final_check is True
|
|
# Older clients omit the new switch; saving another setting keeps it.
|
|
assert (
|
|
service.save_capability_settings(
|
|
service.CapabilitySettings(recording_enabled=False)
|
|
).flash_final_check_enabled
|
|
is enabled
|
|
)
|
|
|
|
|
|
def test_old_checker_config_does_not_enable_flash_review(monkeypatch, tmp_path):
|
|
monkeypatch.delenv("ARTEMIS_FLASH_FINAL_CHECK_ENABLED", raising=False)
|
|
path = tmp_path / "artemis.jsonc"
|
|
path.write_text('{"agent":{"checker":{"enabled":true,"final_check":true}}}', encoding="utf-8")
|
|
assert load_agent_config(path).flash.final_check_enabled is False
|
|
|
|
|
|
def test_screenshot_json_can_override_environment(monkeypatch, tmp_path):
|
|
config_path = tmp_path / "artemis.jsonc"
|
|
config_path.write_text(
|
|
'{"agent": {"screenshot": {"enabled": true, "max_long_edge": 1024}}}',
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setenv("ARTEMIS_SCREENSHOT_LIMIT_ENABLED", "false")
|
|
screenshot = load_agent_config(config_path).screenshot
|
|
assert screenshot.enabled is True
|
|
assert screenshot.max_long_edge == 1024
|
|
|
|
|
|
def test_failed_write_keeps_running_settings(monkeypatch, isolated_settings):
|
|
previous = settings.OPENAI_MODEL
|
|
monkeypatch.setattr(
|
|
service.os, "replace", lambda *args: (_ for _ in ()).throw(OSError("read only"))
|
|
)
|
|
with pytest.raises(OSError):
|
|
service.save_model_settings(service.ModelSettingsUpdate(model="not-saved"))
|
|
assert settings.OPENAI_MODEL == previous
|
|
assert "not-saved" not in isolated_settings.read_text(encoding="utf-8")
|
|
|
|
|
|
def test_key_not_in_validation_errors(isolated_settings):
|
|
with pytest.raises(ValueError) as exc:
|
|
service.save_model_settings(
|
|
service.ModelSettingsUpdate(model="test", api_key="secret\nvalue")
|
|
)
|
|
assert "secret" not in str(exc.value)
|
|
|
|
|
|
def test_reasoning_retry_settings_roundtrip(isolated_settings):
|
|
result = service.save_model_settings(service.ModelSettingsUpdate(model="gpt-5.6-luna"))
|
|
assert result["timeout_seconds"] == 300
|
|
assert result["max_retries"] == 5
|
|
assert result["retry_interval_seconds"] == 10
|
|
assert result["reasoning_effort"] == "high"
|
|
saved = Settings(_env_file=isolated_settings)
|
|
assert saved.OPENAI_REASONING_EFFORT == "high"
|
|
assert parse_llm_config().operator.reasoning_effort == "high"
|
|
service.save_model_settings(
|
|
service.ModelSettingsUpdate(model="custom", reasoning_effort="default")
|
|
)
|
|
assert parse_llm_config().operator.reasoning_effort == "default"
|
|
|
|
|
|
def test_max_reasoning_and_custom_model_survive_reload(isolated_settings):
|
|
result = service.save_model_settings(
|
|
service.ModelSettingsUpdate(model="custom/model-v2", reasoning_effort="max")
|
|
)
|
|
assert result["model"] == "custom/model-v2"
|
|
assert result["reasoning_effort"] == "max"
|
|
assert dotenv_values(isolated_settings)["OPENAI_REASONING_EFFORT"] == "max"
|
|
reloaded = Settings(_env_file=isolated_settings)
|
|
assert reloaded.OPENAI_REASONING_EFFORT == "max"
|
|
assert reloaded.OPENAI_MODEL == "custom/model-v2"
|
|
assert parse_llm_config().operator.reasoning_effort == "max"
|
|
|
|
|
|
@pytest.mark.parametrize("effort", ["high", "max"])
|
|
def test_responses_payload_has_reasoning_and_no_sdk_retry(monkeypatch, effort):
|
|
from artemis.llm.router import ModelEndpoint, ModelFactory
|
|
from unittest.mock import Mock
|
|
|
|
factory = Mock()
|
|
monkeypatch.setattr("langchain_openai.ChatOpenAI", factory)
|
|
ModelFactory.create_model(
|
|
ModelEndpoint(api_key="test-only", reasoning_effort=effort, max_retries=5)
|
|
)
|
|
assert factory.call_args.kwargs["reasoning_effort"] == effort
|
|
assert factory.call_args.kwargs["max_retries"] == 0
|
|
assert factory.call_args.kwargs["use_responses_api"] is True
|
|
ModelFactory.create_model(ModelEndpoint(api_key="test-only", reasoning_effort="default"))
|
|
assert "reasoning_effort" not in factory.call_args.kwargs
|
|
|
|
|
|
def test_execution_protection_saves_reload_without_touching_secret(isolated_settings):
|
|
result = service.save_capability_settings(
|
|
service.CapabilitySettings(
|
|
task_timeout_seconds=3600,
|
|
observation_timeout_seconds=45,
|
|
observation_max_attempts=2,
|
|
device_command_timeout_seconds=8,
|
|
action_failure_attempts=4,
|
|
worker_cleanup_timeout_seconds=15,
|
|
)
|
|
)
|
|
reloaded = Settings(_env_file=isolated_settings)
|
|
assert reloaded.ARTEMIS_TASK_TIMEOUT_SECONDS == result.task_timeout_seconds == 3600
|
|
assert reloaded.ARTEMIS_OBSERVATION_TIMEOUT_SECONDS == 45
|
|
assert reloaded.ARTEMIS_OBSERVATION_MAX_ATTEMPTS == 2
|
|
assert reloaded.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS == 8
|
|
assert reloaded.ARTEMIS_ACTION_FAILURE_ATTEMPTS == 4
|
|
assert reloaded.ARTEMIS_WORKER_CLEANUP_TIMEOUT_SECONDS == 15
|
|
assert reloaded.OPENAI_API_KEY is None
|
|
assert settings.OPENAI_API_KEY.get_secret_value() == "existing-test-secret"
|
|
assert dotenv_values(isolated_settings)["UNRELATED"] == "keep"
|