feat: add Chinese UI, Responses models, and mirrored Windows setup
Localize the web console and persist custom model and capability settings. Use OpenAI Responses exclusively and move runtime limits into environment configuration. Bootstrap uv and the virtual environment on Windows, support optional China mirrors, and package source distributions without environments, build output, or credentials. Update regression tests and repository quality checks for the new behavior.
This commit is contained in:
1 parent
371aa6df56
commit
2ddec53954
183 files changed
+3427
-11857
No files matched your search
@@ -53,10 +53,10 @@ def test_submit_answer_tool_subclass():
|
||||
assert submit_answer.args_schema == SubmitAnswerArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = submit_answer.to_genai_declaration()
|
||||
declaration = submit_answer.to_tool_declaration()
|
||||
assert declaration.name == "submit_answer"
|
||||
assert "analysis" in declaration.parameters.properties
|
||||
assert "actionable_steps" in declaration.parameters.properties
|
||||
assert "analysis" in declaration.parameters["properties"]
|
||||
assert "actionable_steps" in declaration.parameters["properties"]
|
||||
|
||||
|
||||
def test_ask_diagnoser_tool_subclass():
|
||||
@@ -81,9 +81,9 @@ def test_ask_diagnoser_tool_subclass():
|
||||
assert ask_diagnoser.args_schema == AskDiagnoserArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = ask_diagnoser.to_genai_declaration()
|
||||
declaration = ask_diagnoser.to_tool_declaration()
|
||||
assert declaration.name == "ask_diagnoser"
|
||||
assert "query" in declaration.parameters.properties
|
||||
assert "query" in declaration.parameters["properties"]
|
||||
|
||||
|
||||
def test_analyze_logs_tool_subclass():
|
||||
@@ -108,6 +108,6 @@ def test_analyze_logs_tool_subclass():
|
||||
assert analyze_logs.args_schema == AnalyzeLogsArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = analyze_logs.to_genai_declaration()
|
||||
declaration = analyze_logs.to_tool_declaration()
|
||||
assert declaration.name == "analyze_logs"
|
||||
assert "specific_query" in declaration.parameters.properties
|
||||
assert "specific_query" in declaration.parameters["properties"]
|
||||
@@ -37,43 +37,7 @@ async def test_video_analyzer_tools(artemis_context, mock_state, inputs_dir, tmp
|
||||
agent.blackboard_entries = []
|
||||
agent.sub_system_prompt = "You are a sub agent"
|
||||
agent.audio_system_prompt = "You are an audio sub agent"
|
||||
agent.model_name = "gemini-3.7-flash"
|
||||
|
||||
# We must provide a valid FunctionDeclaration for the tool otherwise Pydantic will complain
|
||||
from google.genai import types
|
||||
|
||||
agent.submit_answer_tool = types.FunctionDeclaration(
|
||||
name="submit_answer",
|
||||
description=("Submit the final findings for the requested segment analysis."),
|
||||
parameters=types.Schema(
|
||||
type=types.Type.OBJECT,
|
||||
properties={
|
||||
"summary": types.Schema(type=types.Type.STRING, description="summary"),
|
||||
"analysis": types.Schema(type=types.Type.STRING, description="analysis"),
|
||||
"timeline_events": types.Schema(
|
||||
type=types.Type.ARRAY,
|
||||
items=types.Schema(
|
||||
type=types.Type.OBJECT,
|
||||
properties={
|
||||
"start_time": types.Schema(type=types.Type.NUMBER),
|
||||
"end_time": types.Schema(type=types.Type.NUMBER),
|
||||
"transcription": types.Schema(type=types.Type.STRING),
|
||||
"confidence_score": types.Schema(type=types.Type.NUMBER),
|
||||
"verification_timestamp_secs": types.Schema(type=types.Type.NUMBER),
|
||||
},
|
||||
),
|
||||
),
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
import google.genai as genai
|
||||
from artemis.config import settings
|
||||
|
||||
agent.client = genai.Client(
|
||||
api_key=settings.GOOGLE_API_KEY.get_secret_value() if settings.GOOGLE_API_KEY else None
|
||||
)
|
||||
artemis_context._genai_client = agent.client
|
||||
agent.model_name = "mimo-v2.6-flash"
|
||||
|
||||
import shutil
|
||||
import uuid
|
||||
|
||||
@@ -25,7 +25,7 @@ def test_get_active_model_info_pro_architecture():
|
||||
info = ModelService.get_active_model_info("pro")
|
||||
assert info["name"] == "Pro"
|
||||
assert info["architecture"] == "ARTEMIS Pro"
|
||||
assert info["provider"] == "google"
|
||||
assert info["provider"] == "openai"
|
||||
assert "id" in info
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ def test_get_active_model_info_flash_architecture():
|
||||
info = ModelService.get_active_model_info("flash")
|
||||
assert info["name"] == "Flash"
|
||||
assert info["architecture"] == "ARTEMIS Flash"
|
||||
assert info["provider"] == "google"
|
||||
assert info["provider"] == "openai"
|
||||
|
||||
|
||||
def test_resolve_session_profile_from_device_info():
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import dotenv_values
|
||||
import pytest
|
||||
from pydantic import SecretStr, ValidationError
|
||||
|
||||
from apps.admin_console.services import model_settings_service as service
|
||||
from artemis.config.settings import Settings, settings
|
||||
from artemis.config.agent import load_agent_config
|
||||
from artemis.config.llm import parse_llm_config
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_settings(monkeypatch, tmp_path):
|
||||
env = tmp_path / ".env"
|
||||
env.write_text("# user settings\nADB_PORT=5037\nUNRELATED=keep\n", encoding="utf-8")
|
||||
monkeypatch.setattr(service, "get_env_file", lambda: env)
|
||||
keys = [
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_MODEL",
|
||||
"OPENAI_BASE_URL",
|
||||
"OPENAI_TIMEOUT_SECONDS",
|
||||
"OPENAI_MAX_RETRIES",
|
||||
"ARTEMIS_VIDEO_ENABLED",
|
||||
"ARTEMIS_OUTPUTTER_ENABLED",
|
||||
"ARTEMIS_STEP_SUMMARIZER_ENABLED",
|
||||
]
|
||||
for key in keys:
|
||||
monkeypatch.setattr(settings, key, getattr(settings, key))
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("existing-test-secret"))
|
||||
return env
|
||||
|
||||
|
||||
def test_save_reload_and_blank_key_preserves_secret(isolated_settings):
|
||||
update = service.ModelSettingsUpdate(
|
||||
base_url="https://gateway.example/v1/",
|
||||
model="my-model",
|
||||
api_key="new-test-secret",
|
||||
timeout_seconds=75,
|
||||
max_retries=2,
|
||||
)
|
||||
result = service.save_model_settings(update)
|
||||
assert result["api_key_configured"]
|
||||
assert "new-test-secret" not in repr(result)
|
||||
saved = dotenv_values(isolated_settings)
|
||||
assert saved["OPENAI_API_KEY"] == "new-test-secret"
|
||||
assert saved["OPENAI_MODEL"] == "my-model"
|
||||
assert saved["UNRELATED"] == "keep"
|
||||
assert "# user settings" in isolated_settings.read_text(encoding="utf-8")
|
||||
reloaded = Settings(_env_file=isolated_settings)
|
||||
assert reloaded.OPENAI_MODEL == "my-model"
|
||||
assert parse_llm_config().operator.model == "my-model"
|
||||
assert parse_llm_config().operator.timeout == 75
|
||||
service.save_model_settings(update.model_copy(update={"api_key": SecretStr("")}))
|
||||
assert dotenv_values(isolated_settings)["OPENAI_API_KEY"] == "new-test-secret"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url",
|
||||
[
|
||||
"ftp://example.com",
|
||||
"https://user:pass@example.com",
|
||||
"https://example.com/v1/responses",
|
||||
"https://example.com/v1?key=secret",
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid_base_url(url):
|
||||
with pytest.raises(ValidationError):
|
||||
service.ModelSettingsUpdate(base_url=url, model="test")
|
||||
|
||||
|
||||
def test_capabilities_persist_and_apply_to_new_tasks(isolated_settings):
|
||||
result = service.save_capability_settings(
|
||||
service.CapabilitySettings(
|
||||
video_enabled=False, outputter_enabled=False, step_summarizer_enabled=False
|
||||
)
|
||||
)
|
||||
assert result.video_enabled is False
|
||||
saved = dotenv_values(isolated_settings)
|
||||
assert saved["ARTEMIS_VIDEO_ENABLED"] == "false"
|
||||
config = load_agent_config()
|
||||
assert config.video_analyzer.enabled is False
|
||||
assert config.pro.video_analyzer.enabled is False
|
||||
assert config.outputter.enabled is False
|
||||
assert config.flash.step_summarizer.enabled is False
|
||||
service.save_capability_settings(service.CapabilitySettings())
|
||||
assert load_agent_config().video_analyzer.enabled is True
|
||||
|
||||
|
||||
def test_failed_write_keeps_running_settings(monkeypatch, isolated_settings):
|
||||
previous = settings.OPENAI_MODEL
|
||||
monkeypatch.setattr(
|
||||
service.os, "replace", lambda *args: (_ for _ in ()).throw(OSError("read only"))
|
||||
)
|
||||
with pytest.raises(OSError):
|
||||
service.save_model_settings(service.ModelSettingsUpdate(model="not-saved"))
|
||||
assert settings.OPENAI_MODEL == previous
|
||||
assert "not-saved" not in isolated_settings.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_key_not_in_validation_errors(isolated_settings):
|
||||
with pytest.raises(ValueError) as exc:
|
||||
service.save_model_settings(
|
||||
service.ModelSettingsUpdate(model="test", api_key="secret\nvalue")
|
||||
)
|
||||
assert "secret" not in str(exc.value)
|
||||
@@ -57,7 +57,7 @@ async def test_credentials_endpoint_never_returns_key_material(monkeypatch):
|
||||
assert res.status_code == 200
|
||||
assert honeytoken not in res.text
|
||||
providers = {entry["name"]: entry["configured"] for entry in res.json()["providers"]}
|
||||
assert providers.get("google") is True
|
||||
assert providers.get("openai") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -56,7 +56,7 @@ def test_retrying_llm_payload_keeps_user_visible_retry_details():
|
||||
{
|
||||
"error": "503 UNAVAILABLE: high demand",
|
||||
"delay": 1.18,
|
||||
"provider": "google",
|
||||
"provider": "openai",
|
||||
"source": "provider_sdk",
|
||||
"recoverable": True,
|
||||
"internal_noise": "do not expose",
|
||||
@@ -69,7 +69,7 @@ def test_retrying_llm_payload_keeps_user_visible_retry_details():
|
||||
assert normalized["payload"] == {
|
||||
"error": "503 UNAVAILABLE: high demand",
|
||||
"delay": 1.18,
|
||||
"provider": "google",
|
||||
"provider": "openai",
|
||||
"source": "provider_sdk",
|
||||
"recoverable": True,
|
||||
}
|
||||
@@ -81,7 +81,7 @@ def test_terminal_llm_payload_keeps_wait_and_retry_aggregation():
|
||||
{
|
||||
"delay": 5.0,
|
||||
"scheduled_at": 100.0,
|
||||
"provider": "google",
|
||||
"provider": "openai",
|
||||
"source": "provider_sdk",
|
||||
}
|
||||
]
|
||||
@@ -94,7 +94,7 @@ def test_terminal_llm_payload_keeps_wait_and_retry_aggregation():
|
||||
"error": "503 UNAVAILABLE",
|
||||
"pause": True,
|
||||
"request_id": "request-1",
|
||||
"provider": "google",
|
||||
"provider": "openai",
|
||||
"waited_seconds": 18.5,
|
||||
"retries": retries,
|
||||
"messages": ["private prompt"],
|
||||
@@ -106,7 +106,7 @@ def test_terminal_llm_payload_keeps_wait_and_retry_aggregation():
|
||||
|
||||
assert normalized["payload"] == {
|
||||
"error": "503 UNAVAILABLE",
|
||||
"provider": "google",
|
||||
"provider": "openai",
|
||||
"pause": True,
|
||||
"request_id": "request-1",
|
||||
"waited_seconds": 18.5,
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -301,34 +301,6 @@ def test_enrich_candidates_withholds_degenerate_bounds():
|
||||
assert explorer._enrich_candidates([cand]) == [cand]
|
||||
|
||||
|
||||
def test_native_submit_enriches_registered_candidates():
|
||||
explorer = _indexed_explorer()
|
||||
explorer._register_label("O2", explorer.screen_index.ocr_elements[0])
|
||||
call = MagicMock()
|
||||
call.name = "submit_answer"
|
||||
call.args = {
|
||||
"candidates": [
|
||||
{"label": "O2", "coords": [185, 54], "description": "header"},
|
||||
{"label": "D3", "coords": [900, 100], "description": "icon"},
|
||||
],
|
||||
"fallback_message": "",
|
||||
}
|
||||
turn_record = {"tool_calls": []}
|
||||
|
||||
raw = explorer._process_submit_answer([call], turn_record, [])
|
||||
|
||||
data = json.loads(raw)
|
||||
assert list(data) == ["candidates", "fallback_message"]
|
||||
assert data["candidates"][0] == {
|
||||
"label": "O2",
|
||||
"coords": [185, 54],
|
||||
"description": "header",
|
||||
"bounds": [92, 41, 277, 66],
|
||||
"source": "ocr",
|
||||
}
|
||||
assert data["candidates"][1] == call.args["candidates"][1]
|
||||
|
||||
|
||||
def test_universal_submit_enriches_registered_candidates():
|
||||
explorer = _indexed_explorer()
|
||||
explorer._register_label("1", explorer.screen_index.exact_matches("Settings")[0])
|
||||
|
||||
@@ -18,7 +18,6 @@ import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from google.genai import types
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
import pytest
|
||||
|
||||
@@ -61,36 +60,6 @@ def _state(screenshot: str) -> MagicMock:
|
||||
return state
|
||||
|
||||
|
||||
def _submit_response(coords=(500, 600)) -> MagicMock:
|
||||
call = MagicMock()
|
||||
call.name = "submit_answer"
|
||||
call.args = {
|
||||
"candidates": [{"label": "D1", "coords": list(coords), "description": "target"}],
|
||||
"fallback_message": "",
|
||||
}
|
||||
response = MagicMock()
|
||||
response.function_calls = [call]
|
||||
response.candidates = [MagicMock()]
|
||||
response.candidates[0].content = types.Content(
|
||||
role="model",
|
||||
parts=[types.Part.from_function_call(name="submit_answer", args=call.args)],
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def _native_client(responses) -> MagicMock:
|
||||
client = MagicMock()
|
||||
client.aio.models.generate_content = AsyncMock(side_effect=list(responses))
|
||||
cache = SimpleNamespace(name="cachedContents/abc")
|
||||
client.aio.caches.create = AsyncMock(return_value=cache)
|
||||
client.aio.caches.delete = AsyncMock()
|
||||
client.aio.files.upload = AsyncMock(
|
||||
return_value=SimpleNamespace(uri="files/1", mime_type="image/jpeg", name="files/1")
|
||||
)
|
||||
client.aio.files.delete = AsyncMock()
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def screenshot(tmp_path):
|
||||
path = tmp_path / "screen.jpg"
|
||||
@@ -110,9 +79,7 @@ def _isolated_env(monkeypatch):
|
||||
|
||||
|
||||
def test_construction_creates_no_genai_client():
|
||||
with patch(f"{EXPLORER}.genai.Client") as client_cls:
|
||||
explorer = Explorer(_context())
|
||||
client_cls.assert_not_called()
|
||||
explorer = Explorer(_context())
|
||||
assert explorer.client is None
|
||||
assert explorer.tier is None
|
||||
|
||||
@@ -128,7 +95,7 @@ def test_tier_sets_turn_budget_and_exposed_tools(version, max_turns, expected_to
|
||||
explorer = Explorer(_context())
|
||||
with patch(f"{EXPLORER}.is_ocr_configured", return_value=True):
|
||||
explorer._apply_tier(get_tier(version))
|
||||
native_names = {t.name for t in explorer.get_exposed_tools()}
|
||||
native_names = {t["function"]["name"] for t in explorer.get_exposed_tools()}
|
||||
universal_names = {t["function"]["name"] for t in explorer._universal_exposed_tools()}
|
||||
|
||||
assert explorer.max_turns == max_turns
|
||||
@@ -141,7 +108,7 @@ def test_unconfigured_ocr_and_user_denylist_hide_tools_on_ultra():
|
||||
explorer = Explorer(_context(denylist=["inspect_region"]))
|
||||
with patch(f"{EXPLORER}.is_ocr_configured", return_value=False):
|
||||
explorer._apply_tier(get_tier("ultra"))
|
||||
names = {t.name for t in explorer.get_exposed_tools()}
|
||||
names = {t["function"]["name"] for t in explorer.get_exposed_tools()}
|
||||
assert "get_ocr_list" not in names
|
||||
assert "inspect_region" not in names
|
||||
assert {"detect_objects", "ask_image_processor", "ask_perception_tool"} <= names
|
||||
@@ -152,21 +119,6 @@ def test_unconfigured_ocr_and_user_denylist_hide_tools_on_ultra():
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_gemini_model_without_google_key_uses_universal_engine():
|
||||
with patch.object(settings, "GOOGLE_API_KEY", None):
|
||||
explorer = Explorer(_context(model="gemini-3.8-flash"))
|
||||
assert explorer.use_native_gemini is False
|
||||
|
||||
|
||||
def test_gemini_model_with_shared_client_uses_native_engine():
|
||||
ctx = _context(model="google/gemini-3.8-pro")
|
||||
ctx._genai_client = MagicMock()
|
||||
with patch.object(settings, "GOOGLE_API_KEY", None):
|
||||
explorer = Explorer(ctx)
|
||||
assert explorer.use_native_gemini is True
|
||||
assert explorer.model_name == "gemini-3.8-pro"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_universal_run_without_google_key_never_touches_genai(screenshot):
|
||||
ctx = _context(model="claude-sonnet-4")
|
||||
@@ -191,15 +143,13 @@ async def test_universal_run_without_google_key_never_touches_genai(screenshot):
|
||||
storage.get_image.return_value = None
|
||||
|
||||
with (
|
||||
patch.object(settings, "GOOGLE_API_KEY", None),
|
||||
patch(f"{EXPLORER}.genai.Client") as client_cls,
|
||||
patch.object(settings, "OPENAI_API_KEY", None),
|
||||
patch(f"{UNIVERSAL_RUNNER}.get_llm", return_value=llm),
|
||||
patch(f"{RUN_SETUP}.StorageManager", return_value=storage),
|
||||
):
|
||||
explorer = Explorer(ctx)
|
||||
raw = await explorer.run("find it", "", screenshot, _state(screenshot), version="pro")
|
||||
|
||||
client_cls.assert_not_called()
|
||||
outcome = json.loads(raw)
|
||||
assert outcome["candidates"][0]["coords"] == [100, 200]
|
||||
# The universal engine only sees the pro tier's tools.
|
||||
@@ -283,45 +233,6 @@ async def test_universal_dispatch_rejects_denylisted_tool():
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_caching_tier_defaults():
|
||||
explorer = Explorer(_context())
|
||||
with patch.object(settings, "EXPLORER_CACHING", None):
|
||||
assert explorer._resolve_caching(get_tier("pro"), None) is False
|
||||
assert explorer._resolve_caching(get_tier("ultra"), None) is True
|
||||
|
||||
|
||||
def test_caching_precedence(monkeypatch):
|
||||
ctx = _context()
|
||||
explorer = Explorer(ctx)
|
||||
pro = get_tier("pro")
|
||||
|
||||
with patch.object(settings, "EXPLORER_CACHING", None):
|
||||
ctx.agent_config.explorer = SimpleNamespace(caching=True)
|
||||
assert explorer._resolve_caching(pro, None) is True # agent config beats tier
|
||||
|
||||
monkeypatch.setenv("ARTEMIS_EXPLORER_CACHING", "false")
|
||||
assert explorer._resolve_caching(pro, None) is False # env beats agent config
|
||||
|
||||
assert explorer._resolve_caching(pro, True) is True # explicit beats env
|
||||
|
||||
with patch.object(settings, "EXPLORER_CACHING", True):
|
||||
monkeypatch.delenv("ARTEMIS_EXPLORER_CACHING")
|
||||
ctx.agent_config.explorer = SimpleNamespace(caching=None)
|
||||
ctx.execution_setup = SimpleNamespace(explorer=SimpleNamespace(caching=False))
|
||||
assert explorer._resolve_caching(pro, None) is False # execution setup beats settings
|
||||
|
||||
ctx.execution_setup = SimpleNamespace(explorer=SimpleNamespace(caching=None))
|
||||
assert explorer._resolve_caching(pro, None) is True # settings beat tier
|
||||
|
||||
|
||||
def test_caching_ignores_mock_config_values():
|
||||
"""A MagicMock context must fall through to the tier default, not force True."""
|
||||
ctx = MagicMock()
|
||||
explorer = Explorer(ctx)
|
||||
with patch.object(settings, "EXPLORER_CACHING", None):
|
||||
assert explorer._resolve_caching(get_tier("pro"), None) is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Tier-aware prompt
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -381,7 +292,9 @@ def test_prompt_sections_render_shape():
|
||||
assert lines[0] == "# IDENTITY"
|
||||
assert any(line.startswith("# OPERATING PRINCIPLES") for line in lines)
|
||||
body = [line for line in lines if line and not line.startswith("#")]
|
||||
assert all(line.startswith("- ") or line.startswith("You are") for line in body)
|
||||
assert "输出语言要求" in prompt
|
||||
assert "简体中文" in prompt
|
||||
assert any(line.startswith("You are") for line in body)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -402,14 +315,12 @@ async def test_flash_run_returns_detected_candidates(screenshot):
|
||||
)
|
||||
with (
|
||||
patch(f"{RUN_SETUP}._run_object_detection", detection),
|
||||
patch(f"{EXPLORER}.genai.Client") as client_cls,
|
||||
):
|
||||
explorer = Explorer(_context())
|
||||
raw = await explorer.run(
|
||||
"gear icon | send button", "", screenshot, _state(screenshot), version="flash"
|
||||
)
|
||||
|
||||
client_cls.assert_not_called()
|
||||
outcome = json.loads(raw)
|
||||
assert [c["label"] for c in outcome["candidates"]] == ["D1", "D2"]
|
||||
assert outcome["candidates"][0] == {
|
||||
@@ -435,114 +346,6 @@ async def test_flash_run_reports_missing_detection(screenshot):
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_ultra_run_uses_async_cache_api(screenshot):
|
||||
ctx = _context()
|
||||
client = _native_client([_submit_response()])
|
||||
ctx._genai_client = client
|
||||
storage = MagicMock()
|
||||
storage.get_image.return_value = None
|
||||
|
||||
with (
|
||||
patch(f"{EXPLORER}.genai.Client") as client_cls,
|
||||
patch(f"{RUN_SETUP}.StorageManager", return_value=storage),
|
||||
patch.object(settings, "EXPLORER_CACHING", None),
|
||||
):
|
||||
explorer = Explorer(ctx)
|
||||
raw = await explorer.run("target", "", screenshot, _state(screenshot), version="ultra")
|
||||
|
||||
client_cls.assert_not_called()
|
||||
assert json.loads(raw)["candidates"][0]["coords"] == [500, 600]
|
||||
client.aio.caches.create.assert_awaited_once()
|
||||
client.aio.caches.delete.assert_awaited_once_with(name="cachedContents/abc")
|
||||
config = client.aio.models.generate_content.call_args.kwargs["config"]
|
||||
assert config.cached_content == "cachedContents/abc"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_pro_run_skips_cache_by_default(screenshot):
|
||||
ctx = _context()
|
||||
client = _native_client([_submit_response()])
|
||||
ctx._genai_client = client
|
||||
storage = MagicMock()
|
||||
storage.get_image.return_value = None
|
||||
|
||||
with (
|
||||
patch(f"{RUN_SETUP}.StorageManager", return_value=storage),
|
||||
patch.object(settings, "EXPLORER_CACHING", None),
|
||||
):
|
||||
await Explorer(ctx).run("target", "", screenshot, _state(screenshot), version="pro")
|
||||
|
||||
client.aio.caches.create.assert_not_awaited()
|
||||
client.aio.caches.delete.assert_not_awaited()
|
||||
config = client.aio.models.generate_content.call_args.kwargs["config"]
|
||||
assert {t.name for t in config.tools[0].function_declarations} == {
|
||||
"ask_perception_tool",
|
||||
"submit_answer",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_file_api_uploads_and_deletes_asynchronously(screenshot, monkeypatch):
|
||||
monkeypatch.setenv("ARTEMIS_USE_FILE_API", "true")
|
||||
ctx = _context()
|
||||
client = _native_client([_submit_response()])
|
||||
ctx._genai_client = client
|
||||
storage = MagicMock()
|
||||
storage.get_image.return_value = None
|
||||
|
||||
with patch(f"{RUN_SETUP}.StorageManager", return_value=storage):
|
||||
await Explorer(ctx).run("target", "", screenshot, _state(screenshot), version="pro")
|
||||
|
||||
client.aio.files.upload.assert_awaited_once_with(file=screenshot)
|
||||
client.aio.files.delete.assert_awaited_once_with(name="files/1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_closes_http_client_on_both_engines(screenshot):
|
||||
storage = MagicMock()
|
||||
storage.get_image.return_value = None
|
||||
|
||||
# Native engine
|
||||
ctx = _context()
|
||||
client = _native_client([_submit_response()])
|
||||
ctx._genai_client = client
|
||||
native = Explorer(ctx)
|
||||
native.http_client = SimpleNamespace(aclose=AsyncMock())
|
||||
with patch(f"{RUN_SETUP}.StorageManager", return_value=storage):
|
||||
await native.run("t", "", screenshot, _state(screenshot), version="pro")
|
||||
native_close = native.http_client # already reset
|
||||
assert native_close is None
|
||||
|
||||
# Universal engine (model call fails -> finally still closes)
|
||||
llm = MagicMock()
|
||||
llm.bind_tools.return_value.ainvoke = AsyncMock(side_effect=RuntimeError("boom"))
|
||||
universal = Explorer(_context(model="claude-sonnet-4"))
|
||||
http_client = SimpleNamespace(aclose=AsyncMock())
|
||||
universal.http_client = http_client
|
||||
with (
|
||||
patch(f"{RUN_SETUP}.StorageManager", return_value=storage),
|
||||
patch(f"{UNIVERSAL_RUNNER}.get_llm", return_value=llm),
|
||||
pytest.raises(RuntimeError),
|
||||
):
|
||||
await universal.run("t", "", screenshot, _state(screenshot), version="pro")
|
||||
http_client.aclose.assert_awaited_once()
|
||||
assert universal.http_client is None
|
||||
|
||||
|
||||
def test_missing_tool_calls_without_candidates_does_not_crash():
|
||||
explorer = Explorer(_context())
|
||||
response = MagicMock()
|
||||
response.candidates = None
|
||||
contents = []
|
||||
turn_record = {"tool_calls": []}
|
||||
explorer._handle_missing_tool_calls(contents, response, turn_record, ["hi"], 3, 1)
|
||||
assert contents[0].role == "model"
|
||||
assert contents[0].parts[0].text == "hi"
|
||||
assert "2 more" in contents[1].parts[0].text
|
||||
assert turn_record["tool_calls"][0]["name"] == "hallucinated_plain_text"
|
||||
|
||||
|
||||
def test_tier_table_is_single_source_for_versions():
|
||||
assert set(EXPLORER_TIERS) == {"flash", "pro", "ultra"}
|
||||
assert get_tier("bogus").name == "flash"
|
||||
|
||||
@@ -50,9 +50,9 @@ def mock_context(tmp_path):
|
||||
|
||||
def test_explorer_universal_engine_detection(mock_context):
|
||||
"""Test that Explorer routes to Universal Engine when non-Gemini model is configured."""
|
||||
with patch.object(settings, "GOOGLE_API_KEY", None):
|
||||
with patch.object(settings, "OPENAI_API_KEY", None):
|
||||
agent = Explorer(mock_context)
|
||||
assert agent.use_native_gemini is False
|
||||
assert not hasattr(agent, "_ensure_genai_client")
|
||||
assert agent.model_name == "claude-3-7-sonnet"
|
||||
|
||||
|
||||
|
||||
@@ -54,8 +54,8 @@ from artemis.memory.transcript import (
|
||||
|
||||
# SHA-256 snapshots of the legacy system message with the fixed inputs below.
|
||||
# Update these when an intentional template change alters the rendered prompt.
|
||||
GOLDEN_EMPTY_PLAN = "513e0341014b9a2342f608ad4dd173b7dcf5704ce53ee028ba302a9a1b696f96"
|
||||
GOLDEN_SENTINEL_PLAN = "012fe1b08645266d5a7e518023e20fc6fbfa65d4f2d985847ebb06e0184c67ee"
|
||||
GOLDEN_EMPTY_PLAN = "d715128625911aec46c83138e15aabfa967b385ab4c878509fe36ab4304fae2a"
|
||||
GOLDEN_SENTINEL_PLAN = "e2cf4ede75ffd9e283159f70dcd5c94a169c18e9f6dd4d1e7aacf460e956eaf3"
|
||||
|
||||
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
|
||||
|
||||
@@ -77,7 +77,7 @@ async def _render_legacy_system(plan_and_history: str) -> str:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_legacy_prompt_bytes_match_pre_split_golden():
|
||||
async def test_legacy_prompt_bytes_match_chinese_prompt_golden():
|
||||
empty = await _render_legacy_system("No plan or history yet.")
|
||||
sentinel = await _render_legacy_system("PLAN_AND_HISTORY_SENTINEL\nline2")
|
||||
assert hashlib.sha256(empty.encode("utf-8")).hexdigest() == GOLDEN_EMPTY_PLAN
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,616 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.agents.video_analyzer import chunk_conversation
|
||||
from artemis.agents.video_analyzer.chunk_conversation import (
|
||||
_MISSING_SUMMARY_ERROR,
|
||||
_NO_TOOL_CALL_ERROR,
|
||||
SubAgentAnswerExhausted,
|
||||
_ChunkMedia,
|
||||
_drive_sub_agent_loop,
|
||||
_run_native_chunk_conversation,
|
||||
)
|
||||
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
|
||||
from artemis.context import ArtemisContext
|
||||
from google.genai import types
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_run():
|
||||
# Mock context
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.llm_config = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
|
||||
mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static"))
|
||||
|
||||
# Mock Gemini Client
|
||||
mock_client = MagicMock()
|
||||
mock_client.aio = MagicMock()
|
||||
mock_client.aio.models = MagicMock()
|
||||
mock_chunk = MagicMock()
|
||||
mock_chunk.text = "Final analysis: The action succeeded."
|
||||
mock_chunk.function_calls = []
|
||||
|
||||
async def mock_stream():
|
||||
yield mock_chunk
|
||||
|
||||
mock_client.aio.models.generate_content_stream = AsyncMock(return_value=mock_stream())
|
||||
|
||||
# Mock file reading for prompts
|
||||
mock_prompt = "# Dynamic Video Analyzer..."
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.exists",
|
||||
autospec=True,
|
||||
side_effect=lambda self: not str(self).endswith(".jpg"),
|
||||
),
|
||||
patch("pathlib.Path.read_text", return_value=mock_prompt),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
|
||||
AsyncMock(),
|
||||
),
|
||||
):
|
||||
analyzer = VideoAnalyzer(mock_ctx)
|
||||
result, status = await analyzer.run(
|
||||
time_description="from 10s to 15s", purpose="Verify action"
|
||||
)
|
||||
|
||||
assert result == "Final analysis: The action succeeded."
|
||||
assert status == "success"
|
||||
assert mock_client.aio.models.generate_content_stream.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_preserves_thought_signature():
|
||||
# Mock context
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.llm_config = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
|
||||
mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static"))
|
||||
|
||||
# Mock MobileDeviceController
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.extract_segment_metadata = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
success=True,
|
||||
video_path="/tmp/video.mp4",
|
||||
actual_start_relative_time=10.0,
|
||||
)
|
||||
)
|
||||
|
||||
from google.genai import types
|
||||
|
||||
dummy_signature = b"dummy_thought_signature_bytes_56789"
|
||||
|
||||
# Turn 1: Model calls spawn_sub_agent
|
||||
spawn_call = types.FunctionCall(
|
||||
name="spawn_sub_agent",
|
||||
args={
|
||||
"start_time": 10.0,
|
||||
"end_time": 15.0,
|
||||
"specific_query": "Verify login screen",
|
||||
},
|
||||
)
|
||||
spawn_part = types.Part(
|
||||
function_call=spawn_call,
|
||||
thought_signature=dummy_signature,
|
||||
)
|
||||
|
||||
mock_chunk_1 = MagicMock()
|
||||
mock_chunk_1.text = ""
|
||||
mock_chunk_1.function_calls = [spawn_call]
|
||||
mock_chunk_1.candidates = [
|
||||
types.Candidate(content=types.Content(role="model", parts=[spawn_part]))
|
||||
]
|
||||
|
||||
# Turn 2: Model returns final answer
|
||||
mock_chunk_2 = MagicMock()
|
||||
mock_chunk_2.text = "Final analysis: The spawn succeeded and login button is visible."
|
||||
mock_chunk_2.function_calls = []
|
||||
mock_chunk_2.candidates = [
|
||||
types.Candidate(
|
||||
content=types.Content(
|
||||
role="model",
|
||||
parts=[types.Part.from_text(text=mock_chunk_2.text)],
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
# We mock generate_content_stream
|
||||
call_count = 0
|
||||
captured_contents = []
|
||||
|
||||
async def mock_stream_1():
|
||||
yield mock_chunk_1
|
||||
|
||||
async def mock_stream_2():
|
||||
yield mock_chunk_2
|
||||
|
||||
async def mock_generate_content_stream(model, contents, config):
|
||||
nonlocal call_count
|
||||
has_submit_answer = False
|
||||
if config.tools:
|
||||
for t in config.tools:
|
||||
fd_list = getattr(t, "function_declarations", None)
|
||||
if fd_list:
|
||||
for fd in fd_list:
|
||||
if fd.name == "submit_answer":
|
||||
has_submit_answer = True
|
||||
break
|
||||
if not has_submit_answer:
|
||||
captured_contents.append(list(contents)) # copy history contents passed to call
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
return mock_stream_1()
|
||||
else:
|
||||
return mock_stream_2()
|
||||
else:
|
||||
submit_call = types.FunctionCall(
|
||||
name="submit_answer",
|
||||
args={
|
||||
"confidence_score": 0.9,
|
||||
"summary": "Sub-agent verification finished",
|
||||
"analysis": "Analysis here",
|
||||
"verification_timestamp_secs": 10.0,
|
||||
},
|
||||
)
|
||||
mock_sub_chunk = MagicMock()
|
||||
mock_sub_chunk.text = ""
|
||||
mock_sub_chunk.function_calls = [submit_call]
|
||||
|
||||
async def sub_stream():
|
||||
yield mock_sub_chunk
|
||||
|
||||
return sub_stream()
|
||||
|
||||
# Mock Gemini Client
|
||||
mock_client = MagicMock()
|
||||
mock_client.aio = MagicMock()
|
||||
mock_client.aio.files = MagicMock()
|
||||
|
||||
mock_file = MagicMock()
|
||||
mock_file.name = "dummy_cloud_file_123"
|
||||
mock_file.uri = "https://gemini-file-api/dummy_cloud_file_123"
|
||||
mock_file.mime_type = "video/mp4"
|
||||
mock_client.aio.files.upload = AsyncMock(return_value=mock_file)
|
||||
|
||||
mock_f_state = MagicMock()
|
||||
mock_f_state.state.name = "ACTIVE"
|
||||
mock_client.aio.files.get = AsyncMock(return_value=mock_f_state)
|
||||
mock_client.aio.files.delete = AsyncMock()
|
||||
|
||||
mock_client.aio.models = MagicMock()
|
||||
mock_client.aio.models.generate_content_stream = AsyncMock(
|
||||
side_effect=mock_generate_content_stream
|
||||
)
|
||||
|
||||
# Mock file reading for prompts
|
||||
mock_prompt = "# Dynamic Video Analyzer..."
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.get_controller",
|
||||
return_value=mock_controller,
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.exists",
|
||||
autospec=True,
|
||||
side_effect=lambda self: not str(self).endswith(".jpg"),
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.stat",
|
||||
return_value=os.stat_result((0o100644, 0, 0, 0, 0, 0, 1024 * 1024, 0, 0, 0)),
|
||||
),
|
||||
patch("pathlib.Path.read_text", return_value=mock_prompt),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
|
||||
AsyncMock(),
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
|
||||
AsyncMock(return_value=Path("/tmp/video.mp4")),
|
||||
),
|
||||
):
|
||||
analyzer = VideoAnalyzer(mock_ctx)
|
||||
result, status = await analyzer.run(
|
||||
time_description="from 10s to 15s", purpose="Verify login"
|
||||
)
|
||||
|
||||
assert status == "success"
|
||||
assert "spawn succeeded" in result
|
||||
assert call_count == 2
|
||||
|
||||
# Verify that the second model call's history contains the model turn with the thought_signature
|
||||
second_call_history = captured_contents[1]
|
||||
assert len(second_call_history) >= 4
|
||||
|
||||
model_turn = second_call_history[1]
|
||||
assert model_turn.role == "model"
|
||||
assert len(model_turn.parts) == 1
|
||||
assert model_turn.parts[0].function_call.name == "spawn_sub_agent"
|
||||
assert model_turn.parts[0].thought_signature == dummy_signature
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_sub_agent_confidence_validation():
|
||||
# Mock context
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
mock_ctx.llm_config = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
|
||||
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
|
||||
mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static"))
|
||||
|
||||
# Mock MobileDeviceController
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.extract_segment_metadata = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
success=True,
|
||||
video_path="/tmp/video.mp4",
|
||||
actual_start_relative_time=10.0,
|
||||
)
|
||||
)
|
||||
|
||||
from google.genai import types
|
||||
|
||||
# Main agent responses
|
||||
# Turn 1: Calls spawn_sub_agent
|
||||
spawn_call = types.FunctionCall(
|
||||
name="spawn_sub_agent",
|
||||
args={
|
||||
"start_time": 10.0,
|
||||
"end_time": 15.0,
|
||||
"specific_query": "Verify login screen",
|
||||
},
|
||||
)
|
||||
mock_main_chunk_1 = MagicMock()
|
||||
mock_main_chunk_1.text = ""
|
||||
mock_main_chunk_1.function_calls = [spawn_call]
|
||||
|
||||
# Turn 2: Returns final answer
|
||||
mock_main_chunk_2 = MagicMock()
|
||||
mock_main_chunk_2.text = "Final blackboard status checked."
|
||||
mock_main_chunk_2.function_calls = []
|
||||
|
||||
# Sub-agent responses
|
||||
# Turn 1: Calls submit_answer with invalid confidence_score (string)
|
||||
invalid_submit_call = types.FunctionCall(
|
||||
name="submit_answer",
|
||||
args={
|
||||
"timeline_events": [
|
||||
{
|
||||
"confidence_score": "HIGH",
|
||||
"transcription": "Invalid confidence score call",
|
||||
"verification_timestamp_secs": 10.0,
|
||||
}
|
||||
],
|
||||
"summary": "Invalid confidence score call",
|
||||
"analysis": "Some analysis",
|
||||
},
|
||||
)
|
||||
sub_dummy_signature = b"sub_dummy_signature_9999"
|
||||
sub_part_1 = types.Part(
|
||||
function_call=invalid_submit_call,
|
||||
thought_signature=sub_dummy_signature,
|
||||
)
|
||||
mock_sub_chunk_1 = MagicMock()
|
||||
mock_sub_chunk_1.text = ""
|
||||
mock_sub_chunk_1.function_calls = [invalid_submit_call]
|
||||
mock_sub_chunk_1.candidates = [
|
||||
types.Candidate(content=types.Content(role="model", parts=[sub_part_1]))
|
||||
]
|
||||
|
||||
# Turn 2: Calls submit_answer with valid confidence_score
|
||||
valid_submit_call = types.FunctionCall(
|
||||
name="submit_answer",
|
||||
args={
|
||||
"timeline_events": [
|
||||
{
|
||||
"confidence_score": 0.8,
|
||||
"transcription": "Valid confidence score call",
|
||||
"verification_timestamp_secs": 10.0,
|
||||
}
|
||||
],
|
||||
"summary": "Valid confidence score call",
|
||||
"analysis": "Some analysis",
|
||||
},
|
||||
)
|
||||
mock_sub_chunk_2 = MagicMock()
|
||||
mock_sub_chunk_2.text = ""
|
||||
mock_sub_chunk_2.function_calls = [valid_submit_call]
|
||||
mock_sub_chunk_2.candidates = [
|
||||
types.Candidate(
|
||||
content=types.Content(role="model", parts=[types.Part(function_call=valid_submit_call)])
|
||||
)
|
||||
]
|
||||
|
||||
# Stream generator mocks
|
||||
main_calls = 0
|
||||
sub_calls = 0
|
||||
captured_sub_contents = []
|
||||
|
||||
async def mock_main_stream():
|
||||
nonlocal main_calls
|
||||
if main_calls == 0:
|
||||
main_calls += 1
|
||||
yield mock_main_chunk_1
|
||||
else:
|
||||
yield mock_main_chunk_2
|
||||
|
||||
async def mock_sub_stream():
|
||||
nonlocal sub_calls
|
||||
if sub_calls == 0:
|
||||
sub_calls += 1
|
||||
yield mock_sub_chunk_1
|
||||
else:
|
||||
yield mock_sub_chunk_2
|
||||
|
||||
async def mock_generate_content_stream(model, contents, config):
|
||||
system_instruction = config.system_instruction
|
||||
if "Dynamic Video Analyzer" in str(system_instruction):
|
||||
return mock_main_stream()
|
||||
else:
|
||||
captured_sub_contents.append(list(contents))
|
||||
return mock_sub_stream()
|
||||
|
||||
# Mock Gemini Client
|
||||
mock_client = MagicMock()
|
||||
mock_client.aio = MagicMock()
|
||||
mock_client.aio.files = MagicMock()
|
||||
|
||||
mock_file = MagicMock()
|
||||
mock_file.name = "dummy_cloud_file_123"
|
||||
mock_file.uri = "https://gemini-file-api/dummy_cloud_file_123"
|
||||
mock_file.mime_type = "video/mp4"
|
||||
mock_client.aio.files.upload = AsyncMock(return_value=mock_file)
|
||||
|
||||
mock_f_state = MagicMock()
|
||||
mock_f_state.state.name = "ACTIVE"
|
||||
mock_client.aio.files.get = AsyncMock(return_value=mock_f_state)
|
||||
mock_client.aio.files.delete = AsyncMock()
|
||||
|
||||
mock_client.aio.models = MagicMock()
|
||||
mock_client.aio.models.generate_content_stream = AsyncMock(
|
||||
side_effect=mock_generate_content_stream
|
||||
)
|
||||
|
||||
# Mock file reading for prompts
|
||||
mock_main_prompt = "# Dynamic Video Analyzer..."
|
||||
mock_sub_prompt = "# Sub Agent..."
|
||||
|
||||
def mock_read_text(self, encoding="utf-8"):
|
||||
if "video_analyzer.md" in str(self):
|
||||
return mock_main_prompt
|
||||
elif "video_sub_agent.md" in str(self):
|
||||
return mock_sub_prompt
|
||||
return ""
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
|
||||
return_value=mock_client,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.get_controller",
|
||||
return_value=mock_controller,
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.exists",
|
||||
autospec=True,
|
||||
side_effect=lambda self: not str(self).endswith(".jpg"),
|
||||
),
|
||||
patch(
|
||||
"pathlib.Path.stat",
|
||||
return_value=os.stat_result((0o100644, 0, 0, 0, 0, 0, 1024 * 1024, 0, 0, 0)),
|
||||
),
|
||||
patch("pathlib.Path.read_text", mock_read_text),
|
||||
patch("pathlib.Path.mkdir"),
|
||||
patch("pathlib.Path.touch"),
|
||||
patch("pathlib.Path.is_dir", return_value=True),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
|
||||
AsyncMock(),
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
|
||||
AsyncMock(return_value=Path("/tmp/video.mp4")),
|
||||
),
|
||||
patch(
|
||||
"asyncio.create_subprocess_exec",
|
||||
AsyncMock(return_value=MagicMock(communicate=AsyncMock(return_value=(b"", b"")))),
|
||||
),
|
||||
):
|
||||
analyzer = VideoAnalyzer(mock_ctx)
|
||||
result, status = await analyzer.run(
|
||||
time_description="from 10s to 15s", purpose="Verify login"
|
||||
)
|
||||
|
||||
assert status == "success"
|
||||
assert sub_calls == 1
|
||||
assert len(analyzer.blackboard_entries) == 1
|
||||
entry = analyzer.blackboard_entries[0]
|
||||
assert entry["confidence_score"] == 0.8
|
||||
assert entry["summary"] == "Valid confidence score call"
|
||||
|
||||
# Verify that sub-agent turn 2 received the model turn with the thought_signature
|
||||
assert len(captured_sub_contents) >= 2
|
||||
sub_turn_2_contents = captured_sub_contents[1]
|
||||
model_turns = [c for c in sub_turn_2_contents if c.role == "model"]
|
||||
assert len(model_turns) == 1
|
||||
assert model_turns[0].parts[0].thought_signature == sub_dummy_signature
|
||||
|
||||
|
||||
# ------------------------------------------------- static sub-agent loop
|
||||
|
||||
|
||||
def _static_media(tmp_path) -> _ChunkMedia:
|
||||
media = _ChunkMedia()
|
||||
media.path = tmp_path / "segment.mp4"
|
||||
media.path.write_bytes(b"video")
|
||||
media.compressed_path = media.path
|
||||
media.actual_start = 0.0
|
||||
media.prompt_with_context = "Watch the toggle."
|
||||
return media
|
||||
|
||||
|
||||
def _valid_static_event() -> dict:
|
||||
return {
|
||||
"start_time": 1.0,
|
||||
"end_time": 2.0,
|
||||
"transcription": "Battery Saver toggled on",
|
||||
"confidence_score": 0.9,
|
||||
"verification_timestamp_secs": 1.5,
|
||||
}
|
||||
|
||||
|
||||
def _scripted_static_turns(*turns):
|
||||
"""Fakes ``_stream_sub_agent_turn`` with ``(text, function_calls)`` per turn."""
|
||||
scripted = list(turns)
|
||||
|
||||
async def fake_turn(analyzer, sub_agent_contents, trace_id):
|
||||
text, function_calls = scripted.pop(0)
|
||||
return text, function_calls, []
|
||||
|
||||
return fake_turn
|
||||
|
||||
|
||||
def _static_analyzer() -> MagicMock:
|
||||
analyzer = MagicMock()
|
||||
analyzer.model_name = "gemini-3.7-flash"
|
||||
analyzer.sub_system_prompt = "system"
|
||||
analyzer.ctx.data_engine = None
|
||||
return analyzer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_static_submit_answer_without_summary_is_sent_back_then_accepted(tmp_path):
|
||||
first = types.FunctionCall(
|
||||
name="submit_answer", args={"timeline_events": [_valid_static_event()]}
|
||||
)
|
||||
second = types.FunctionCall(
|
||||
name="submit_answer",
|
||||
args={"summary": "s2", "analysis": "a2", "timeline_events": []},
|
||||
)
|
||||
contents: list = []
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
chunk_conversation,
|
||||
"_stream_sub_agent_turn",
|
||||
new=_scripted_static_turns(("", [first]), ("", [second])),
|
||||
),
|
||||
patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist,
|
||||
):
|
||||
answer = await _drive_sub_agent_loop(
|
||||
_static_analyzer(), contents, None, _static_media(tmp_path), 0.0, 5.0, "verify"
|
||||
)
|
||||
|
||||
assert answer == ("s2", "a2")
|
||||
# Only the accepted answer reaches persistence.
|
||||
persist.assert_awaited_once()
|
||||
assert persist.await_args.args[1] == []
|
||||
rejection = contents[-1]
|
||||
assert rejection.role == "user"
|
||||
assert rejection.parts[0].function_response.response["error"] == _MISSING_SUMMARY_ERROR
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_static_loop_raises_when_no_turn_calls_submit_answer(tmp_path):
|
||||
with (
|
||||
patch.object(
|
||||
chunk_conversation,
|
||||
"_stream_sub_agent_turn",
|
||||
new=_scripted_static_turns(("text only", []), ("still text", [])),
|
||||
),
|
||||
patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist,
|
||||
pytest.raises(SubAgentAnswerExhausted) as raised,
|
||||
):
|
||||
await _drive_sub_agent_loop(
|
||||
_static_analyzer(), [], None, _static_media(tmp_path), 0.0, 5.0, "verify"
|
||||
)
|
||||
|
||||
assert raised.value.reason == _NO_TOOL_CALL_ERROR
|
||||
persist.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_static_last_turn_timeline_failure_raises_and_persists_nothing(tmp_path):
|
||||
bad = dict(_valid_static_event(), confidence_score="HIGH")
|
||||
out_of_range = dict(_valid_static_event(), verification_timestamp_secs=42.0)
|
||||
first = types.FunctionCall(
|
||||
name="submit_answer", args={"summary": "s1", "timeline_events": [bad]}
|
||||
)
|
||||
second = types.FunctionCall(
|
||||
name="submit_answer", args={"summary": "s2", "timeline_events": [out_of_range]}
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
chunk_conversation,
|
||||
"_stream_sub_agent_turn",
|
||||
new=_scripted_static_turns(("", [first]), ("", [second])),
|
||||
),
|
||||
patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist,
|
||||
pytest.raises(SubAgentAnswerExhausted) as raised,
|
||||
):
|
||||
await _drive_sub_agent_loop(
|
||||
_static_analyzer(), [], None, _static_media(tmp_path), 0.0, 5.0, "verify"
|
||||
)
|
||||
|
||||
assert "verification_timestamp_secs" in raised.value.reason
|
||||
assert "42.0" in str(raised.value)
|
||||
persist.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exhausted_static_chunk_is_not_committed_to_the_blackboard(tmp_path):
|
||||
analyzer = _static_analyzer()
|
||||
uploaded = SimpleNamespace(name="files/abc", uri="https://files/abc", mime_type="video/mp4")
|
||||
analyzer.upload_and_poll_file = AsyncMock(return_value=uploaded)
|
||||
analyzer.client.aio.files.delete = AsyncMock()
|
||||
analyzer.cloud_files_to_cleanup = set()
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
chunk_conversation,
|
||||
"_stream_sub_agent_turn",
|
||||
new=_scripted_static_turns(("text only", []), ("still text", [])),
|
||||
),
|
||||
pytest.raises(SubAgentAnswerExhausted),
|
||||
):
|
||||
await _run_native_chunk_conversation(
|
||||
analyzer, _static_media(tmp_path), 0.0, 5.0, 0.0, 5.0, "verify", "lease-1"
|
||||
)
|
||||
|
||||
analyzer.blackboard.complete_segment.assert_not_called()
|
||||
analyzer.client.aio.files.delete.assert_awaited_once_with(name="files/abc")
|
||||
@@ -50,9 +50,9 @@ def test_video_analyzer_universal_engine_detection(mock_context):
|
||||
"""Test that VideoAnalyzer routes to Universal Engine when non-Gemini model is set."""
|
||||
from artemis.config import settings
|
||||
|
||||
with patch.object(settings, "GOOGLE_API_KEY", None):
|
||||
with patch.object(settings, "OPENAI_API_KEY", None):
|
||||
agent = VideoAnalyzer(mock_context)
|
||||
assert agent.use_native_gemini is False
|
||||
assert not hasattr(agent, "_ensure_genai_client")
|
||||
assert agent.model_name == "claude-3-7-sonnet"
|
||||
|
||||
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Native Gemini audio agent loop: only accepted answers reach the blackboard."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from artemis.agents.video_analyzer.audio_native import (
|
||||
_BAD_CONFIDENCE_ERROR,
|
||||
_AudioMedia,
|
||||
_drive_audio_agent_loop,
|
||||
_run_native_audio_conversation,
|
||||
)
|
||||
from artemis.agents.video_analyzer.chunk_conversation import (
|
||||
_MISSING_SUMMARY_ERROR,
|
||||
_NO_TOOL_CALL_ERROR,
|
||||
SUB_AGENT_MAX_TURNS,
|
||||
)
|
||||
from artemis.agents.video_analyzer.reliability import (
|
||||
SubAgentAnswerExhausted,
|
||||
VideoFailureCategory,
|
||||
classify_video_failure,
|
||||
)
|
||||
from artemis.agents.video_analyzer.universal_tools import build_submit_answer_declaration
|
||||
from google.genai import types
|
||||
import pytest
|
||||
|
||||
|
||||
def _audio_response(text: str, function_calls: list) -> SimpleNamespace:
|
||||
"""Fakes a non-streamed ``generate_content`` response for one audio turn."""
|
||||
parts = [types.Part(function_call=fc) for fc in function_calls]
|
||||
if text:
|
||||
parts.append(types.Part.from_text(text=text))
|
||||
return SimpleNamespace(
|
||||
text=text,
|
||||
function_calls=function_calls,
|
||||
candidates=[SimpleNamespace(content=SimpleNamespace(parts=parts))],
|
||||
)
|
||||
|
||||
|
||||
def _audio_analyzer(*turns) -> MagicMock:
|
||||
"""Analyzer stub whose Gemini client answers the scripted ``(text, calls)`` turns."""
|
||||
analyzer = MagicMock()
|
||||
analyzer.model_name = "gemini-3.7-flash"
|
||||
analyzer.audio_system_prompt = "system"
|
||||
analyzer.submit_answer_tool = build_submit_answer_declaration()
|
||||
analyzer.model_call_timeout_seconds = 5.0
|
||||
analyzer.ctx.data_engine = None
|
||||
analyzer.client.aio.models.generate_content = AsyncMock(
|
||||
side_effect=[_audio_response(text, calls) for text, calls in turns]
|
||||
)
|
||||
return analyzer
|
||||
|
||||
|
||||
def _audio_media(tmp_path) -> _AudioMedia:
|
||||
media = _AudioMedia()
|
||||
media.audio_path = tmp_path / "segment.wav"
|
||||
media.audio_path.write_bytes(b"audio")
|
||||
media.actual_start = 0.0
|
||||
media.prompt_with_context = "Listen for the chime."
|
||||
return media
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_submit_answer_without_summary_is_sent_back_then_accepted():
|
||||
first = types.FunctionCall(
|
||||
name="submit_answer", args={"summary": " ", "confidence_score": 0.9}
|
||||
)
|
||||
second = types.FunctionCall(
|
||||
name="submit_answer",
|
||||
args={"summary": "chime\nheard", "analysis": "a2", "confidence_score": 0.7},
|
||||
)
|
||||
analyzer = _audio_analyzer(("", [first]), ("", [second]))
|
||||
contents: list = []
|
||||
|
||||
answer = await _drive_audio_agent_loop(analyzer, contents)
|
||||
|
||||
assert answer == ("chime heard", "a2", 0.7)
|
||||
assert analyzer.client.aio.models.generate_content.await_count == 2
|
||||
# The blank summary went back to the model as a function_response error.
|
||||
rejections = [
|
||||
c
|
||||
for c in contents
|
||||
if c.role == "user" and c.parts and c.parts[0].function_response is not None
|
||||
]
|
||||
assert len(rejections) == 1
|
||||
assert rejections[0].parts[0].function_response.response["error"] == _MISSING_SUMMARY_ERROR
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_loop_raises_when_no_turn_calls_submit_answer():
|
||||
analyzer = _audio_analyzer(("text only", []), ("still text", []))
|
||||
|
||||
with pytest.raises(SubAgentAnswerExhausted) as raised:
|
||||
await _drive_audio_agent_loop(analyzer, [])
|
||||
|
||||
assert raised.value.turns == SUB_AGENT_MAX_TURNS
|
||||
assert raised.value.reason == _NO_TOOL_CALL_ERROR
|
||||
assert "No summary provided" not in str(raised.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_loop_raises_when_confidence_score_is_never_valid():
|
||||
first = types.FunctionCall(
|
||||
name="submit_answer", args={"summary": "s1", "confidence_score": "HIGH"}
|
||||
)
|
||||
second = types.FunctionCall(
|
||||
name="submit_answer", args={"summary": "s2", "confidence_score": 1.5}
|
||||
)
|
||||
analyzer = _audio_analyzer(("", [first]), ("", [second]))
|
||||
contents: list = []
|
||||
|
||||
with pytest.raises(SubAgentAnswerExhausted) as raised:
|
||||
await _drive_audio_agent_loop(analyzer, contents)
|
||||
|
||||
assert raised.value.reason == _BAD_CONFIDENCE_ERROR
|
||||
assert contents[-1].role == "user"
|
||||
assert contents[-1].parts[0].function_response.response["error"] == _BAD_CONFIDENCE_ERROR
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exhausted_audio_segment_is_not_committed_and_cloud_file_is_deleted(tmp_path):
|
||||
analyzer = _audio_analyzer(("text only", []), ("still text", []))
|
||||
uploaded = SimpleNamespace(name="files/abc", uri="https://files/abc", mime_type="audio/wav")
|
||||
analyzer.upload_and_poll_file = AsyncMock(return_value=uploaded)
|
||||
analyzer.client.aio.files.delete = AsyncMock()
|
||||
analyzer.cloud_files_to_cleanup = {"files/abc"}
|
||||
analyzer._record_blackboard_entry = MagicMock()
|
||||
analyzer.blackboard = MagicMock()
|
||||
|
||||
with pytest.raises(SubAgentAnswerExhausted):
|
||||
await _run_native_audio_conversation(
|
||||
analyzer, _audio_media(tmp_path), 0.0, 5.0, 0.0, 5.0, "chime", "lease-1"
|
||||
)
|
||||
|
||||
analyzer.blackboard.complete_segment.assert_not_called()
|
||||
analyzer._record_blackboard_entry.assert_not_called()
|
||||
analyzer.client.aio.files.delete.assert_awaited_once_with(name="files/abc")
|
||||
assert analyzer.cloud_files_to_cleanup == set()
|
||||
|
||||
|
||||
def test_sub_agent_exhaustion_is_classified_unknown():
|
||||
failure = classify_video_failure(SubAgentAnswerExhausted(2, "x"))
|
||||
|
||||
assert failure.category is VideoFailureCategory.UNKNOWN
|
||||
@@ -143,7 +143,7 @@ def analyzer_context():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spawn_sub_agent_reuses_complete_persistent_coverage(analyzer_context):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(analyzer_context)
|
||||
claim = analyzer.blackboard.claim_segment(10.0, 20.0, "verify login")
|
||||
analyzer.blackboard.complete_segment(
|
||||
@@ -165,7 +165,7 @@ async def test_spawn_sub_agent_reuses_complete_persistent_coverage(analyzer_cont
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_spawn_sub_agent_returns_partial_and_keeps_prior_success(analyzer_context):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(analyzer_context)
|
||||
claim = analyzer.blackboard.claim_segment(0.0, 60.0, "summarize flow")
|
||||
analyzer.blackboard.complete_segment(
|
||||
@@ -207,7 +207,7 @@ async def test_video_coordinator_retries_transient_failures():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_audio_analysis_reuses_exact_completed_coverage(analyzer_context):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(analyzer_context)
|
||||
claim = analyzer.blackboard.claim_segment(2.0, 6.0, "hear notification", modality="audio")
|
||||
analyzer.blackboard.complete_segment(
|
||||
|
||||
@@ -80,7 +80,7 @@ def test_circuit_breaker_opens_and_half_opens():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_timeout_chunk_is_bisected_and_successful_leaves_survive():
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
|
||||
async def analyze(start: float, end: float, query: str) -> str:
|
||||
@@ -98,7 +98,7 @@ async def test_timeout_chunk_is_bisected_and_successful_leaves_survive():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_universal_primary_failure_uses_configured_fallback():
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
|
||||
primary = MagicMock()
|
||||
@@ -126,57 +126,8 @@ async def test_universal_primary_failure_uses_configured_fallback():
|
||||
assert get_model.call_args_list[1].kwargs["use_fallback"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_chunk_exhaustion_commits_universal_fallback(tmp_path):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
analyzer.use_native_gemini = True
|
||||
raw_video = tmp_path / "segment.mp4"
|
||||
raw_video.write_bytes(b"video")
|
||||
extracted = SimpleNamespace(
|
||||
success=True,
|
||||
video_path=raw_video,
|
||||
actual_start_relative_time=0.0,
|
||||
duration_seconds=5.0,
|
||||
warning=None,
|
||||
)
|
||||
controller = SimpleNamespace(extract_segment_metadata=AsyncMock(return_value=extracted))
|
||||
fallback_text = "[from 0.0s to 5.0s] Summary: fallback recovered Analysis: grounded"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.get_controller",
|
||||
return_value=controller,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
|
||||
new=AsyncMock(return_value=raw_video),
|
||||
),
|
||||
patch.object(
|
||||
analyzer,
|
||||
"upload_and_poll_file",
|
||||
new=AsyncMock(side_effect=RuntimeError("503 provider unavailable")),
|
||||
) as upload,
|
||||
patch.object(
|
||||
analyzer,
|
||||
"_exec_single_chunk_universal",
|
||||
new=AsyncMock(return_value=fallback_text),
|
||||
) as fallback,
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.asyncio.sleep",
|
||||
new=AsyncMock(),
|
||||
),
|
||||
):
|
||||
result = await analyzer._exec_single_chunk(0.0, 5.0, "verify transition")
|
||||
|
||||
assert result == fallback_text
|
||||
assert upload.await_count == 2
|
||||
fallback.assert_awaited_once()
|
||||
assert analyzer.blackboard.missing_intervals(0.0, 5.0, "verify transition") == []
|
||||
|
||||
|
||||
def test_action_timestamps_drive_dense_sampling():
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
step = SimpleNamespace(timestamp=104.0, action_taken={"name": "tap"})
|
||||
storage = SimpleNamespace(get_steps=lambda _session: [step])
|
||||
@@ -304,7 +255,7 @@ class _AgenticRejected(Exception):
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_chunk_sized_for_agentic_ceiling_is_not_run_statically(tmp_path):
|
||||
"""A chunk planned at 600s whose run degraded before it started is handed back."""
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
analyzer.use_native_gemini = True
|
||||
# A sibling chunk already flipped the run to static (60s ceiling).
|
||||
@@ -318,7 +269,7 @@ async def test_pending_chunk_sized_for_agentic_ceiling_is_not_run_statically(tmp
|
||||
return_value=controller,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.chunk_native._run_native_chunk_conversation",
|
||||
"artemis.agents.video_analyzer.chunk_runner._prepare_chunk_media",
|
||||
new=AsyncMock(),
|
||||
) as static_path,
|
||||
patch.object(analyzer, "_exec_single_chunk_universal", new=AsyncMock()) as universal,
|
||||
@@ -335,83 +286,10 @@ async def test_pending_chunk_sized_for_agentic_ceiling_is_not_run_statically(tmp
|
||||
assert analyzer.blackboard.missing_intervals(0.0, 150.0, "verify toggle") == [(0.0, 150.0)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sibling_chunk_degraded_mid_run_is_replanned_into_static_pieces(tmp_path):
|
||||
"""Two 600s chunks: the first is rejected while the second is still in media prep."""
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
analyzer.use_native_gemini = True
|
||||
analyzer.video_processing = "agentic"
|
||||
analyzer.chunk_size_seconds = analyzer.agentic_chunk_size_seconds
|
||||
assert analyzer.chunk_size_seconds == 600.0
|
||||
|
||||
raw_video = tmp_path / "segment.mp4"
|
||||
raw_video.write_bytes(b"video")
|
||||
first_rejected = asyncio.Event()
|
||||
|
||||
async def extract(start: float, end: float):
|
||||
if start >= 600.0 and analyzer.video_processing == "agentic":
|
||||
# The second 600s chunk is mid-prep when the first one degrades.
|
||||
await first_rejected.wait()
|
||||
return SimpleNamespace(
|
||||
success=True,
|
||||
video_path=raw_video,
|
||||
actual_start_relative_time=start,
|
||||
duration_seconds=end - start,
|
||||
warning=None,
|
||||
)
|
||||
|
||||
controller = SimpleNamespace(extract_segment_metadata=AsyncMock(side_effect=extract))
|
||||
agentic_calls: list[tuple[float, float]] = []
|
||||
|
||||
async def agentic(analyzer_, media, cs, ce, *rest):
|
||||
agentic_calls.append((cs, ce))
|
||||
first_rejected.set()
|
||||
raise _AgenticRejected("400 INVALID_ARGUMENT: processing 'agentic' is not supported")
|
||||
|
||||
async def static_answer(analyzer_, media, cs, ce, *rest):
|
||||
return f"[from {cs:.1f}s to {ce:.1f}s] Summary: static Analysis: ok"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.get_controller",
|
||||
return_value=controller,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
|
||||
new=AsyncMock(return_value=raw_video),
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.chunk_native.run_agentic_chunk_conversation",
|
||||
new=AsyncMock(side_effect=agentic),
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.video_analyzer.chunk_native._run_native_chunk_conversation",
|
||||
new=AsyncMock(side_effect=static_answer),
|
||||
) as static_path,
|
||||
patch.object(analyzer, "_exec_single_chunk_universal", new=AsyncMock()) as universal,
|
||||
patch("artemis.agents.video_analyzer.video_analyzer._record_llm_event"),
|
||||
):
|
||||
result = await analyzer.exec_spawn_sub_agent(0.0, 1200.0, "verify toggle")
|
||||
|
||||
# Only the first chunk ever reached the agentic API; the second never ran
|
||||
# its 600s clip statically and neither went to the universal fallback.
|
||||
assert agentic_calls == [(0.0, 600.0)]
|
||||
universal.assert_not_awaited()
|
||||
segments = sorted((call.args[2], call.args[3]) for call in static_path.await_args_list)
|
||||
assert len(segments) == 20
|
||||
assert segments[0] == (0.0, 60.0)
|
||||
assert segments[-1] == (1140.0, 1200.0)
|
||||
assert all(ce - cs <= 60.0 for cs, ce in segments)
|
||||
assert "PARTIAL" not in result
|
||||
assert result.count("Summary: static") == 20
|
||||
assert analyzer.video_processing == "static"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_short_degraded_chunk_is_retried_statically_once():
|
||||
"""A 5s chunk (below min_chunk*2) is re-run as-is instead of failing."""
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
calls: list[tuple[float, float]] = []
|
||||
|
||||
@@ -436,7 +314,7 @@ async def test_short_degraded_chunk_is_retried_statically_once():
|
||||
@pytest.mark.asyncio
|
||||
async def test_static_retry_of_degraded_chunk_is_bounded():
|
||||
"""A second degrade on the same interval is terminal: no loop, no bisect."""
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
|
||||
with patch("artemis.agents.video_analyzer.video_analyzer.settings.OPENAI_API_KEY", None):
|
||||
analyzer = VideoAnalyzer(_context())
|
||||
calls: list[tuple[float, float]] = []
|
||||
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""The video-processing knob is typed once, in the Google provider module."""
|
||||
|
||||
from typing import get_args
|
||||
|
||||
from pydantic import ValidationError
|
||||
import pytest
|
||||
|
||||
from artemis.config.agent import VideoAnalyzerConfig
|
||||
from artemis.llm.google import VideoProcessing
|
||||
|
||||
|
||||
def test_processing_field_reuses_the_provider_literal():
|
||||
assert VideoAnalyzerConfig.model_fields["processing"].annotation is VideoProcessing
|
||||
assert set(get_args(VideoProcessing)) == {"auto", "agentic", "static"}
|
||||
|
||||
|
||||
def test_processing_defaults_to_auto_and_accepts_every_mode():
|
||||
assert VideoAnalyzerConfig().processing == "auto"
|
||||
for mode in get_args(VideoProcessing):
|
||||
assert VideoAnalyzerConfig(processing=mode).processing == mode
|
||||
|
||||
|
||||
def test_processing_rejects_unknown_modes():
|
||||
with pytest.raises(ValidationError, match="processing"):
|
||||
VideoAnalyzerConfig(processing="bogus")
|
||||
@@ -52,7 +52,7 @@ async def test_readiness_engine_run_all():
|
||||
assert "python_runtime" in probe_ids
|
||||
assert "system_config" in probe_ids
|
||||
assert "android_adb" in probe_ids
|
||||
assert "gemini_api_key" in probe_ids
|
||||
assert "openai_api_key" in probe_ids
|
||||
assert "vision_ocr_key" in probe_ids
|
||||
assert "toolchain" in probe_ids
|
||||
|
||||
@@ -357,7 +357,7 @@ async def test_adb_probe_prefers_unlocked_device_when_one_is_locked(monkeypatch)
|
||||
async def test_llm_credentials_probe_structure():
|
||||
"""Verify LLMCredentialsProbe returns correct category and schema."""
|
||||
probe = LLMCredentialsProbe()
|
||||
assert probe.probe_id == "gemini_api_key"
|
||||
assert probe.probe_id == "openai_api_key"
|
||||
assert probe.category == ProbeCategory.CREDENTIALS
|
||||
assert probe.is_blocker is True
|
||||
|
||||
@@ -393,19 +393,17 @@ async def test_probe_target_serial_forwards_to_adb_probe():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_credentials_probe_and_dynamic_update():
|
||||
"""Verify dynamic API key updates and metadata reflection."""
|
||||
async def test_credentials_probe_and_dynamic_update(monkeypatch):
|
||||
from artemis.config import settings
|
||||
from pydantic import SecretStr
|
||||
|
||||
settings.set_api_key("google", "test_gemini_key_1234567890", persist_to_env=False)
|
||||
|
||||
probe = LLMCredentialsProbe()
|
||||
result = await probe.probe()
|
||||
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("test_key_1234567890"))
|
||||
result = await LLMCredentialsProbe().probe()
|
||||
assert result.status == ProbeStatus.PASS
|
||||
assert "current_key" in result.metadata
|
||||
assert result.metadata["current_key"] == "test_gemini_key_1234567890"
|
||||
assert "api_keys" in result.metadata
|
||||
assert result.metadata["api_keys"]["google"] == "test_gemini_key_1234567890"
|
||||
assert result.id == "openai_api_key"
|
||||
assert "test_key_1234567890" not in repr(result.metadata)
|
||||
assert "current_key" not in result.metadata
|
||||
assert "api_keys" not in result.metadata
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -129,7 +129,7 @@ def test_sort_by_fix_order_follows_check_order_and_appends_unknown_ids():
|
||||
"python_runtime",
|
||||
"system_config",
|
||||
"integration_host",
|
||||
"gemini_api_key",
|
||||
"openai_api_key",
|
||||
"android_adb",
|
||||
"toolchain",
|
||||
"vision_ocr_key",
|
||||
|
||||
@@ -21,6 +21,7 @@ import shutil
|
||||
import tempfile
|
||||
import time
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from pydantic import SecretStr
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -108,7 +109,7 @@ def _credentials_metadata() -> dict:
|
||||
return {
|
||||
"providers": [
|
||||
{
|
||||
"provider": "google",
|
||||
"provider": "openai",
|
||||
"label": "Gemini",
|
||||
"masked": "AIza...abcd",
|
||||
"raw_key": "SECRET-GOOGLE",
|
||||
@@ -140,7 +141,7 @@ def _healthy_probes() -> list[ProbeResult]:
|
||||
_probe("system_config", ProbeStatus.PASS),
|
||||
_probe("toolchain", ProbeStatus.PASS, blocker=False, category=ProbeCategory.TOOLCHAIN),
|
||||
_probe(
|
||||
"gemini_api_key",
|
||||
"openai_api_key",
|
||||
ProbeStatus.PASS,
|
||||
category=ProbeCategory.CREDENTIALS,
|
||||
metadata=_credentials_metadata(),
|
||||
@@ -286,6 +287,15 @@ def _run(
|
||||
):
|
||||
"""Run the tool with every side-effecting collaborator stubbed."""
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(
|
||||
patch.object(
|
||||
type(diagnose.settings),
|
||||
"get_api_key",
|
||||
side_effect=lambda p: SecretStr(
|
||||
{"openai": "SECRET-OPENAI", "ocr": "SECRET-OCR"}[p]
|
||||
),
|
||||
)
|
||||
)
|
||||
fake_helper = MagicMock()
|
||||
fake_helper.status = MagicMock(
|
||||
side_effect=lambda serial: dict(helper_status or _helper_healthy(serial))
|
||||
@@ -462,7 +472,7 @@ def test_ready_environment_reports_ready_with_slim_shape(temp_trace_env):
|
||||
"python_runtime",
|
||||
"system_config",
|
||||
"integration_host",
|
||||
"gemini_api_key",
|
||||
"openai_api_key",
|
||||
"android_adb",
|
||||
]
|
||||
assert result["device"] == {
|
||||
@@ -508,10 +518,10 @@ def test_failing_checks_keep_scrubbed_facts_and_fix_list(temp_trace_env):
|
||||
probes[5].status = ProbeStatus.WARN
|
||||
result = _run(probes)
|
||||
|
||||
creds = next(c for c in result["checks"] if c["id"] == "gemini_api_key")
|
||||
creds = next(c for c in result["checks"] if c["id"] == "openai_api_key")
|
||||
assert creds["category"] == "auth"
|
||||
assert creds["facts"]["providers"] == [
|
||||
{"provider": "google", "label": "Gemini", "masked": "AIza...abcd"},
|
||||
{"provider": "openai", "label": "Gemini", "masked": "AIza...abcd"},
|
||||
{"provider": "openai", "label": "ChatGPT", "masked": "sk-...wxyz"},
|
||||
]
|
||||
assert "api_keys" not in creds["facts"]
|
||||
@@ -635,7 +645,7 @@ def test_default_emulator_command_without_installed_avd_is_not_run(temp_trace_en
|
||||
def test_missing_credentials_point_to_env_file_not_chat(temp_trace_env):
|
||||
probes = _healthy_probes()
|
||||
probes[3] = _probe(
|
||||
"gemini_api_key",
|
||||
"openai_api_key",
|
||||
ProbeStatus.FAIL,
|
||||
category=ProbeCategory.CREDENTIALS,
|
||||
summary="Key Missing",
|
||||
@@ -816,51 +826,19 @@ def test_verify_credentials_reports_every_provider_without_leaking_keys(temp_tra
|
||||
|
||||
assert result["verdict"] == "ready"
|
||||
assert result["credentials"] == [
|
||||
{"provider": "google", "label": "Gemini", "valid": True, "message": "verified"},
|
||||
{"provider": "openai", "label": "ChatGPT", "valid": True, "message": "verified"},
|
||||
{"provider": "openai", "label": "OpenAI Responses", "valid": True, "message": "verified"},
|
||||
{"provider": "ocr", "label": "Vision OCR", "valid": True, "message": "verified"},
|
||||
]
|
||||
called = {call.args[0]: call.args[1] for call in validate.await_args_list}
|
||||
assert called == {"google": "SECRET-GOOGLE", "openai": "SECRET-OPENAI", "ocr": "SECRET-OCR"}
|
||||
assert called == {"openai": "SECRET-OPENAI", "ocr": "SECRET-OCR"}
|
||||
assert all(call.kwargs["timeout"] == 12.0 for call in validate.await_args_list)
|
||||
assert "SECRET" not in repr(result)
|
||||
|
||||
|
||||
def test_verify_credentials_calls_endpoint_providers_by_url(temp_trace_env):
|
||||
"""A custom / Ollama / vLLM entry carries its base URL as the "key": the
|
||||
check must hit that endpoint (no auth header) instead of validating the
|
||||
URL as if it were an API key."""
|
||||
validate = AsyncMock(return_value=(True, "verified"))
|
||||
cred = _probe(
|
||||
"gemini_api_key",
|
||||
ProbeStatus.PASS,
|
||||
category=ProbeCategory.CREDENTIALS,
|
||||
metadata={
|
||||
"providers": [
|
||||
{
|
||||
"provider": "ollama",
|
||||
"label": "Local Ollama",
|
||||
"raw_key": "http://localhost:11434/v1",
|
||||
},
|
||||
{"provider": "google", "label": "Gemini", "raw_key": "SECRET-GOOGLE"},
|
||||
],
|
||||
"api_keys": {},
|
||||
},
|
||||
)
|
||||
probes = [p for p in _healthy_probes() if p.id != "gemini_api_key"] + [cred]
|
||||
_run(probes, validate=validate, verify_credentials=True)
|
||||
|
||||
calls = {call.args[0]: call for call in validate.await_args_list}
|
||||
assert calls["ollama"].args[1] == "EMPTY"
|
||||
assert calls["ollama"].kwargs["base_url"] == "http://localhost:11434/v1"
|
||||
assert calls["google"].args[1] == "SECRET-GOOGLE"
|
||||
assert "base_url" not in calls["google"].kwargs
|
||||
|
||||
|
||||
def test_invalid_primary_credential_blocks_and_redacts_message(temp_trace_env):
|
||||
async def _validate(provider, api_key, timeout=12.0):
|
||||
if provider == "google":
|
||||
return False, f"Gemini API verification failed (400): key {api_key} invalid"
|
||||
if provider == "openai":
|
||||
return False, f"OpenAI Responses API verification failed (400): key {api_key} invalid"
|
||||
if provider == "ocr":
|
||||
raise RuntimeError("boom SECRET-OCR")
|
||||
return True, "ok"
|
||||
@@ -874,29 +852,30 @@ def test_invalid_primary_credential_blocks_and_redacts_message(temp_trace_env):
|
||||
|
||||
assert result["verdict"] == "blocked"
|
||||
assert "SECRET" not in repr(result)
|
||||
google, openai, ocr = result["credentials"]
|
||||
assert google["valid"] is False and "***" in google["message"]
|
||||
assert openai["valid"] is True
|
||||
openai, ocr = result["credentials"]
|
||||
assert openai["valid"] is False and "***" in openai["message"]
|
||||
assert ocr["valid"] is False and ocr["message"].startswith("verification raised RuntimeError")
|
||||
steps = result["next_steps"]
|
||||
required = next(s for s in steps if s.startswith("[REQUIRED] Gemini API key (google)"))
|
||||
required = next(
|
||||
s for s in steps if s.startswith("[REQUIRED] OpenAI Responses API key (openai)")
|
||||
)
|
||||
assert "failed live verification" in required
|
||||
assert steps[steps.index(required) + 1].startswith(" Ask the user to add a provider key")
|
||||
assert any("/home/u/.env" in s for s in steps)
|
||||
assert any(s.startswith("[OPTIONAL] Vision OCR API key (ocr)") for s in steps)
|
||||
assert steps[-1].startswith("After changing the env file")
|
||||
assert "Gemini key verification" in result["summary"]
|
||||
assert "OpenAI Responses key verification" in result["summary"]
|
||||
|
||||
|
||||
def test_invalid_secondary_credential_only_degrades(temp_trace_env):
|
||||
async def _validate(provider, api_key, timeout=12.0):
|
||||
return (provider != "openai", "ok" if provider != "openai" else "401 unauthorized")
|
||||
return (provider != "ocr", "ok" if provider != "ocr" else "401 unauthorized")
|
||||
|
||||
result = _run(
|
||||
_healthy_probes(), validate=AsyncMock(side_effect=_validate), verify_credentials=True
|
||||
)
|
||||
assert result["verdict"] == "degraded"
|
||||
assert any(s.startswith("[OPTIONAL] ChatGPT API key (openai)") for s in result["next_steps"])
|
||||
assert any(s.startswith("[OPTIONAL] Vision OCR API key (ocr)") for s in result["next_steps"])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
@@ -1005,7 +1005,7 @@ async def test_capsule_outage_without_fallback_exhausts_to_failed():
|
||||
assert service.get_summary("chunk:1-2") is None
|
||||
|
||||
|
||||
def test_capsule_fallback_model_resolution_google_only_and_not_primary():
|
||||
def test_capsule_fallback_model_resolution_openai_only_and_not_primary():
|
||||
mgr = HistoryChunkManager(
|
||||
capsule_service=StubCapsuleService(),
|
||||
chunking_config=SimpleNamespace(
|
||||
@@ -1024,13 +1024,13 @@ def test_capsule_fallback_model_resolution_google_only_and_not_primary():
|
||||
)
|
||||
|
||||
assert (
|
||||
mgr._resolve_capsule_fallback_model(ctx_with("google", "gemini-3.6-flash"))
|
||||
mgr._resolve_capsule_fallback_model(ctx_with("openai", "gemini-3.6-flash"))
|
||||
== "gemini-3.6-flash"
|
||||
)
|
||||
# Non-google fallbacks cannot ride the raw google model path.
|
||||
assert mgr._resolve_capsule_fallback_model(ctx_with("openai", "gpt-4o-mini")) is None
|
||||
# Non-Responses providers are rejected.
|
||||
assert mgr._resolve_capsule_fallback_model(ctx_with("google", "gpt-4o-mini")) is None
|
||||
# A fallback identical to the primary adds nothing.
|
||||
assert mgr._resolve_capsule_fallback_model(ctx_with("google", "gemini-3.7-flash")) is None
|
||||
assert mgr._resolve_capsule_fallback_model(ctx_with("openai", "gemini-3.7-flash")) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -728,7 +728,7 @@ def _all_pass_probes():
|
||||
_probe("toolchain", blocker=False),
|
||||
_probe("android_adb", metadata={"installed": True, "adb_keys": {"is_corrupted": False}}),
|
||||
_probe(
|
||||
"gemini_api_key",
|
||||
"openai_api_key",
|
||||
summary="Active (Gemini)",
|
||||
description="Gemini credential is active (sk-RAW...cdef).",
|
||||
metadata={
|
||||
@@ -815,7 +815,7 @@ def test_cli_doctor_all_pass_renders_table_in_fix_order(monkeypatch):
|
||||
"Title python_runtime",
|
||||
"Title system_config",
|
||||
"Title integration_host",
|
||||
"Title gemini_api_key",
|
||||
"Title openai_api_key",
|
||||
"Title android_adb",
|
||||
"Title toolchain",
|
||||
"Title vision_ocr_key",
|
||||
@@ -917,7 +917,7 @@ def test_cli_doctor_json_shape_and_ready_verdict(monkeypatch):
|
||||
"python_runtime",
|
||||
"system_config",
|
||||
"integration_host",
|
||||
"gemini_api_key",
|
||||
"openai_api_key",
|
||||
"android_adb",
|
||||
"toolchain",
|
||||
"vision_ocr_key",
|
||||
@@ -935,10 +935,10 @@ def test_cli_doctor_json_blocked_verdict_and_exit_code(monkeypatch):
|
||||
"""A blocker that is not PASS makes the verdict 'blocked' and the exit code 1."""
|
||||
import json
|
||||
|
||||
probes = [p for p in _all_pass_probes() if p.id != "gemini_api_key"]
|
||||
probes = [p for p in _all_pass_probes() if p.id != "openai_api_key"]
|
||||
probes.append(
|
||||
_probe(
|
||||
"gemini_api_key",
|
||||
"openai_api_key",
|
||||
status="fail",
|
||||
summary="Key Missing",
|
||||
description="No LLM credential found.",
|
||||
@@ -954,7 +954,7 @@ def test_cli_doctor_json_blocked_verdict_and_exit_code(monkeypatch):
|
||||
assert result.exit_code == 1, result.output
|
||||
doc = json.loads(result.output)
|
||||
assert doc["verdict"] == "blocked"
|
||||
key_check = next(c for c in doc["checks"] if c["id"] == "gemini_api_key")
|
||||
key_check = next(c for c in doc["checks"] if c["id"] == "openai_api_key")
|
||||
assert key_check["status"] == "fail"
|
||||
assert key_check["required"] is True
|
||||
assert key_check["fix"] == [
|
||||
|
||||
@@ -93,25 +93,23 @@ def test_settings_and_api_key_fallbacks(monkeypatch):
|
||||
OCR_API_KEY=SecretStr("ocr_key_789"),
|
||||
)
|
||||
# Automatic fallback should populate GOOGLE_API_KEY from GEMINI_API_KEY
|
||||
assert custom_settings.GOOGLE_API_KEY is not None
|
||||
assert custom_settings.GOOGLE_API_KEY.get_secret_value() == "gemini_key_123"
|
||||
assert custom_settings.get_api_key("ocr").get_secret_value() == "ocr_key_789"
|
||||
|
||||
assert custom_settings.get_api_key("google").get_secret_value() == "gemini_key_123"
|
||||
assert custom_settings.get_api_key("gemini").get_secret_value() == "gemini_key_123"
|
||||
assert custom_settings.get_api_key("google") is None
|
||||
assert custom_settings.get_api_key("gemini") is None
|
||||
assert custom_settings.get_api_key("openai").get_secret_value() == "openai_key_456"
|
||||
assert custom_settings.get_api_key("nonexistent") is None
|
||||
|
||||
# Test dynamic set_api_key
|
||||
custom_settings.set_api_key("xai", "xai_test_key", persist_to_env=False)
|
||||
assert custom_settings.get_api_key("xai").get_secret_value() == "xai_test_key"
|
||||
custom_settings.set_api_key("openai", "new_test_key", persist_to_env=False)
|
||||
assert custom_settings.get_api_key("openai").get_secret_value() == "new_test_key"
|
||||
|
||||
|
||||
def test_placeholder_api_key_filtering(monkeypatch):
|
||||
"""Test that default template placeholders are filtered out and treated as unconfigured."""
|
||||
from artemis.config.settings import is_placeholder_key
|
||||
|
||||
assert is_placeholder_key("your_gemini_api_key_here") is True
|
||||
assert is_placeholder_key("your_openai_api_key_here") is True
|
||||
assert is_placeholder_key("your_google_cloud_vision_api_key_here") is True
|
||||
assert is_placeholder_key("your_openai_api_key_here") is True
|
||||
assert is_placeholder_key("your_custom_key_here") is True
|
||||
@@ -138,7 +136,7 @@ def test_placeholder_api_key_filtering(monkeypatch):
|
||||
|
||||
placeholder_settings = Settings(
|
||||
_env_file=None,
|
||||
GEMINI_API_KEY=SecretStr("your_gemini_api_key_here"),
|
||||
GEMINI_API_KEY=SecretStr("your_openai_api_key_here"),
|
||||
OPENAI_API_KEY=SecretStr("your_openai_api_key_here"),
|
||||
OCR_API_KEY=SecretStr("your_google_cloud_vision_api_key_here"),
|
||||
)
|
||||
|
||||
@@ -34,38 +34,42 @@ def test_extract_error_message():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validate_empty_and_placeholder():
|
||||
is_valid, msg = await validate_api_key("google", "")
|
||||
is_valid, msg = await validate_api_key("openai", "")
|
||||
assert not is_valid
|
||||
assert "empty" in msg.lower()
|
||||
|
||||
is_valid, msg = await validate_api_key("google", "API_KEY")
|
||||
is_valid, msg = await validate_api_key("openai", "API_KEY")
|
||||
assert not is_valid
|
||||
assert "placeholder" in msg.lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validate_gemini_success():
|
||||
with patch("httpx.AsyncClient.get", new_callable=AsyncMock) as mock_get:
|
||||
async def test_validate_responses_success():
|
||||
with patch("httpx.AsyncClient.post", new_callable=AsyncMock) as mock_get:
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.json.return_value = {"output": [{"type": "message"}]}
|
||||
mock_get.return_value = mock_resp
|
||||
|
||||
is_valid, msg = await validate_api_key("google", "valid_key_12345")
|
||||
is_valid, msg = await validate_api_key("openai", "valid_key_12345")
|
||||
assert is_valid
|
||||
assert "verified successfully" in msg
|
||||
assert "验证成功" in msg
|
||||
assert mock_get.call_args.args[0].endswith("/responses")
|
||||
assert mock_get.call_args.kwargs["json"]["store"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validate_gemini_failure():
|
||||
with patch("httpx.AsyncClient.get", new_callable=AsyncMock) as mock_get:
|
||||
async def test_validate_responses_failure():
|
||||
with patch("httpx.AsyncClient.post", new_callable=AsyncMock) as mock_get:
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 400
|
||||
mock_resp.json.return_value = {"error": {"message": "API key not valid."}}
|
||||
mock_get.return_value = mock_resp
|
||||
|
||||
is_valid, msg = await validate_api_key("google", "bad_key_12345")
|
||||
is_valid, msg = await validate_api_key("openai", "bad_key_12345")
|
||||
assert not is_valid
|
||||
assert "API key not valid" in msg
|
||||
assert "HTTP 400" in msg
|
||||
assert "bad_key" not in msg
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Check download presets and install locked packages from an isolated local index."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.unit.test_windows_bootstrap import run_bootstrap
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
pytestmark = pytest.mark.skipif(
|
||||
sys.platform != "win32" or not shutil.which("uv"), reason="Windows and uv required"
|
||||
)
|
||||
|
||||
|
||||
def powershell(tmp_path, script, **env):
|
||||
path = tmp_path / "test.ps1"
|
||||
path.write_text(
|
||||
"$ErrorActionPreference = 'Stop'\n. $env:DOWNLOAD_SCRIPT\n" + script, encoding="ascii"
|
||||
)
|
||||
clean_env = {
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if not key.upper().startswith(("UV_", "NPM_CONFIG_"))
|
||||
}
|
||||
return subprocess.run(
|
||||
["powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", str(path)],
|
||||
env={
|
||||
**clean_env,
|
||||
"DOWNLOAD_SCRIPT": str(ROOT / "scripts/download_config.ps1"),
|
||||
"TEST_ROOT": str(tmp_path),
|
||||
**env,
|
||||
},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=45,
|
||||
)
|
||||
|
||||
|
||||
def test_presets_overrides_and_process_environment(tmp_path):
|
||||
(tmp_path / "config").mkdir()
|
||||
config = json.loads((ROOT / "config/downloads.json").read_text())
|
||||
config["mode"] = "China"
|
||||
config["overrides"]["node_mirror"] = "https://intranet.example/node"
|
||||
(tmp_path / "config/downloads.json").write_text(json.dumps(config))
|
||||
result = powershell(
|
||||
tmp_path,
|
||||
"""
|
||||
Initialize-ArtemisDownloads -ProjectRoot $env:TEST_ROOT
|
||||
if ($ArtemisDownloads.mode -ne 'China') { throw 'Preset not selected' }
|
||||
if ($ArtemisDownloads.node_mirror -ne 'https://intranet.example/node') { throw 'Override ignored' }
|
||||
if ($ArtemisDownloads.python_index -ne 'https://intranet.example/simple') { throw 'Environment ignored' }
|
||||
if ($env:npm_config_registry -ne 'https://registry.npmmirror.com') { throw 'npm mirror missing' }
|
||||
if ($env:UV_HTTP_TIMEOUT -ne '180') { throw 'Timeout missing' }
|
||||
Remove-Item Env:npm_config_registry
|
||||
Remove-Item Env:UV_PYTHON_INSTALL_MIRROR
|
||||
Initialize-ArtemisDownloads -ProjectRoot $env:TEST_ROOT -Mode Official
|
||||
if ($ArtemisDownloads.mode -ne 'Official') { throw 'CLI mode ignored' }
|
||||
if ($env:npm_config_registry) { throw 'Official mode selected a China registry' }
|
||||
""",
|
||||
UV_DEFAULT_INDEX="https://intranet.example/simple",
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
|
||||
@pytest.mark.parametrize("corrupt", [False, True])
|
||||
def test_uv_mirror_bootstrap_verifies_wheel_without_python(tmp_path, corrupt):
|
||||
with zipfile.ZipFile(tmp_path / "fixture.zip", "w") as wheel:
|
||||
wheel.writestr("uv-1.2.3.data/scripts/uv.exe", b"standalone fixture")
|
||||
digest = hashlib.sha256((tmp_path / "fixture.zip").read_bytes()).hexdigest()
|
||||
if corrupt:
|
||||
digest = "0" * 64
|
||||
(tmp_path / "index.html").write_text(
|
||||
f'<a href="../../packages/uv-1.2.3-py3-none-win_amd64.whl#sha256={digest}">uv</a>'
|
||||
)
|
||||
result = powershell(
|
||||
tmp_path,
|
||||
"""
|
||||
function Invoke-DownloadFile {
|
||||
param($Uri, $OutFile)
|
||||
if ($Uri -eq 'https://mirror.example/simple/uv/') {
|
||||
Copy-Item -LiteralPath (Join-Path $env:TEST_ROOT 'index.html') -Destination $OutFile
|
||||
} elseif ($Uri -eq 'https://mirror.example/packages/uv-1.2.3-py3-none-win_amd64.whl') {
|
||||
Copy-Item -LiteralPath (Join-Path $env:TEST_ROOT 'fixture.zip') -Destination $OutFile
|
||||
} else { throw "Unexpected URL: $Uri" }
|
||||
return $true
|
||||
}
|
||||
Install-ArtemisUvFromIndex -IndexUrl 'https://mirror.example/simple' -InstallDir (Join-Path $env:TEST_ROOT 'bin')
|
||||
""",
|
||||
PROCESSOR_ARCHITECTURE="AMD64",
|
||||
PROCESSOR_ARCHITEW6432="AMD64",
|
||||
)
|
||||
if corrupt:
|
||||
assert result.returncode != 0
|
||||
assert "SHA256 verification" in result.stderr
|
||||
assert not (tmp_path / "bin/uv.exe").exists()
|
||||
else:
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert (tmp_path / "bin/uv.exe").read_bytes() == b"standalone fixture"
|
||||
|
||||
|
||||
def test_mirror_installs_locked_version_and_reuses_it_without_changing_lock(tmp_path):
|
||||
project = tmp_path / "project"
|
||||
project.mkdir()
|
||||
package_dir = tmp_path / "simple/mirror-fixture"
|
||||
package_dir.mkdir(parents=True)
|
||||
wheel = package_dir / "mirror_fixture-1.0.0-py3-none-any.whl"
|
||||
with zipfile.ZipFile(wheel, "w") as archive:
|
||||
archive.writestr("mirror_fixture.py", 'VERSION = "1.0.0"\n')
|
||||
archive.writestr(
|
||||
"mirror_fixture-1.0.0.dist-info/METADATA",
|
||||
"Metadata-Version: 2.1\nName: mirror-fixture\nVersion: 1.0.0\n",
|
||||
)
|
||||
archive.writestr(
|
||||
"mirror_fixture-1.0.0.dist-info/WHEEL",
|
||||
"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n",
|
||||
)
|
||||
archive.writestr("mirror_fixture-1.0.0.dist-info/RECORD", "")
|
||||
digest = hashlib.sha256(wheel.read_bytes()).hexdigest()
|
||||
(package_dir / "index.html").write_text(
|
||||
f'<a href="{wheel.name}#sha256={digest}">{wheel.name}</a>'
|
||||
)
|
||||
(project / "pyproject.toml").write_text(
|
||||
'[project]\nname="bootstrap-fixture"\nversion="0.1.0"\n'
|
||||
'requires-python=">=3.12"\ndependencies=["mirror-fixture==1.0.0"]\n'
|
||||
)
|
||||
for module in ("artemis", "uvicorn", "fastapi", "langchain_openai"):
|
||||
(project / f"{module}.py").write_text("")
|
||||
lock = f"""version = 1
|
||||
revision = 3
|
||||
requires-python = ">=3.12"
|
||||
[[package]]
|
||||
name = "bootstrap-fixture"
|
||||
version = "0.1.0"
|
||||
source = {{ virtual = "." }}
|
||||
dependencies = [{{ name = "mirror-fixture" }}]
|
||||
[package.metadata]
|
||||
requires-dist = [{{ name = "mirror-fixture", specifier = "==1.0.0" }}]
|
||||
[[package]]
|
||||
name = "mirror-fixture"
|
||||
version = "1.0.0"
|
||||
source = {{ registry = "https://pypi.org/simple" }}
|
||||
wheels = [{{ url = "https://unreachable.invalid/{wheel.name}", hash = "sha256:{digest}", size = {wheel.stat().st_size} }}]
|
||||
"""
|
||||
(project / "uv.lock").write_text(lock)
|
||||
index = (tmp_path / "simple").as_uri()
|
||||
first = run_bootstrap(project, mirror_index=index)
|
||||
assert first.returncode == 0, first.stdout + first.stderr
|
||||
assert (project / ".venv/Lib/site-packages/mirror_fixture.py").exists()
|
||||
assert (project / "uv.lock").read_text() == lock
|
||||
second = run_bootstrap(project, mirror_index=index)
|
||||
assert second.returncode == 0, second.stdout + second.stderr
|
||||
assert "dependencies are already ready" in second.stdout
|
||||
assert (project / "uv.lock").read_text() == lock
|
||||
@@ -1,79 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
from google.genai.errors import APIError
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.skip(
|
||||
reason="Exploratory live API verification test for Gemini SDK role support"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
"""Initializes the Gemini client. Requires GEMINI_API_KEY in environment."""
|
||||
import os
|
||||
from artemis.config.settings import is_placeholder_key
|
||||
|
||||
key = os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
|
||||
if not key or is_placeholder_key(key):
|
||||
pytest.skip(
|
||||
"Valid GEMINI_API_KEY / GOOGLE_API_KEY not configured for live Gemini role tests"
|
||||
)
|
||||
try:
|
||||
return genai.Client()
|
||||
except Exception as e:
|
||||
pytest.skip(f"Could not initialize genai.Client: {e}")
|
||||
|
||||
|
||||
def test_gemini_role_tool_fails(client):
|
||||
"""Confirm that using role='tool' results in a 400 APIError from Gemini."""
|
||||
contents = [
|
||||
types.Content(
|
||||
role="tool",
|
||||
parts=[types.Part.from_text(text="This is a test payload for tool execution.")],
|
||||
)
|
||||
]
|
||||
|
||||
with pytest.raises(APIError) as exc_info:
|
||||
client.models.generate_content(
|
||||
model="gemini-3.7-flash",
|
||||
contents=contents,
|
||||
)
|
||||
|
||||
# Assert it failed specifically due to the invalid role string
|
||||
error_msg = str(exc_info.value)
|
||||
assert "Invalid role string" in error_msg or "400" in error_msg
|
||||
|
||||
|
||||
def test_gemini_role_user_succeeds(client):
|
||||
"""Confirm that using role='user' (which is the correct fix) succeeds."""
|
||||
contents = [
|
||||
types.Content(
|
||||
role="user",
|
||||
parts=[
|
||||
types.Part.from_text(text="Acknowledge this message with 'ok' and nothing else.")
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
# This should not raise any exceptions
|
||||
response = client.models.generate_content(
|
||||
model="gemini-3.7-flash",
|
||||
contents=contents,
|
||||
)
|
||||
|
||||
assert response.text is not None
|
||||
@@ -1,146 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Provider identity and Gemini capability table."""
|
||||
|
||||
from artemis.llm.google import (
|
||||
gemini_version,
|
||||
is_agentic_video_auto_eligible,
|
||||
is_gemini_model,
|
||||
is_google_chat_model,
|
||||
is_google_family_provider,
|
||||
is_google_provider,
|
||||
resolve_video_processing,
|
||||
strip_provider_prefix,
|
||||
supports_agentic_video,
|
||||
supports_thinking_level,
|
||||
)
|
||||
from artemis.llm.router import ModelProvider
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value, google, family",
|
||||
[
|
||||
("google", True, True),
|
||||
("Gemini", True, True),
|
||||
(ModelProvider.GOOGLE, True, True),
|
||||
(ModelProvider.GEMINI, True, True),
|
||||
("vertexai", False, True),
|
||||
("vertex-ai", False, True),
|
||||
(ModelProvider.VERTEX_AI, False, True),
|
||||
("openai", False, False),
|
||||
("anthropic", False, False),
|
||||
(None, False, False),
|
||||
("", False, False),
|
||||
],
|
||||
)
|
||||
def test_provider_identity_tiers(value, google, family):
|
||||
assert is_google_provider(value) is google
|
||||
assert is_google_family_provider(value) is family
|
||||
|
||||
|
||||
def test_google_chat_model_is_detected_by_class_name():
|
||||
class ChatGoogleGenerativeAI: # noqa: N801 - mirrors the real class name
|
||||
pass
|
||||
|
||||
class ChatVertexAI:
|
||||
pass
|
||||
|
||||
class ChatOpenAI:
|
||||
pass
|
||||
|
||||
assert is_google_chat_model(ChatGoogleGenerativeAI()) is True
|
||||
assert is_google_chat_model(ChatVertexAI()) is True
|
||||
assert is_google_chat_model(ChatOpenAI()) is False
|
||||
assert is_google_chat_model(None) is False
|
||||
|
||||
|
||||
def test_model_name_helpers():
|
||||
assert strip_provider_prefix("google/gemini-3.8-flash") == "gemini-3.8-flash"
|
||||
assert strip_provider_prefix("gemini-3.8-flash") == "gemini-3.8-flash"
|
||||
assert strip_provider_prefix(None) == ""
|
||||
assert is_gemini_model("openrouter/google/gemini-3.7-flash") is True
|
||||
assert is_gemini_model("claude-sonnet-5") is False
|
||||
assert gemini_version("gemini-3.8-flash") == (3, 8)
|
||||
assert gemini_version("gemini-3.5-flash-lite-preview") == (3, 5)
|
||||
# Variants still report their family version (thinking-level gate).
|
||||
assert gemini_version("gemini-3.8-flash-image") == (3, 8)
|
||||
assert gemini_version("gemini-2.5-flash-tts") == (2, 5)
|
||||
assert gemini_version("gemini-robotics-er-2-preview") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, expected",
|
||||
[
|
||||
("gemini-3.8-flash", True),
|
||||
("gemini-3.5-flash-lite", True),
|
||||
("gemini-2.5-flash", False),
|
||||
("gemini-2.0-flash", False),
|
||||
("gemini-1.5-pro", False),
|
||||
("gemini-robotics-er-2-preview", True),
|
||||
("gpt-5", True),
|
||||
],
|
||||
)
|
||||
def test_supports_thinking_level(model, expected):
|
||||
assert supports_thinking_level(model) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, supported, auto",
|
||||
[
|
||||
("gemini-3.8-flash", True, True),
|
||||
("gemini-3.7-flash", True, True),
|
||||
("gemini-3.6-flash", True, True),
|
||||
("google/gemini-3.9-flash", True, True),
|
||||
("gemini-3.5-flash-lite", True, False),
|
||||
("gemini-3.5-flash", False, False),
|
||||
("gemini-3.8-pro", False, False),
|
||||
("gemini-2.5-flash", False, False),
|
||||
("gemini-robotics-er-2-preview", False, False),
|
||||
("claude-sonnet-5", False, False),
|
||||
# Release suffixes stay in the base family.
|
||||
("models/gemini-3.7-flash-preview-09-2026", True, True),
|
||||
("gemini-3.8-flash-latest", True, True),
|
||||
("gemini-3.8-flash-001", True, True),
|
||||
("gemini-3.8-flash-exp", True, True),
|
||||
("gemini-3.5-flash-lite-preview", True, False),
|
||||
# Variant models carry their own capabilities and never inherit.
|
||||
("gemini-3.8-flash-image", False, False),
|
||||
("gemini-3.8-flash-tts", False, False),
|
||||
("gemini-3.8-flash-live", False, False),
|
||||
("gemini-3.8-flash-native-audio", False, False),
|
||||
("gemini-3.8-flash-image-preview", False, False),
|
||||
("gemini-3.8-flash-8b", False, False),
|
||||
],
|
||||
)
|
||||
def test_agentic_video_capability(model, supported, auto):
|
||||
assert supports_agentic_video(model) is supported
|
||||
assert is_agentic_video_auto_eligible(model) is auto
|
||||
|
||||
|
||||
def test_resolve_video_processing_honours_knob_and_capability():
|
||||
assert resolve_video_processing("auto", "gemini-3.8-flash") == "agentic"
|
||||
assert resolve_video_processing("auto", "gemini-3.5-flash-lite") == "static"
|
||||
assert resolve_video_processing("auto", "gemini-2.5-flash") == "static"
|
||||
assert resolve_video_processing("agentic", "gemini-3.5-flash-lite") == "agentic"
|
||||
assert resolve_video_processing("agentic", "gemini-2.5-flash") == "static"
|
||||
assert resolve_video_processing("static", "gemini-3.8-flash") == "static"
|
||||
# Explicit "agentic" is honoured on Flash-Lite but never on a variant.
|
||||
assert resolve_video_processing("agentic", "gemini-3.8-flash-image") == "static"
|
||||
assert resolve_video_processing("auto", "gemini-3.8-flash-tts") == "static"
|
||||
# Unknown or non-string knobs behave like "auto".
|
||||
assert resolve_video_processing(None, "gemini-3.8-flash") == "agentic"
|
||||
assert resolve_video_processing(object(), "gemini-3.8-flash") == "agentic"
|
||||
assert resolve_video_processing("bogus", "gemini-3.5-flash-lite") == "static"
|
||||
@@ -16,7 +16,7 @@
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from artemis.llm.google import normalize_usage, usage_from_message
|
||||
from artemis.llm.usage import normalize_usage, usage_from_message
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
|
||||
|
||||
@@ -1,164 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for ModelFactory instantiation and grounding tool wiring."""
|
||||
|
||||
from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider
|
||||
|
||||
|
||||
def test_model_factory_anthropic_instantiation():
|
||||
"""Verify Anthropic Claude model instantiation with thinking budget."""
|
||||
anth_ep = ModelEndpoint(
|
||||
provider=ModelProvider.ANTHROPIC,
|
||||
model_name="claude-3-7-sonnet-20250219",
|
||||
api_key="sk-ant-test-key",
|
||||
reasoning_effort="high",
|
||||
)
|
||||
model = ModelFactory.create_model(anth_ep)
|
||||
assert model.model == "claude-3-7-sonnet-20250219"
|
||||
assert getattr(model, "thinking", None) == {"type": "enabled", "budget_tokens": 32768}
|
||||
|
||||
|
||||
def test_model_factory_openai_instantiation():
|
||||
"""Verify OpenAI model instantiation with reasoning effort."""
|
||||
oai_ep = ModelEndpoint(
|
||||
provider=ModelProvider.OPENAI,
|
||||
model_name="o3-mini",
|
||||
api_key="sk-test-key",
|
||||
reasoning_effort="medium",
|
||||
)
|
||||
model = ModelFactory.create_model(oai_ep)
|
||||
assert model.model_name == "o3-mini"
|
||||
assert getattr(model, "reasoning_effort", None) == "medium"
|
||||
|
||||
|
||||
def test_robust_chat_model_wrapper_grounding_google():
|
||||
"""Verify RobustChatModelWrapper auto-injects google_search and server-side tool config for Gemini."""
|
||||
from artemis.services.llm import RobustChatModelWrapper
|
||||
from langchain_core.tools import tool
|
||||
|
||||
@tool
|
||||
def dummy_tool(query: str) -> str:
|
||||
"""A test tool."""
|
||||
return query
|
||||
|
||||
class MockGeminiModel:
|
||||
def __init__(self):
|
||||
self.bound_tools = []
|
||||
self.bound_kwargs = {}
|
||||
|
||||
def bind_tools(self, tools, *args, **kwargs):
|
||||
self.bound_tools = tools
|
||||
self.bound_kwargs = kwargs
|
||||
return self
|
||||
|
||||
# Case 1: enable_grounding=True automatically attaches google_search and tool_config
|
||||
ep_google = ModelEndpoint(
|
||||
provider=ModelProvider.GOOGLE,
|
||||
model_name="gemini-2.5-flash",
|
||||
enable_grounding=True,
|
||||
)
|
||||
wrapper = RobustChatModelWrapper(MockGeminiModel(), endpoint=ep_google)
|
||||
bound = wrapper.bind_tools([dummy_tool])
|
||||
|
||||
assert any(isinstance(t, dict) and "google_search" in t for t in bound.base_model.bound_tools)
|
||||
assert bound.base_model.bound_kwargs.get("tool_config") == {
|
||||
"include_server_side_tool_invocations": True
|
||||
}
|
||||
|
||||
# Case 2: Explicit google_search in tools automatically ensures tool_config
|
||||
ep_google_no_ground = ModelEndpoint(
|
||||
provider=ModelProvider.GOOGLE,
|
||||
model_name="gemini-2.5-flash",
|
||||
enable_grounding=False,
|
||||
)
|
||||
wrapper_explicit = RobustChatModelWrapper(MockGeminiModel(), endpoint=ep_google_no_ground)
|
||||
bound_explicit = wrapper_explicit.bind_tools([dummy_tool, {"google_search": {}}])
|
||||
|
||||
assert any(
|
||||
isinstance(t, dict) and "google_search" in t for t in bound_explicit.base_model.bound_tools
|
||||
)
|
||||
assert bound_explicit.base_model.bound_kwargs.get("tool_config") == {
|
||||
"include_server_side_tool_invocations": True
|
||||
}
|
||||
|
||||
|
||||
def test_robust_chat_model_wrapper_grounding_non_google():
|
||||
"""Verify RobustChatModelWrapper ignores grounding and strips google_search for non-Google providers."""
|
||||
from artemis.services.llm import RobustChatModelWrapper
|
||||
from langchain_core.tools import tool
|
||||
|
||||
@tool
|
||||
def dummy_tool(query: str) -> str:
|
||||
"""A test tool."""
|
||||
return query
|
||||
|
||||
class MockOpenAIModel:
|
||||
def __init__(self):
|
||||
self.bound_tools = []
|
||||
self.bound_kwargs = {}
|
||||
|
||||
def bind_tools(self, tools, *args, **kwargs):
|
||||
self.bound_tools = tools
|
||||
self.bound_kwargs = kwargs
|
||||
return self
|
||||
|
||||
ep_openai = ModelEndpoint(
|
||||
provider=ModelProvider.OPENAI,
|
||||
model_name="gpt-4o",
|
||||
enable_grounding=True,
|
||||
)
|
||||
wrapper = RobustChatModelWrapper(MockOpenAIModel(), endpoint=ep_openai)
|
||||
bound = wrapper.bind_tools([dummy_tool, {"google_search": {}}])
|
||||
|
||||
# Should ignore grounding and strip {"google_search": {}}
|
||||
assert len(bound.base_model.bound_tools) == 1
|
||||
assert bound.base_model.bound_tools[0].name == "dummy_tool"
|
||||
assert "tool_config" not in bound.base_model.bound_kwargs
|
||||
|
||||
|
||||
def test_chat_google_generative_ai_process_tool_config_patch():
|
||||
"""Verify patched ChatGoogleGenerativeAI preserves include_server_side_tool_invocations even with tool_choice."""
|
||||
from artemis.llm.router import ModelFactory
|
||||
from langchain_google_genai import ChatGoogleGenerativeAI
|
||||
from langchain_core.tools import tool
|
||||
|
||||
@tool
|
||||
def dummy_tool(query: str) -> str:
|
||||
"""A test tool."""
|
||||
return query
|
||||
|
||||
# Trigger patching via Google model creation
|
||||
ep = ModelEndpoint(provider=ModelProvider.GOOGLE, model_name="gemini-2.5-flash", api_key="fake")
|
||||
model = ModelFactory.create_model(ep)
|
||||
assert isinstance(model, ChatGoogleGenerativeAI)
|
||||
|
||||
formatted_tools = model._format_tools([dummy_tool, {"google_search": {}}], None)
|
||||
|
||||
# 1. With tool_choice="auto"
|
||||
config_auto = model._process_tool_config("auto", None, formatted_tools)
|
||||
assert config_auto is not None
|
||||
assert config_auto.include_server_side_tool_invocations is True
|
||||
|
||||
# 2. With tool_choice="any"
|
||||
config_any = model._process_tool_config("any", None, formatted_tools)
|
||||
assert config_any is not None
|
||||
assert config_any.include_server_side_tool_invocations is True
|
||||
|
||||
# 3. With explicit tool_config
|
||||
config_explicit = model._process_tool_config(
|
||||
None, {"include_server_side_tool_invocations": True}, formatted_tools
|
||||
)
|
||||
assert config_explicit is not None
|
||||
assert config_explicit.include_server_side_tool_invocations is True
|
||||
@@ -72,83 +72,6 @@ def test_pause_error_is_persisted_and_published(monkeypatch, tmp_path):
|
||||
assert isinstance(payload["timestamp"], float)
|
||||
|
||||
|
||||
def test_provider_sdk_retry_is_persisted_and_published(monkeypatch):
|
||||
engine = _RecordingEngine()
|
||||
monkeypatch.setattr(llm, "_CURRENT_DATA_ENGINE", engine)
|
||||
record = logging.LogRecord(
|
||||
name="google_genai._api_client",
|
||||
level=logging.INFO,
|
||||
pathname=__file__,
|
||||
lineno=1,
|
||||
msg=(
|
||||
"Retrying google.genai request in 1.18 seconds as it raised "
|
||||
"ServerError: 503 UNAVAILABLE: high demand"
|
||||
),
|
||||
args=(),
|
||||
exc_info=None,
|
||||
)
|
||||
|
||||
request_token = llm._ACTIVE_LLM_REQUEST.set(
|
||||
{
|
||||
"request_id": "request-1",
|
||||
"provider": "google",
|
||||
"started_at": 100.0,
|
||||
"retries": [],
|
||||
}
|
||||
)
|
||||
try:
|
||||
llm._ProviderRetryTelemetryHandler().emit(record)
|
||||
finally:
|
||||
llm._ACTIVE_LLM_REQUEST.reset(request_token)
|
||||
|
||||
assert engine.trace_kwargs["type"] == "llm_call"
|
||||
assert engine.trace_kwargs["name"] == "llm_retry"
|
||||
assert engine.trace_kwargs["status"] == "retrying"
|
||||
assert engine.trace_kwargs["payload"] == {
|
||||
"error": "ServerError: 503 UNAVAILABLE: high demand",
|
||||
"delay": 1.18,
|
||||
"provider": "google",
|
||||
"source": "provider_sdk",
|
||||
"recoverable": True,
|
||||
"request_id": "request-1",
|
||||
"scheduled_at": engine.trace_kwargs["payload"]["scheduled_at"],
|
||||
}
|
||||
event_type, payload = engine.published
|
||||
assert event_type == "llm_retrying"
|
||||
assert payload["trace_id"] == "trace-1"
|
||||
assert payload["delay"] == 1.18
|
||||
assert payload["error"].startswith("ServerError: 503")
|
||||
|
||||
|
||||
def test_provider_sdk_retry_is_hidden_for_non_gemini_request(monkeypatch):
|
||||
engine = _RecordingEngine()
|
||||
monkeypatch.setattr(llm, "_CURRENT_DATA_ENGINE", engine)
|
||||
record = logging.LogRecord(
|
||||
name="google_genai._api_client",
|
||||
level=logging.INFO,
|
||||
pathname=__file__,
|
||||
lineno=1,
|
||||
msg="Retrying request in 2 seconds as it raised 503 UNAVAILABLE",
|
||||
args=(),
|
||||
exc_info=None,
|
||||
)
|
||||
request_token = llm._ACTIVE_LLM_REQUEST.set(
|
||||
{
|
||||
"request_id": "request-openai",
|
||||
"provider": "openai",
|
||||
"started_at": 100.0,
|
||||
"retries": [],
|
||||
}
|
||||
)
|
||||
try:
|
||||
llm._ProviderRetryTelemetryHandler().emit(record)
|
||||
finally:
|
||||
llm._ACTIVE_LLM_REQUEST.reset(request_token)
|
||||
|
||||
assert engine.trace_kwargs is None
|
||||
assert engine.published is None
|
||||
|
||||
|
||||
def test_terminal_failure_aggregates_observed_retries_and_total_wait(monkeypatch, tmp_path):
|
||||
engine = _RecordingEngine()
|
||||
monkeypatch.setattr(llm, "_CURRENT_DATA_ENGINE", engine)
|
||||
@@ -158,7 +81,7 @@ def test_terminal_failure_aggregates_observed_retries_and_total_wait(monkeypatch
|
||||
{
|
||||
"error": "503 high demand",
|
||||
"delay": 10.0,
|
||||
"provider": "google",
|
||||
"provider": "openai",
|
||||
"source": "provider_sdk",
|
||||
"request_id": "request-2",
|
||||
"scheduled_at": 101.0,
|
||||
@@ -168,7 +91,7 @@ def test_terminal_failure_aggregates_observed_retries_and_total_wait(monkeypatch
|
||||
request_token = llm._ACTIVE_LLM_REQUEST.set(
|
||||
{
|
||||
"request_id": "request-2",
|
||||
"provider": "google",
|
||||
"provider": "openai",
|
||||
"started_at": 100.0,
|
||||
"retries": retries,
|
||||
}
|
||||
@@ -180,6 +103,6 @@ def test_terminal_failure_aggregates_observed_retries_and_total_wait(monkeypatch
|
||||
|
||||
payload = engine.trace_kwargs["payload"]
|
||||
assert payload["request_id"] == "request-2"
|
||||
assert payload["provider"] == "google"
|
||||
assert payload["provider"] == "openai"
|
||||
assert payload["waited_seconds"] == 20.0
|
||||
assert payload["retries"] == retries
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Transport-level checks: every model call must use /responses."""
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from langchain_core.messages import HumanMessage, ToolMessage
|
||||
from pydantic import BaseModel, SecretStr, ValidationError
|
||||
|
||||
from artemis.config.settings import settings
|
||||
from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider
|
||||
from artemis.services.llm import RobustChatModelWrapper
|
||||
|
||||
|
||||
def response_body(output):
|
||||
return {
|
||||
"id": "resp_test",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "test-model",
|
||||
"output": output,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13},
|
||||
}
|
||||
|
||||
|
||||
def text_output(text):
|
||||
return [
|
||||
{
|
||||
"id": "msg_test",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": text, "annotations": []}],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def transport_model(monkeypatch):
|
||||
import langchain_openai
|
||||
|
||||
original = langchain_openai.ChatOpenAI
|
||||
requests = []
|
||||
outputs = []
|
||||
|
||||
def handle(request):
|
||||
requests.append(request)
|
||||
return httpx.Response(
|
||||
200, json=response_body(outputs.pop(0) if outputs else text_output("OK"))
|
||||
)
|
||||
|
||||
client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
|
||||
monkeypatch.setattr(
|
||||
langchain_openai,
|
||||
"ChatOpenAI",
|
||||
lambda **kwargs: original(**kwargs, http_async_client=client),
|
||||
)
|
||||
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("test-secret"))
|
||||
monkeypatch.setattr(settings, "OPENAI_BASE_URL", "https://responses.example/v1")
|
||||
monkeypatch.delenv("ARTEMIS_FAKE_LLM", raising=False)
|
||||
model = ModelFactory.create_model(ModelEndpoint(model_name="test-model", timeout_seconds=23))
|
||||
return model, requests, outputs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_and_image_use_responses_transport(transport_model):
|
||||
model, requests, _ = transport_model
|
||||
result = await model.ainvoke(
|
||||
[
|
||||
HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": "Describe"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}},
|
||||
]
|
||||
)
|
||||
]
|
||||
)
|
||||
assert requests[0].url == "https://responses.example/v1/responses"
|
||||
payload = json.loads(requests[0].content)
|
||||
assert payload["store"] is False
|
||||
assert payload["input"][0]["content"][1]["type"] == "input_image"
|
||||
assert "messages" not in payload
|
||||
assert result.usage_metadata["total_tokens"] == 13
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_function_call_roundtrip_uses_call_id(transport_model):
|
||||
model, requests, outputs = transport_model
|
||||
outputs.append(
|
||||
[
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "lookup",
|
||||
"call_id": "call_1",
|
||||
"id": "fc_1",
|
||||
"arguments": '{"query":"hello"}',
|
||||
"status": "completed",
|
||||
}
|
||||
]
|
||||
)
|
||||
tool = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "lookup",
|
||||
"description": "Look up a value",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"required": ["query"],
|
||||
},
|
||||
},
|
||||
}
|
||||
messages = [HumanMessage(content="Look up hello")]
|
||||
answer = await model.bind_tools([tool]).ainvoke(messages)
|
||||
assert answer.tool_calls[0]["id"] == "call_1"
|
||||
await model.ainvoke([*messages, answer, ToolMessage(content="found", tool_call_id="call_1")])
|
||||
payload = json.loads(requests[1].content)
|
||||
assert any(
|
||||
item.get("type") == "function_call_output" and item["call_id"] == "call_1"
|
||||
for item in payload["input"]
|
||||
)
|
||||
assert all(r.url.path == "/v1/responses" for r in requests)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structured_output_uses_responses_text_format(transport_model):
|
||||
class Verdict(BaseModel):
|
||||
ok: bool
|
||||
|
||||
model, requests, outputs = transport_model
|
||||
outputs.append(text_output('{"ok":true}'))
|
||||
result = await model.with_structured_output(Verdict, method="json_schema", strict=True).ainvoke(
|
||||
"Check"
|
||||
)
|
||||
assert result.ok
|
||||
payload = json.loads(requests[0].content)
|
||||
assert payload["text"]["format"]["type"] == "json_schema"
|
||||
assert "response_format" not in payload
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider", ["google", "gemini", "anthropic", "openrouter", "ollama", "custom"]
|
||||
)
|
||||
def test_other_provider_transports_are_rejected(provider):
|
||||
with pytest.raises(ValueError, match="only OpenAI Responses"):
|
||||
ModelProvider.from_string(provider)
|
||||
with pytest.raises(ValidationError):
|
||||
ModelEndpoint(provider=provider)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_retries", [0, 3])
|
||||
def test_client_settings_and_cache_credentials(monkeypatch, max_retries):
|
||||
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("first"))
|
||||
monkeypatch.setattr(settings, "OPENAI_MAX_RETRIES", max_retries)
|
||||
endpoint = ModelEndpoint(api_key="test", model_name="gpt-5-mini", reasoning_effort="low")
|
||||
model = ModelFactory.create_model(endpoint)
|
||||
assert model.use_responses_api is True
|
||||
assert model.max_retries == max_retries
|
||||
assert model.reasoning_effort == "low"
|
||||
dynamic = ModelEndpoint()
|
||||
key = dynamic.cache_key()
|
||||
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("second"))
|
||||
assert dynamic.cache_key() != key
|
||||
assert "first" not in repr(key)
|
||||
|
||||
|
||||
def test_web_search_uses_openai_tool():
|
||||
base = MagicMock()
|
||||
wrapper = RobustChatModelWrapper(base, endpoint=ModelEndpoint(enable_grounding=True))
|
||||
wrapper.bind_tools([])
|
||||
base.bind_tools.assert_called_once_with([{"type": "web_search"}])
|
||||
|
||||
|
||||
def test_video_text_fallback_extracts_responses_blocks():
|
||||
from langchain_core.messages import AIMessage
|
||||
from artemis.agents.video_analyzer.universal_engine import _extract_submit_payload
|
||||
|
||||
result = AIMessage(
|
||||
content=[
|
||||
{"type": "reasoning", "summary": [], "id": "rs_1"},
|
||||
{"type": "text", "text": "已完成视频画面检查。", "annotations": []},
|
||||
]
|
||||
)
|
||||
summary, analysis, events = _extract_submit_payload(result)
|
||||
assert summary == analysis == "已完成视频画面检查。"
|
||||
assert events == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wrapper_structured_output_uses_responses_function_call(transport_model):
|
||||
class Verdict(BaseModel):
|
||||
ok: bool
|
||||
|
||||
model, requests, outputs = transport_model
|
||||
outputs.append(
|
||||
[
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": "Verdict",
|
||||
"call_id": "call_verdict",
|
||||
"id": "fc_verdict",
|
||||
"arguments": '{"ok":true}',
|
||||
"status": "completed",
|
||||
}
|
||||
]
|
||||
)
|
||||
wrapper = RobustChatModelWrapper(model, endpoint=ModelEndpoint(model_name="test-model"))
|
||||
result = await wrapper.with_structured_output(Verdict).ainvoke("Check")
|
||||
assert result.ok is True
|
||||
payload = json.loads(requests[0].content)
|
||||
assert requests[0].url.path == "/v1/responses"
|
||||
assert payload["tools"][0]["name"] == "Verdict"
|
||||
assert payload.get("text", {}).get("format", {}).get("type") != "json_schema"
|
||||
@@ -73,7 +73,10 @@ def test_planner_validation_node_defaults_to_lightweight_judge():
|
||||
assert node.model == safety_net.model
|
||||
assert node.provider == safety_net.provider
|
||||
assert node.temperature == 0.0
|
||||
assert "lite" in node.model
|
||||
from artemis.config.settings import settings
|
||||
|
||||
assert node.provider == "openai"
|
||||
assert node.model == settings.OPENAI_MODEL
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import zipfile
|
||||
|
||||
|
||||
def test_archive_keeps_source_and_excludes_local_builds_and_secrets(tmp_path):
|
||||
script = Path(__file__).resolve().parents[2] / "scripts/package_source.py"
|
||||
spec = importlib.util.spec_from_file_location("package_source", script)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
keep = ["start.bat", ".env.example", "uv.lock", "apps/showcase_ui/src/main.ts"]
|
||||
omit = [
|
||||
".env",
|
||||
".env.local",
|
||||
".venv/pyvenv.cfg",
|
||||
"nested/node_modules/foo.js",
|
||||
"apps/showcase_ui/dist/main.js",
|
||||
"artemis/resources/showcase_ui/main.js",
|
||||
"__pycache__/a.pyc",
|
||||
"traces/data.db",
|
||||
"helper.apk",
|
||||
"debug.keystore",
|
||||
".git/config",
|
||||
]
|
||||
for name in keep + omit:
|
||||
target = tmp_path / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text("fixture")
|
||||
output = tmp_path / "artifacts/source.zip"
|
||||
count, digest = module.package_source(tmp_path, output)
|
||||
assert count == len(keep)
|
||||
assert len(digest) == 64
|
||||
with zipfile.ZipFile(output) as archive:
|
||||
assert archive.testzip() is None
|
||||
assert set(archive.namelist()) == {f"artemis/{name}" for name in keep}
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Exercise the Windows bootstrap in disposable projects without installing host tools."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
pytestmark = pytest.mark.skipif(
|
||||
sys.platform != "win32" or not shutil.which("uv"), reason="Windows and uv required"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def project(tmp_path):
|
||||
# Import fixtures let the final smoke check run without downloading project dependencies.
|
||||
path = tmp_path / "source with spaces!"
|
||||
path.mkdir()
|
||||
(path / "pyproject.toml").write_text(
|
||||
'[project]\nname="bootstrap-fixture"\nversion="0.1.0"\nrequires-python=">=3.12"\ndependencies=[]\n'
|
||||
)
|
||||
for name in ("artemis", "uvicorn", "fastapi", "langchain_openai"):
|
||||
(path / f"{name}.py").write_text("")
|
||||
subprocess.run(["uv", "lock", "--offline"], cwd=path, check=True, capture_output=True)
|
||||
return path
|
||||
|
||||
|
||||
def run_bootstrap(
|
||||
project, *, missing_uv=False, installer_failure=False, sync_failure=False, mirror_index=""
|
||||
):
|
||||
script = project / "check.ps1"
|
||||
script.write_text(
|
||||
"""
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$realUv = (Get-Command uv).Source
|
||||
. $env:BOOTSTRAP_SCRIPT
|
||||
. (Join-Path (Split-Path $env:BOOTSTRAP_SCRIPT -Parent) 'download_config.ps1')
|
||||
if ($env:TEST_MIRROR) { $script:ArtemisDownloads = @{ python_index = $env:TEST_MIRROR } }
|
||||
$script:uvInstalled = $env:SIMULATE_MISSING_UV -ne '1'
|
||||
function Test-CommandExists { param($Command); return $script:uvInstalled }
|
||||
function Update-EnvironmentPath {}
|
||||
function Invoke-DownloadFile {
|
||||
param($Uri, $OutFile, $TimeoutSec)
|
||||
if ($Uri -ne 'https://astral.sh/uv/install.ps1') { throw 'Unexpected installer source' }
|
||||
Set-Content -LiteralPath $OutFile -Value '# fixture installer'
|
||||
return $true
|
||||
}
|
||||
function powershell.exe {
|
||||
Write-Host 'INSTALLER_CALLED'
|
||||
if ($env:SIMULATE_INSTALL_FAILURE -eq '1') { $global:LASTEXITCODE = 25; return }
|
||||
$script:uvInstalled = $true
|
||||
$global:LASTEXITCODE = 0
|
||||
}
|
||||
function uv {
|
||||
if ($env:SIMULATE_SYNC_FAILURE -eq '1' -and $args[0] -eq 'sync') {
|
||||
$global:LASTEXITCODE = 9
|
||||
return
|
||||
}
|
||||
& $realUv @args
|
||||
}
|
||||
try {
|
||||
$python = Initialize-ArtemisPython -ProjectRoot $PSScriptRoot
|
||||
Write-Host "PYTHON_READY=$python"
|
||||
exit 0
|
||||
} catch { Write-Host "EXPECTED_ERROR=$_"; exit 1 }
|
||||
""",
|
||||
encoding="ascii",
|
||||
)
|
||||
env = dict(os.environ)
|
||||
env.update(
|
||||
BOOTSTRAP_SCRIPT=str(ROOT / "scripts/bootstrap_python.ps1"),
|
||||
SIMULATE_MISSING_UV=str(int(missing_uv)),
|
||||
SIMULATE_INSTALL_FAILURE=str(int(installer_failure)),
|
||||
SIMULATE_SYNC_FAILURE=str(int(sync_failure)),
|
||||
UV_OFFLINE="0" if mirror_index else "1",
|
||||
TEST_MIRROR=mirror_index,
|
||||
)
|
||||
return subprocess.run(
|
||||
["powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", str(script)],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
|
||||
def test_first_setup_and_repeat_reuse_environment(project):
|
||||
first = run_bootstrap(project, missing_uv=True)
|
||||
assert first.returncode == 0, first.stdout + first.stderr
|
||||
assert "INSTALLER_CALLED" in first.stdout
|
||||
assert "Creating .venv" in first.stdout
|
||||
config = project / ".venv/pyvenv.cfg"
|
||||
before = config.stat().st_mtime_ns
|
||||
second = run_bootstrap(project)
|
||||
assert second.returncode == 0, second.stdout + second.stderr
|
||||
assert "INSTALLER_CALLED" not in second.stdout
|
||||
assert "Reusing the existing" in second.stdout
|
||||
assert "dependencies are already ready" in second.stdout
|
||||
assert config.stat().st_mtime_ns == before
|
||||
|
||||
|
||||
def test_installer_failure_stops_before_environment_creation(project):
|
||||
result = run_bootstrap(project, missing_uv=True, installer_failure=True)
|
||||
assert result.returncode == 1
|
||||
assert "uv installation failed" in result.stdout
|
||||
assert not (project / ".venv").exists()
|
||||
|
||||
|
||||
def test_dependency_failure_stops_before_launch(project):
|
||||
result = run_bootstrap(project, sync_failure=True)
|
||||
assert result.returncode == 1
|
||||
assert "Dependency installation failed" in result.stdout
|
||||
assert "PYTHON_READY=" not in result.stdout
|
||||
|
||||
|
||||
def test_incomplete_environment_is_preserved(project):
|
||||
(project / ".venv").mkdir()
|
||||
sentinel = project / ".venv/keep.txt"
|
||||
sentinel.write_text("existing data")
|
||||
result = run_bootstrap(project)
|
||||
assert result.returncode == 1
|
||||
assert "incomplete" in result.stdout
|
||||
assert sentinel.read_text() == "existing data"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exit_code", [0, 7])
|
||||
def test_batch_preserves_exit_code_and_handles_space_and_exclamation_paths(tmp_path, exit_code):
|
||||
root = tmp_path / "source with spaces!"
|
||||
(root / "scripts").mkdir(parents=True)
|
||||
shutil.copy2(ROOT / "start.bat", root / "start.bat")
|
||||
(root / "scripts/start.ps1").write_text(
|
||||
"param([switch]$SetupOnly)\n"
|
||||
"if (-not $SetupOnly) { exit 19 }\n"
|
||||
"if ((Get-Location).Path -ne (Split-Path $PSScriptRoot -Parent)) { exit 20 }\n"
|
||||
f"exit {exit_code}\n",
|
||||
encoding="ascii",
|
||||
)
|
||||
result = subprocess.run(
|
||||
["cmd.exe", "/d", "/c", "call", str(root / "start.bat"), "-SetupOnly"],
|
||||
cwd=tmp_path,
|
||||
env={**os.environ, "ARTEMIS_NO_PAUSE": "1"},
|
||||
capture_output=True,
|
||||
timeout=30,
|
||||
)
|
||||
assert result.returncode == exit_code, result.stdout + result.stderr
|
||||
@@ -55,9 +55,9 @@ def test_tool_declaration_mirrors_the_args_schema(tool):
|
||||
assert lc_tool.args_schema is tool.args_schema
|
||||
|
||||
# And the GenAI export agrees on names and required fields.
|
||||
genai = tool.to_genai_declaration()
|
||||
assert set(genai.parameters.properties) == set(properties)
|
||||
assert list(genai.parameters.required or []) == declaration.parameters["required"]
|
||||
genai = tool.to_tool_declaration()
|
||||
assert set(genai.parameters["properties"]) == set(properties)
|
||||
assert list(genai.parameters.get("required") or []) == declaration.parameters["required"]
|
||||
|
||||
|
||||
def test_declared_types_and_enums():
|
||||
|
||||
@@ -47,10 +47,10 @@ def test_append_note_pure_tool_subclass():
|
||||
assert append_note_pure.args_schema == AppendNoteArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = append_note_pure.to_genai_declaration()
|
||||
declaration = append_note_pure.to_tool_declaration()
|
||||
assert declaration.name == "append_note_pure"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "content" in declaration.parameters.properties
|
||||
assert "key" in declaration.parameters["properties"]
|
||||
assert "content" in declaration.parameters["properties"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -51,10 +51,10 @@ def test_append_note_tool_subclass():
|
||||
assert append_note.args_schema == AppendNoteArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = append_note.to_genai_declaration()
|
||||
declaration = append_note.to_tool_declaration()
|
||||
assert declaration.name == "append_note"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "content" in declaration.parameters.properties
|
||||
assert "key" in declaration.parameters["properties"]
|
||||
assert "content" in declaration.parameters["properties"]
|
||||
|
||||
# Wrapper check
|
||||
assert append_note_wrapper is not None
|
||||
|
||||
@@ -436,9 +436,9 @@ def test_run_adb_command_tool_subclass():
|
||||
assert run_adb_command.args_schema == RunAdbCommandArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = run_adb_command.to_genai_declaration()
|
||||
declaration = run_adb_command.to_tool_declaration()
|
||||
assert declaration.name == "run_adb_command"
|
||||
assert "CommandLine" in declaration.parameters.properties
|
||||
assert "CommandLine" in declaration.parameters["properties"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -479,9 +479,9 @@ def test_manage_task_tool_subclass():
|
||||
assert manage_task.args_schema == ManageTaskArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = manage_task.to_genai_declaration()
|
||||
declaration = manage_task.to_tool_declaration()
|
||||
assert declaration.name == "manage_task"
|
||||
assert "Action" in declaration.parameters.properties
|
||||
assert "Action" in declaration.parameters["properties"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -541,10 +541,10 @@ def test_analyze_task_output_tool_subclass():
|
||||
assert analyze_task_output.args_schema == AnalyzeTaskOutputArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = analyze_task_output.to_genai_declaration()
|
||||
declaration = analyze_task_output.to_tool_declaration()
|
||||
assert declaration.name == "analyze_task_output"
|
||||
assert "TaskId" in declaration.parameters.properties
|
||||
assert "Query" in declaration.parameters.properties
|
||||
assert "TaskId" in declaration.parameters["properties"]
|
||||
assert "Query" in declaration.parameters["properties"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -606,9 +606,9 @@ def test_run_short_adb_command_tool_subclass():
|
||||
assert run_short_adb_command.args_schema == RunShortAdbCommandArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = run_short_adb_command.to_genai_declaration()
|
||||
declaration = run_short_adb_command.to_tool_declaration()
|
||||
assert declaration.name == "run_short_adb_command"
|
||||
assert "CommandLine" in declaration.parameters.properties
|
||||
assert "CommandLine" in declaration.parameters["properties"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -195,9 +195,9 @@ class TestCommitteeTool(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(ask_committee.args_schema, AskCommitteeArgs)
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = ask_committee.to_genai_declaration()
|
||||
declaration = ask_committee.to_tool_declaration()
|
||||
self.assertEqual(declaration.name, "ask_committee")
|
||||
self.assertIn("avatar_directive", declaration.parameters.properties)
|
||||
self.assertIn("avatar_directive", declaration.parameters["properties"])
|
||||
|
||||
# Wrapper check
|
||||
self.assertIsNotNone(ask_committee_wrapper)
|
||||
|
||||
@@ -109,10 +109,10 @@ def test_tool_contract_is_tier_agnostic():
|
||||
assert fields["context_feedback"].default == ""
|
||||
assert fields["context_feedback"].description == ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION
|
||||
|
||||
declaration = ask_explorer.to_genai_declaration()
|
||||
declaration = ask_explorer.to_tool_declaration()
|
||||
assert declaration.name == "ask_explorer"
|
||||
assert set(declaration.parameters.properties) == {"query", "context_feedback"}
|
||||
assert declaration.parameters.required == ["query"]
|
||||
assert set(declaration.parameters["properties"]) == {"query", "context_feedback"}
|
||||
assert declaration.parameters.get("required") == ["query"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [None, "flash", "pro", "ultra"])
|
||||
|
||||
@@ -88,7 +88,7 @@ def test_ask_image_processor_tool_subclass():
|
||||
assert ask_image_processor.args_schema == AskVisionCoderArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = ask_image_processor.to_genai_declaration()
|
||||
declaration = ask_image_processor.to_tool_declaration()
|
||||
assert declaration.name == "ask_image_processor"
|
||||
assert "instruction" in declaration.parameters.properties
|
||||
assert "target_image_id" in declaration.parameters.properties
|
||||
assert "instruction" in declaration.parameters["properties"]
|
||||
assert "target_image_id" in declaration.parameters["properties"]
|
||||
@@ -61,9 +61,9 @@ def test_launch_app_tool_subclass():
|
||||
assert launch_app.args_schema == LaunchAppArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = launch_app.to_genai_declaration()
|
||||
declaration = launch_app.to_tool_declaration()
|
||||
assert declaration.name == "launch_app"
|
||||
assert "app_name" in declaration.parameters.properties
|
||||
assert "app_name" in declaration.parameters["properties"]
|
||||
|
||||
# Wrapper check
|
||||
assert launch_app_wrapper is not None
|
||||
|
||||
@@ -47,7 +47,7 @@ def test_list_notes_pure_tool_subclass():
|
||||
assert list_notes_pure.args_schema == ListNotesArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = list_notes_pure.to_genai_declaration()
|
||||
declaration = list_notes_pure.to_tool_declaration()
|
||||
assert declaration.name == "list_notes_pure"
|
||||
|
||||
|
||||
|
||||
@@ -51,7 +51,7 @@ def test_list_notes_tool_subclass():
|
||||
assert list_notes.args_schema == ListNotesArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = list_notes.to_genai_declaration()
|
||||
declaration = list_notes.to_tool_declaration()
|
||||
assert declaration.name == "list_notes"
|
||||
|
||||
# Wrapper check
|
||||
|
||||
@@ -101,9 +101,9 @@ def test_analyze_logs_tool_subclass():
|
||||
assert analyze_logs.args_schema == AnalyzeLogsArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = analyze_logs.to_genai_declaration()
|
||||
declaration = analyze_logs.to_tool_declaration()
|
||||
assert declaration.name == "analyze_logs"
|
||||
assert "specific_query" in declaration.parameters.properties
|
||||
assert "specific_query" in declaration.parameters["properties"]
|
||||
|
||||
# Wrapper check
|
||||
assert analyze_logs_wrapper is not None
|
||||
|
||||
@@ -66,14 +66,14 @@ def test_object_detection_tool_subclass():
|
||||
assert operator_object_detection.args_schema == OperatorObjectDetectionArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = object_detection.to_genai_declaration()
|
||||
declaration = object_detection.to_tool_declaration()
|
||||
assert declaration.name == "object_detection"
|
||||
assert "image_path" in declaration.parameters.properties
|
||||
assert "queries" in declaration.parameters.properties
|
||||
assert "image_path" in declaration.parameters["properties"]
|
||||
assert "queries" in declaration.parameters["properties"]
|
||||
|
||||
op_declaration = operator_object_detection.to_genai_declaration()
|
||||
op_declaration = operator_object_detection.to_tool_declaration()
|
||||
assert op_declaration.name == "object_detection"
|
||||
assert "queries" in op_declaration.parameters.properties
|
||||
assert "queries" in op_declaration.parameters["properties"]
|
||||
|
||||
# Wrappers
|
||||
assert object_detection_wrapper.tool_fn_getter == get_object_detector_tool
|
||||
|
||||
@@ -77,7 +77,7 @@ def test_ocr_tool_subclass():
|
||||
assert ocr_recognition.args_schema == OcrArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = ocr_recognition.to_genai_declaration()
|
||||
declaration = ocr_recognition.to_tool_declaration()
|
||||
assert declaration.name == "ocr_recognition"
|
||||
|
||||
# Wrapper check
|
||||
|
||||
@@ -59,7 +59,7 @@ def test_get_ui_hierarchy_tool_subclass():
|
||||
assert get_ui_hierarchy.args_schema == GetUiHierarchyArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = get_ui_hierarchy.to_genai_declaration()
|
||||
declaration = get_ui_hierarchy.to_tool_declaration()
|
||||
assert declaration.name == "get_ui_hierarchy"
|
||||
|
||||
# Wrapper check
|
||||
|
||||
@@ -46,11 +46,11 @@ def test_read_logs_tool_subclass():
|
||||
assert read_logs.args_schema == ReadLogsArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = read_logs.to_genai_declaration()
|
||||
declaration = read_logs.to_tool_declaration()
|
||||
assert declaration.name == "read_logs"
|
||||
assert "lines" in declaration.parameters.properties
|
||||
assert "since_time" in declaration.parameters.properties
|
||||
assert "until_time" in declaration.parameters.properties
|
||||
assert "lines" in declaration.parameters["properties"]
|
||||
assert "since_time" in declaration.parameters["properties"]
|
||||
assert "until_time" in declaration.parameters["properties"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -47,11 +47,11 @@ def test_read_note_pure_tool_subclass():
|
||||
assert read_note_pure.args_schema == ReadNoteArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = read_note_pure.to_genai_declaration()
|
||||
declaration = read_note_pure.to_tool_declaration()
|
||||
assert declaration.name == "read_note_pure"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "start_line" in declaration.parameters.properties
|
||||
assert "end_line" in declaration.parameters.properties
|
||||
assert "key" in declaration.parameters["properties"]
|
||||
assert "start_line" in declaration.parameters["properties"]
|
||||
assert "end_line" in declaration.parameters["properties"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -51,11 +51,11 @@ def test_read_note_tool_subclass():
|
||||
assert read_note.args_schema == ReadNoteArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = read_note.to_genai_declaration()
|
||||
declaration = read_note.to_tool_declaration()
|
||||
assert declaration.name == "read_note"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "start_line" in declaration.parameters.properties
|
||||
assert "end_line" in declaration.parameters.properties
|
||||
assert "key" in declaration.parameters["properties"]
|
||||
assert "start_line" in declaration.parameters["properties"]
|
||||
assert "end_line" in declaration.parameters["properties"]
|
||||
|
||||
# Wrapper check
|
||||
assert read_note_wrapper is not None
|
||||
|
||||
@@ -47,10 +47,10 @@ def test_save_note_pure_tool_subclass():
|
||||
assert save_note_pure.args_schema == SaveNoteArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = save_note_pure.to_genai_declaration()
|
||||
declaration = save_note_pure.to_tool_declaration()
|
||||
assert declaration.name == "save_note_pure"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "content" in declaration.parameters.properties
|
||||
assert "key" in declaration.parameters["properties"]
|
||||
assert "content" in declaration.parameters["properties"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -51,10 +51,10 @@ def test_save_note_tool_subclass():
|
||||
assert save_note.args_schema == SaveNoteArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = save_note.to_genai_declaration()
|
||||
declaration = save_note.to_tool_declaration()
|
||||
assert declaration.name == "save_note"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "content" in declaration.parameters.properties
|
||||
assert "key" in declaration.parameters["properties"]
|
||||
assert "content" in declaration.parameters["properties"]
|
||||
|
||||
# Wrapper check
|
||||
assert save_note_wrapper is not None
|
||||
|
||||
@@ -186,14 +186,14 @@ def test_search_logs_tool_subclass():
|
||||
assert search_logs.args_schema == SearchLogsArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = search_logs.to_genai_declaration()
|
||||
declaration = search_logs.to_tool_declaration()
|
||||
assert declaration.name == "search_logs"
|
||||
assert "keyword" in declaration.parameters.properties
|
||||
assert "is_regex" in declaration.parameters.properties
|
||||
assert "lines" in declaration.parameters.properties
|
||||
assert "since_time" in declaration.parameters.properties
|
||||
assert "until_time" in declaration.parameters.properties
|
||||
assert "context_lines" in declaration.parameters.properties
|
||||
assert "keyword" in declaration.parameters["properties"]
|
||||
assert "is_regex" in declaration.parameters["properties"]
|
||||
assert "lines" in declaration.parameters["properties"]
|
||||
assert "since_time" in declaration.parameters["properties"]
|
||||
assert "until_time" in declaration.parameters["properties"]
|
||||
assert "context_lines" in declaration.parameters["properties"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -48,11 +48,11 @@ def test_update_note_pure_tool_subclass():
|
||||
assert update_note_pure.args_schema == UpdateNoteArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = update_note_pure.to_genai_declaration()
|
||||
declaration = update_note_pure.to_tool_declaration()
|
||||
assert declaration.name == "update_note_pure"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "target" in declaration.parameters.properties
|
||||
assert "replacement" in declaration.parameters.properties
|
||||
assert "key" in declaration.parameters["properties"]
|
||||
assert "target" in declaration.parameters["properties"]
|
||||
assert "replacement" in declaration.parameters["properties"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -52,11 +52,11 @@ def test_update_note_tool_subclass():
|
||||
assert update_note.args_schema == UpdateNoteArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = update_note.to_genai_declaration()
|
||||
declaration = update_note.to_tool_declaration()
|
||||
assert declaration.name == "update_note"
|
||||
assert "key" in declaration.parameters.properties
|
||||
assert "target" in declaration.parameters.properties
|
||||
assert "replacement" in declaration.parameters.properties
|
||||
assert "key" in declaration.parameters["properties"]
|
||||
assert "target" in declaration.parameters["properties"]
|
||||
assert "replacement" in declaration.parameters["properties"]
|
||||
|
||||
# Wrapper check
|
||||
assert update_note_wrapper is not None
|
||||
|
||||
@@ -44,10 +44,10 @@ def test_video_analyzer_pure_tool_subclass():
|
||||
assert video_analyzer_pure.args_schema == VideoAnalyzerArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = video_analyzer_pure.to_genai_declaration()
|
||||
declaration = video_analyzer_pure.to_tool_declaration()
|
||||
assert declaration.name == "video_analyzer_pure"
|
||||
assert "time_description" in declaration.parameters.properties
|
||||
assert "purpose" in declaration.parameters.properties
|
||||
assert "time_description" in declaration.parameters["properties"]
|
||||
assert "purpose" in declaration.parameters["properties"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -52,10 +52,10 @@ def test_video_analyzer_tool_subclass():
|
||||
assert video_analyzer.args_schema == VideoAnalyzerArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = video_analyzer.to_genai_declaration()
|
||||
declaration = video_analyzer.to_tool_declaration()
|
||||
assert declaration.name == "video_analyzer"
|
||||
assert "time_description" in declaration.parameters.properties
|
||||
assert "purpose" in declaration.parameters.properties
|
||||
assert "time_description" in declaration.parameters["properties"]
|
||||
assert "purpose" in declaration.parameters["properties"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -47,9 +47,9 @@ def test_wait_tool_subclass():
|
||||
assert wait.args_schema == WaitArgs
|
||||
|
||||
# GenAI FunctionDeclaration export
|
||||
declaration = wait.to_genai_declaration()
|
||||
declaration = wait.to_tool_declaration()
|
||||
assert declaration.name == "wait"
|
||||
assert "seconds" in declaration.parameters.properties
|
||||
assert "seconds" in declaration.parameters["properties"]
|
||||
|
||||
# Wrapper check
|
||||
assert wait_wrapper is not None
|
||||
|
||||
@@ -140,7 +140,7 @@ def test_explorer_does_not_expose_get_ocr_list_when_unconfigured():
|
||||
explorer.denylisted_tools = set()
|
||||
|
||||
exposed = explorer.get_exposed_tools()
|
||||
exposed_names = [t.name for t in exposed]
|
||||
exposed_names = [t["function"]["name"] for t in exposed]
|
||||
assert "get_ocr_list" not in exposed_names
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user