fix: preserve exploration tool history and cap context budget
This commit is contained in:
1 parent
ee256ad7ce
commit
621f30e437
17 files changed
+211
-23
No files matched your search
@@ -80,7 +80,7 @@ def test_usage_sums_all_agents_and_tracks_latest_operator_context(tmp_path):
|
||||
assert usage["cached_tokens"] == 90_000
|
||||
assert usage["operator_context_tokens"] == 150_000
|
||||
assert usage["operator_context_updated_at"] == 13.0
|
||||
assert usage["operator_context_window_tokens"] == 1_000_000
|
||||
assert usage["operator_context_window_tokens"] == 300_000
|
||||
assert usage["profile"] == "pro"
|
||||
assert usage["run_tuning"] == {"verification_level": "checkpoints", "explorer_mode": "pro"}
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
|
||||
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.agents.operator.prompts import load_operator_prompts
|
||||
@@ -141,6 +141,67 @@ def _transcript_state(**overrides):
|
||||
return state
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"exploration,plugins", [(False, False), (True, False), (False, True), (True, True)]
|
||||
)
|
||||
async def test_injected_prefix_and_compression_preserve_tool_pairs(
|
||||
monkeypatch, exploration, plugins
|
||||
):
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
ctx = _transcript_ctx()
|
||||
ctx.exploration_id = "test-exploration" if exploration else None
|
||||
ctx.plugins = SimpleNamespace(prompt=lambda: "Plugin instructions") if plugins else None
|
||||
monkeypatch.setattr(
|
||||
"artemis.exploration.runtime.exploration_messages",
|
||||
lambda _: [
|
||||
SystemMessage(content="Exploration instructions"),
|
||||
HumanMessage(content="User guidance"),
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr("artemis.agents.operator.operator.get_llm", lambda **_: MagicMock())
|
||||
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
|
||||
requests = []
|
||||
wire_model = ChatOpenAI(model="test", api_key="test", use_responses_api=True, store=False)
|
||||
|
||||
async def tool_loop(current_messages, **_kwargs):
|
||||
messages = current_messages
|
||||
payload = wire_model._get_request_payload(messages)
|
||||
requests.append(payload["input"])
|
||||
call_id = f"call_{len(requests)}"
|
||||
messages.extend(
|
||||
[
|
||||
AIMessage(
|
||||
content="", tool_calls=[{"name": "click", "args": {"target": 1}, "id": call_id}]
|
||||
),
|
||||
ToolMessage(content="Action Recorded", tool_call_id=call_id),
|
||||
]
|
||||
)
|
||||
return [{"action": "tap"}], None, None, False
|
||||
|
||||
monkeypatch.setattr(node, "_invoke_llm_loop", tool_loop)
|
||||
for turn in range(8):
|
||||
await node(_transcript_state(current_step_id=f"step-{turn}"))
|
||||
if turn == 4:
|
||||
ctx.transcript_ledger.freeze_turns(2, [HumanMessage(content="Summarized first turns")])
|
||||
|
||||
for items in requests:
|
||||
pending, seen = set(), set()
|
||||
for item in items:
|
||||
if item.get("type") == "function_call":
|
||||
assert item["call_id"] not in seen
|
||||
pending.add(item["call_id"])
|
||||
seen.add(item["call_id"])
|
||||
elif item.get("type") == "function_call_output":
|
||||
assert item["call_id"] in pending
|
||||
pending.remove(item["call_id"])
|
||||
assert not pending
|
||||
assert {item["call_id"] for item in requests[-1] if item.get("type") == "function_call"} == {
|
||||
f"call_{n}" for n in range(3, 8)
|
||||
}
|
||||
|
||||
|
||||
def _action_llm(captured: list):
|
||||
mock_llm = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
|
||||
@@ -215,6 +215,40 @@ async def test_with_fallback_refuses_to_hide_bad_requests():
|
||||
assert calls["fallback"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("method", ["ainvoke", "complete"])
|
||||
async def test_streamed_tool_pairing_error_does_not_retry_pause_or_fallback(monkeypatch, method):
|
||||
import httpx
|
||||
from openai import APIError
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
error = APIError(
|
||||
"No tool call found for function call output with call_id call_test.",
|
||||
request=httpx.Request("POST", "https://example.test/responses"),
|
||||
body=None,
|
||||
)
|
||||
calls = []
|
||||
|
||||
class BrokenHistory:
|
||||
async def ainvoke(self, *_args, **_kwargs):
|
||||
calls.append("invoke")
|
||||
raise error
|
||||
|
||||
async def astream(self, *_args, **_kwargs):
|
||||
calls.append("stream")
|
||||
raise error
|
||||
yield # Make this a stream whose error arrives before any chunks.
|
||||
|
||||
pause, fallback = AsyncMock(), AsyncMock()
|
||||
monkeypatch.setattr(llm_service, "_wait_for_resume", pause)
|
||||
wrapper = RobustChatModelWrapper(BrokenHistory())
|
||||
with pytest.raises(LLMPermanentError):
|
||||
await with_fallback(lambda: getattr(wrapper, method)([]), fallback)
|
||||
assert len(calls) == 1
|
||||
pause.assert_not_called()
|
||||
fallback.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acomplete_structured_corrective_reask():
|
||||
class CorrectsItself:
|
||||
|
||||
@@ -19,6 +19,16 @@ import pytest
|
||||
from artemis.config import AgentGlobalConfig, load_agent_config
|
||||
|
||||
|
||||
def test_transcript_budget_cannot_exceed_300k_context_ceiling():
|
||||
from pydantic import ValidationError
|
||||
from artemis.config.agent import MemoryTranscriptConfig
|
||||
|
||||
assert MemoryTranscriptConfig().context_budget_tokens == 80_000
|
||||
assert MemoryTranscriptConfig(context_budget_tokens=300_000).context_budget_tokens == 300_000
|
||||
with pytest.raises(ValidationError):
|
||||
MemoryTranscriptConfig(context_budget_tokens=300_001)
|
||||
|
||||
|
||||
def test_memory_defaults():
|
||||
cfg = AgentGlobalConfig.model_validate({})
|
||||
assert cfg.flash.step_summarizer.timeout_seconds == 120.0
|
||||
|
||||
Reference in new issue
Block a user