perf: preserve conversation prefixes across agent turns
This commit is contained in:
1 parent
1440a1f508
commit
f1bb5d02aa
19 files changed
+552
-205
No files matched your search
@@ -58,6 +58,7 @@ from artemis.llm.reliability import (
|
||||
retry_policy_for,
|
||||
)
|
||||
from artemis.llm.messages import adapt_responses_messages
|
||||
from artemis.llm.prefix_cache import PromptPrefixTracker
|
||||
from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider
|
||||
from artemis.services.token_meter import record_llm_usage
|
||||
from artemis.llm.usage import usage_from_message
|
||||
@@ -671,6 +672,27 @@ class RobustChatModelWrapper:
|
||||
record_llm_usage(engine, response, source=self._endpoint_key())
|
||||
|
||||
def _diagnostics(self, args, kwargs):
|
||||
prefix = None
|
||||
if self.ctx is not None:
|
||||
tracker = getattr(self.ctx, "prompt_prefix_tracker", None)
|
||||
if not isinstance(tracker, PromptPrefixTracker):
|
||||
tracker = PromptPrefixTracker()
|
||||
self.ctx.prompt_prefix_tracker = tracker
|
||||
bound_options = getattr(self.base_model, "kwargs", {})
|
||||
options = {
|
||||
**(bound_options if isinstance(bound_options, dict) else {}),
|
||||
**{
|
||||
key: kwargs[key]
|
||||
for key in ("tools", "tool_choice", "response_format")
|
||||
if key in kwargs
|
||||
},
|
||||
}
|
||||
identity = f"{kwargs.get('prompt_cache_key')}:{self.role}:{self._endpoint_key()}"
|
||||
prefix = tracker.observe(identity, args[0] if args else kwargs.get("input"), options)
|
||||
ledger = getattr(self.ctx, "transcript_ledger", None)
|
||||
if ledger is not None:
|
||||
prefix["context_phase"] = getattr(ledger, "context_phase", 0)
|
||||
prefix["last_compaction"] = getattr(ledger, "last_compaction", None)
|
||||
return {
|
||||
"provider": self._provider_value(),
|
||||
"model": self.endpoint.model_name if self.endpoint else None,
|
||||
@@ -680,6 +702,7 @@ class RobustChatModelWrapper:
|
||||
"is_fallback": self.is_fallback,
|
||||
"images": input_images(args, kwargs),
|
||||
"prompt_cache_key": kwargs.get("prompt_cache_key"),
|
||||
"prefix_cache": prefix,
|
||||
}
|
||||
|
||||
async def _invoke_attempt(self, *args, **kwargs):
|
||||
|
||||
Reference in new issue
Block a user