fix: preserve exploration tool history and cap context budget
This commit is contained in:
1 parent
ee256ad7ce
commit
621f30e437
17 files changed
+211
-23
No files matched your search
@@ -401,7 +401,7 @@ limitations under the License.
|
|||||||
@if (runInfoContextPercent() !== null) {
|
@if (runInfoContextPercent() !== null) {
|
||||||
<span class="run-info-value">
|
<span class="run-info-value">
|
||||||
{{ runInfoContextPercent() }}%
|
{{ runInfoContextPercent() }}%
|
||||||
<span class="run-info-sub-inline">/ {{ formatRunTokens(runUsage()?.operator_context_window_tokens) }}</span>
|
<span class="run-info-sub-inline">/ {{ formatRunTokens(runInfoContextWindow()) }}</span>
|
||||||
</span>
|
</span>
|
||||||
} @else {
|
} @else {
|
||||||
<span class="run-info-value is-muted">—</span>
|
<span class="run-info-value is-muted">—</span>
|
||||||
@@ -425,7 +425,9 @@ limitations under the License.
|
|||||||
<span class="run-info-label">屏幕识别</span>
|
<span class="run-info-label">屏幕识别</span>
|
||||||
<span class="run-info-value">{{ runTuningLabel('explore', runInfoTuning()?.explorer_mode) }}</span>
|
<span class="run-info-value">{{ runTuningLabel('explore', runInfoTuning()?.explorer_mode) }}</span>
|
||||||
</div>
|
</div>
|
||||||
} @else {
|
} @else if (runInfoMode() === 'explore') {
|
||||||
|
<div class="run-info-note">探索支持对话指导与人工示范,模型和时限使用独立的探索设置。</div>
|
||||||
|
} @else if (runInfoMode() === 'flash') {
|
||||||
<div class="run-info-note">Flash 无需额外设置。</div>
|
<div class="run-info-note">Flash 无需额外设置。</div>
|
||||||
}
|
}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -35,6 +35,8 @@ import { OverlayModule, ConnectedPosition } from '@angular/cdk/overlay';
|
|||||||
import { StepBlock, PhaseBlock, StepEvent, ActionParam, CheckerResult, StreamResetNotice, DEFAULT_STREAM_RESET_MESSAGE } from '../../core/models/stream.model';
|
import { StepBlock, PhaseBlock, StepEvent, ActionParam, CheckerResult, StreamResetNotice, DEFAULT_STREAM_RESET_MESSAGE } from '../../core/models/stream.model';
|
||||||
import {
|
import {
|
||||||
contextPercent,
|
contextPercent,
|
||||||
|
contextWindow,
|
||||||
|
runMode,
|
||||||
formatCompactTokens,
|
formatCompactTokens,
|
||||||
formatElapsed,
|
formatElapsed,
|
||||||
parseRunTuning,
|
parseRunTuning,
|
||||||
@@ -297,7 +299,9 @@ export class AgentStreamComponent implements AfterViewInit {
|
|||||||
public runUsage = signal<SessionUsage | null>(null);
|
public runUsage = signal<SessionUsage | null>(null);
|
||||||
private runInfoClock = signal<number>(Date.now());
|
private runInfoClock = signal<number>(Date.now());
|
||||||
|
|
||||||
public runInfoIsPro = computed(() => this.getModelClass(this.agentService.viewedModel()?.name) === 'is-pro');
|
public runInfoMode = computed(() => runMode(this.agentService.currentSession(), this.agentService.viewedModel()));
|
||||||
|
public runInfoIsPro = computed(() => this.runInfoMode() === 'pro');
|
||||||
|
public runInfoContextWindow = computed(() => contextWindow(this.runUsage()?.operator_context_window_tokens));
|
||||||
|
|
||||||
public runInfoElapsed = computed(() => {
|
public runInfoElapsed = computed(() => {
|
||||||
const session = this.agentService.currentSession();
|
const session = this.agentService.currentSession();
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import {
|
import {
|
||||||
contextPercent,
|
contextPercent,
|
||||||
|
runMode,
|
||||||
formatCompactTokens,
|
formatCompactTokens,
|
||||||
formatElapsed,
|
formatElapsed,
|
||||||
parseRunTuning,
|
parseRunTuning,
|
||||||
@@ -35,13 +36,14 @@ describe('run-info.util', () => {
|
|||||||
|
|
||||||
describe('contextPercent', () => {
|
describe('contextPercent', () => {
|
||||||
it('reports one-decimal share of the window and clamps at 100', () => {
|
it('reports one-decimal share of the window and clamps at 100', () => {
|
||||||
expect(contextPercent(150_000, 1_000_000)).toBe(15);
|
expect(contextPercent(150_000, 300_000)).toBe(50);
|
||||||
expect(contextPercent(123_456, 1_000_000)).toBe(12.3);
|
expect(contextPercent(123_456, 300_000)).toBe(41.2);
|
||||||
expect(contextPercent(2_000_000, 1_000_000)).toBe(100);
|
expect(contextPercent(2_000_000, 1_000_000)).toBe(100);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('falls back to the 1M window and is null when usage is unknown', () => {
|
it('falls back to the 300k window and is null when usage is unknown', () => {
|
||||||
expect(contextPercent(250_000, undefined)).toBe(25);
|
expect(contextPercent(150_000, undefined)).toBe(50);
|
||||||
|
expect(contextPercent(150_000, 1_000_000)).toBe(50);
|
||||||
expect(contextPercent(null, 1_000_000)).toBeNull();
|
expect(contextPercent(null, 1_000_000)).toBeNull();
|
||||||
expect(contextPercent(undefined, 1_000_000)).toBeNull();
|
expect(contextPercent(undefined, 1_000_000)).toBeNull();
|
||||||
});
|
});
|
||||||
@@ -90,3 +92,23 @@ describe('run-info.util', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
describe('runMode', () => {
|
||||||
|
const model = { name: 'mimo-v2.6-pro', id: 'mimo-v2.6-pro', provider: 'openai', architecture: 'flash' };
|
||||||
|
it('uses recorded execution mode, never the API model name', () => {
|
||||||
|
expect(runMode({ profile: 'pro' }, model)).toBe('pro');
|
||||||
|
expect(runMode(null, model)).toBe('flash');
|
||||||
|
expect(runMode(null, { ...model, name: 'mimo-v2.6-pro', architecture: undefined })).toBe('unknown');
|
||||||
|
});
|
||||||
|
it('distinguishes exploration from a Flash or Pro validation rerun', () => {
|
||||||
|
expect(runMode({ profile: 'pro', exploration_kind: 'exploration' }, model)).toBe('explore');
|
||||||
|
expect(runMode({ profile: 'pro', exploration_kind: 'validation' }, model)).toBe('pro');
|
||||||
|
expect(runMode({ profile: 'flash', exploration_kind: 'validation' }, model)).toBe('flash');
|
||||||
|
});
|
||||||
|
it('supports older saved mode names without defaulting missing data to Flash', () => {
|
||||||
|
expect(runMode(null, { ...model, name: 'ARTEMIS Pro', architecture: undefined })).toBe('pro');
|
||||||
|
expect(runMode(null, { ...model, name: '探索', architecture: undefined })).toBe('explore');
|
||||||
|
expect(runMode(null, null)).toBe('unknown');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -27,10 +27,24 @@ import {
|
|||||||
DEFAULT_EXPLORER_MODE,
|
DEFAULT_EXPLORER_MODE,
|
||||||
DEFAULT_VERIFICATION_LEVEL
|
DEFAULT_VERIFICATION_LEVEL
|
||||||
} from '../core/models/pro-tuning.model';
|
} from '../core/models/pro-tuning.model';
|
||||||
import { SessionRunTuning } from '../core/models/session.model';
|
import { ModelInfo, Session, SessionRunTuning } from '../core/models/session.model';
|
||||||
|
|
||||||
/** Denominator for "context used" when the backend does not report one. */
|
/** Denominator for "context used" when the backend does not report one. */
|
||||||
export const OPERATOR_CONTEXT_WINDOW_TOKENS = 1_000_000;
|
export const OPERATOR_CONTEXT_WINDOW_TOKENS = 300_000;
|
||||||
|
|
||||||
|
/** Execution mode is independent of the API model name (e.g. MiMo Pro). */
|
||||||
|
export function runMode(session?: Partial<Session> | null, model?: ModelInfo | null): 'flash' | 'pro' | 'explore' | 'unknown' {
|
||||||
|
if (session?.exploration_kind === 'exploration') return 'explore';
|
||||||
|
const recorded = session?.profile || session?.model_info?.architecture || model?.architecture;
|
||||||
|
const mode = (recorded || session?.model_info?.name || model?.name || '').toLowerCase().replace(/^artemis\s+/, '');
|
||||||
|
if (mode === '探索' || mode === 'explore') return 'explore';
|
||||||
|
return mode === 'flash' || mode === 'pro' ? mode : 'unknown';
|
||||||
|
}
|
||||||
|
|
||||||
|
export function contextWindow(windowTokens?: number | null): number {
|
||||||
|
const value = Number(windowTokens);
|
||||||
|
return value > 0 && Number.isFinite(value) ? Math.min(value, OPERATOR_CONTEXT_WINDOW_TOKENS) : OPERATOR_CONTEXT_WINDOW_TOKENS;
|
||||||
|
}
|
||||||
|
|
||||||
/** `42s`, `12m 05s`, `1h 02m 05s`. Negative or invalid input reads as `0s`. */
|
/** `42s`, `12m 05s`, `1h 02m 05s`. Negative or invalid input reads as `0s`. */
|
||||||
export function formatElapsed(totalSeconds: number | null | undefined): string {
|
export function formatElapsed(totalSeconds: number | null | undefined): string {
|
||||||
@@ -58,7 +72,7 @@ export function contextPercent(
|
|||||||
windowTokens: number | null | undefined
|
windowTokens: number | null | undefined
|
||||||
): number | null {
|
): number | null {
|
||||||
const used = Number(usedTokens);
|
const used = Number(usedTokens);
|
||||||
const window = Number(windowTokens) || OPERATOR_CONTEXT_WINDOW_TOKENS;
|
const window = contextWindow(windowTokens);
|
||||||
if (!Number.isFinite(used) || used < 0 || usedTokens === null || usedTokens === undefined) {
|
if (!Number.isFinite(used) || used < 0 || usedTokens === null || usedTokens === undefined) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ from collections.abc import Awaitable
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from langchain_core.messages import SystemMessage, ToolMessage
|
from langchain_core.messages import BaseMessage, SystemMessage, ToolMessage
|
||||||
from langchain_core.tools import BaseTool
|
from langchain_core.tools import BaseTool
|
||||||
|
|
||||||
from artemis.agents.validator.tool_declarations import normalize_coordinate_target
|
from artemis.agents.validator.tool_declarations import normalize_coordinate_target
|
||||||
@@ -146,9 +146,9 @@ class OperatorNode:
|
|||||||
from artemis.config.agent import MemoryTranscriptConfig
|
from artemis.config.agent import MemoryTranscriptConfig
|
||||||
|
|
||||||
self._transcript_cfg = MemoryTranscriptConfig()
|
self._transcript_cfg = MemoryTranscriptConfig()
|
||||||
# Index into the rendered message list where this turn's tail begins;
|
# Keep the actual observation, not an index: plugin/exploration context
|
||||||
# set by the transcript build and consumed after the tool loop.
|
# may be prepended after render, shifting every message's position.
|
||||||
self._transcript_turn_base: int | None = None
|
self._transcript_turn_tail: BaseMessage | None = None
|
||||||
self._turn_plan_baseline: str | None = None
|
self._turn_plan_baseline: str | None = None
|
||||||
self._turn_ledger_streak: int = 0
|
self._turn_ledger_streak: int = 0
|
||||||
# Whether the previous turn's replies carried no visible reasoning text;
|
# Whether the previous turn's replies carried no visible reasoning text;
|
||||||
@@ -225,7 +225,7 @@ class OperatorNode:
|
|||||||
active_background_tasks: list[dict] = None,
|
active_background_tasks: list[dict] = None,
|
||||||
newly_finished_tasks: list[dict] = None,
|
newly_finished_tasks: list[dict] = None,
|
||||||
) -> list:
|
) -> list:
|
||||||
self._transcript_turn_base = None
|
self._transcript_turn_tail = None
|
||||||
if getattr(self._transcript_cfg, "enabled", False):
|
if getattr(self._transcript_cfg, "enabled", False):
|
||||||
try:
|
try:
|
||||||
return await self._build_prompt_transcript(
|
return await self._build_prompt_transcript(
|
||||||
@@ -243,7 +243,7 @@ class OperatorNode:
|
|||||||
f"Transcript prompt path failed; falling back to the legacy"
|
f"Transcript prompt path failed; falling back to the legacy"
|
||||||
f" 2-message build for this turn: {e}"
|
f" 2-message build for this turn: {e}"
|
||||||
)
|
)
|
||||||
self._transcript_turn_base = None
|
self._transcript_turn_tail = None
|
||||||
|
|
||||||
plan_and_history = self._build_plan_and_history(steps, task_plan)
|
plan_and_history = self._build_plan_and_history(steps, task_plan)
|
||||||
|
|
||||||
@@ -468,18 +468,19 @@ class OperatorNode:
|
|||||||
mark_ephemeral(tail, builder.ephemeral_indices)
|
mark_ephemeral(tail, builder.ephemeral_indices)
|
||||||
|
|
||||||
messages = ledger.render([tail])
|
messages = ledger.render([tail])
|
||||||
self._transcript_turn_base = len(messages) - 1
|
self._transcript_turn_tail = tail
|
||||||
return messages
|
return messages
|
||||||
|
|
||||||
def _stage_transcript_turn(self, messages: list) -> None:
|
def _stage_transcript_turn(self, messages: list) -> None:
|
||||||
"""Hand this turn's tail + tool-loop products to the ledger."""
|
"""Hand this turn's tail + tool-loop products to the ledger."""
|
||||||
base = self._transcript_turn_base
|
tail = self._transcript_turn_tail
|
||||||
self._transcript_turn_base = None
|
self._transcript_turn_tail = None
|
||||||
if base is None:
|
if tail is None:
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
ledger = getattr(self.ctx, "transcript_ledger", None)
|
ledger = getattr(self.ctx, "transcript_ledger", None)
|
||||||
if ledger is not None:
|
if ledger is not None:
|
||||||
|
base = next(i for i, message in enumerate(messages) if message is tail)
|
||||||
ledger.stage_turn(messages[base:])
|
ledger.stage_turn(messages[base:])
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Failed to stage transcript turn: {e}")
|
logger.error(f"Failed to stage transcript turn: {e}")
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ from pydantic import BaseModel, Field, model_validator
|
|||||||
from artemis.config.constants import (
|
from artemis.config.constants import (
|
||||||
AGENT_CONFIG_FILENAME,
|
AGENT_CONFIG_FILENAME,
|
||||||
DEFAULT_EXPLORER_VERSION,
|
DEFAULT_EXPLORER_VERSION,
|
||||||
|
MAX_CONTEXT_WINDOW_TOKENS,
|
||||||
ExplorerVersion,
|
ExplorerVersion,
|
||||||
)
|
)
|
||||||
from artemis.config.paths import ROOT_DIR, get_config_path
|
from artemis.config.paths import ROOT_DIR, get_config_path
|
||||||
@@ -483,6 +484,7 @@ class MemoryTranscriptConfig(BaseModel):
|
|||||||
context_budget_tokens: int = Field(
|
context_budget_tokens: int = Field(
|
||||||
default=80000,
|
default=80000,
|
||||||
ge=1024,
|
ge=1024,
|
||||||
|
le=MAX_CONTEXT_WINDOW_TOKENS,
|
||||||
description=(
|
description=(
|
||||||
"Measured-token budget for the transcript context (provisional"
|
"Measured-token budget for the transcript context (provisional"
|
||||||
" default pending the on-device baseline; §3.4). The soft/hard"
|
" default pending the on-device baseline; §3.4). The soft/hard"
|
||||||
|
|||||||
@@ -16,6 +16,9 @@
|
|||||||
|
|
||||||
from typing import Literal
|
from typing import Literal
|
||||||
|
|
||||||
|
# Application context ceiling; independent from output and session token totals.
|
||||||
|
MAX_CONTEXT_WINDOW_TOKENS = 300_000
|
||||||
|
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
# Environment Variable Keys
|
# Environment Variable Keys
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
|
|||||||
@@ -81,6 +81,11 @@ _AUTH_MARKERS = ("unauthorized", "forbidden", "api key", "credential")
|
|||||||
# "deadline exceeded"/"deadline_exceeded" cover google-genai SDK timeouts that
|
# "deadline exceeded"/"deadline_exceeded" cover google-genai SDK timeouts that
|
||||||
# surface without an HTTP status code attached.
|
# surface without an HTTP status code attached.
|
||||||
_TIMEOUT_MARKERS = ("timed out", "timeout", "deadline exceeded", "deadline_exceeded")
|
_TIMEOUT_MARKERS = ("timed out", "timeout", "deadline exceeded", "deadline_exceeded")
|
||||||
|
_TOOL_HISTORY_ERRORS = (
|
||||||
|
"no tool call found for function call output",
|
||||||
|
"no tool output found for function call",
|
||||||
|
"tool_call_id not found",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def classify_failure(error: BaseException) -> Failure:
|
def classify_failure(error: BaseException) -> Failure:
|
||||||
@@ -91,6 +96,12 @@ def classify_failure(error: BaseException) -> Failure:
|
|||||||
code = extract_status_code(error)
|
code = extract_status_code(error)
|
||||||
message = str(error).lower()
|
message = str(error).lower()
|
||||||
|
|
||||||
|
# Compatible Responses endpoints can report input validation inside an
|
||||||
|
# HTTP 200 stream as APIError without a status code. Retrying the same
|
||||||
|
# broken call/output history cannot repair it, even on a fallback model.
|
||||||
|
if any(marker in message for marker in _TOOL_HISTORY_ERRORS):
|
||||||
|
return Failure(FailureCategory.BAD_REQUEST, False, False)
|
||||||
|
|
||||||
if code == 429 or any(marker in message for marker in _RATE_LIMIT_MARKERS):
|
if code == 429 or any(marker in message for marker in _RATE_LIMIT_MARKERS):
|
||||||
return Failure(FailureCategory.RATE_LIMIT, True, True)
|
return Failure(FailureCategory.RATE_LIMIT, True, True)
|
||||||
if code in {500, 502, 503, 504} or any(marker in message for marker in _UNAVAILABLE_MARKERS):
|
if code in {500, 502, 503, 504} or any(marker in message for marker in _UNAVAILABLE_MARKERS):
|
||||||
|
|||||||
@@ -94,6 +94,7 @@
|
|||||||
},
|
},
|
||||||
"transcript": {
|
"transcript": {
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
|
// 历史压缩预算,允许范围 1024–300000;上下文参照上限固定为 300k。
|
||||||
"context_budget_tokens": 80000,
|
"context_budget_tokens": 80000,
|
||||||
"start_ratio": 0.35,
|
"start_ratio": 0.35,
|
||||||
"soft_ratio": 0.7,
|
"soft_ratio": 0.7,
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ import threading
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from artemis.data_engine.context_vars import CURRENT_NODE_NAME
|
from artemis.data_engine.context_vars import CURRENT_NODE_NAME
|
||||||
|
from artemis.config.constants import MAX_CONTEXT_WINDOW_TOKENS
|
||||||
from artemis.llm.usage import usage_from_message
|
from artemis.llm.usage import usage_from_message
|
||||||
from artemis.utils.logger import get_logger
|
from artemis.utils.logger import get_logger
|
||||||
|
|
||||||
@@ -52,7 +53,7 @@ _METERS: dict[str, "SessionTokenMeter"] = {}
|
|||||||
# measured against when the UI reports "context used". The prompt size of the
|
# measured against when the UI reports "context used". The prompt size of the
|
||||||
# last executor call (``context_base_tokens`` on its ``llm_usage`` trace) is
|
# last executor call (``context_base_tokens`` on its ``llm_usage`` trace) is
|
||||||
# the numerator; this is the denominator.
|
# the numerator; this is the denominator.
|
||||||
OPERATOR_CONTEXT_WINDOW_TOKENS = 1_000_000
|
OPERATOR_CONTEXT_WINDOW_TOKENS = MAX_CONTEXT_WINDOW_TOKENS
|
||||||
|
|
||||||
# Traced node names whose LLM calls carry the live executor context. Both the
|
# Traced node names whose LLM calls carry the live executor context. Both the
|
||||||
# Pro Operator and the Flash runner are "the operator" from the user's view.
|
# Pro Operator and the Flash runner are "the operator" from the user's view.
|
||||||
|
|||||||
@@ -94,6 +94,7 @@
|
|||||||
},
|
},
|
||||||
"transcript": {
|
"transcript": {
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
|
// 历史压缩预算,允许范围 1024–300000;上下文参照上限固定为 300k。
|
||||||
"context_budget_tokens": 80000,
|
"context_budget_tokens": 80000,
|
||||||
"start_ratio": 0.35,
|
"start_ratio": 0.35,
|
||||||
"soft_ratio": 0.7,
|
"soft_ratio": 0.7,
|
||||||
|
|||||||
@@ -80,3 +80,5 @@ Flash 和 Pro 默认复用只读 Checker,以任务提示词和用户指导中
|
|||||||
2.0.0 插件协议与可运行示例见 [开发规则](plugin-development.md)。单任务和批次全部资料包含各来源截图/录像、插件附件及包内证据索引。
|
2.0.0 插件协议与可运行示例见 [开发规则](plugin-development.md)。单任务和批次全部资料包含各来源截图/录像、插件附件及包内证据索引。
|
||||||
|
|
||||||
通过 `mobile_config(section="capabilities")` 可读取 `recording` 参数;用户授权后可保存全局默认。单任务用 `runtime_overrides={"recording":{"codec":"h264","computer_scope":"window","max_long_edge":1280,"fps":15}}`,批次在 `task_config.recording` 提供同样对象。JSON 配置的 `agent.recording` 覆盖环境默认;不要把录像参数当作模型截图参数。录屏开关继续独立且默认关闭。
|
通过 `mobile_config(section="capabilities")` 可读取 `recording` 参数;用户授权后可保存全局默认。单任务用 `runtime_overrides={"recording":{"codec":"h264","computer_scope":"window","max_long_edge":1280,"fps":15}}`,批次在 `task_config.recording` 提供同样对象。JSON 配置的 `agent.recording` 覆盖环境默认;不要把录像参数当作模型截图参数。录屏开关继续独立且默认关闭。
|
||||||
|
|
||||||
|
运行信息的上下文参照上限固定为 300,000 tokens,前后端使用相同口径。历史压缩预算默认 80,000,JSON 的 `agent.memory.transcript.context_budget_tokens` 可调但不能超过 300,000;这是压缩触发预算,不是单次生成的输出 token 数,也不是整项任务累计 token 的限额。
|
||||||
@@ -46,6 +46,25 @@ def mock_driver():
|
|||||||
return MockDeviceDriver(device_id="fixture-mock-device", width=1080, height=2400)
|
return MockDeviceDriver(device_id="fixture-mock-device", width=1080, height=2400)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def no_host_data_engine_ipc(monkeypatch, tmp_path, request):
|
||||||
|
"""Unit-test events must not reach a user's running desktop queue.
|
||||||
|
|
||||||
|
The reconnect path also reads a port file, so overriding the inherited
|
||||||
|
environment port alone does not isolate tests. IPC tests can explicitly
|
||||||
|
override these discovery functions with their own endpoints or mocks.
|
||||||
|
"""
|
||||||
|
if any(
|
||||||
|
request.node.get_closest_marker(name)
|
||||||
|
for name in ("android", "integration", "e2e", "manual")
|
||||||
|
):
|
||||||
|
return
|
||||||
|
monkeypatch.setattr("artemis.data_engine.engine.read_ipc_port", lambda: None)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"artemis.data_engine.engine.get_ipc_port_file", lambda: tmp_path / "no-host-ipc"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
def isolated_plugin_registry(monkeypatch, tmp_path, request):
|
def isolated_plugin_registry(monkeypatch, tmp_path, request):
|
||||||
"""Unit tests never load or mutate the user's installed plugins and credentials."""
|
"""Unit tests never load or mutate the user's installed plugins and credentials."""
|
||||||
|
|||||||
@@ -80,7 +80,7 @@ def test_usage_sums_all_agents_and_tracks_latest_operator_context(tmp_path):
|
|||||||
assert usage["cached_tokens"] == 90_000
|
assert usage["cached_tokens"] == 90_000
|
||||||
assert usage["operator_context_tokens"] == 150_000
|
assert usage["operator_context_tokens"] == 150_000
|
||||||
assert usage["operator_context_updated_at"] == 13.0
|
assert usage["operator_context_updated_at"] == 13.0
|
||||||
assert usage["operator_context_window_tokens"] == 1_000_000
|
assert usage["operator_context_window_tokens"] == 300_000
|
||||||
assert usage["profile"] == "pro"
|
assert usage["profile"] == "pro"
|
||||||
assert usage["run_tuning"] == {"verification_level": "checkpoints", "explorer_mode": "pro"}
|
assert usage["run_tuning"] == {"verification_level": "checkpoints", "explorer_mode": "pro"}
|
||||||
|
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ from types import SimpleNamespace
|
|||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from langchain_core.messages import HumanMessage, SystemMessage
|
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
|
||||||
|
|
||||||
from artemis.agents.operator.operator import OperatorNode
|
from artemis.agents.operator.operator import OperatorNode
|
||||||
from artemis.agents.operator.prompts import load_operator_prompts
|
from artemis.agents.operator.prompts import load_operator_prompts
|
||||||
@@ -141,6 +141,67 @@ def _transcript_state(**overrides):
|
|||||||
return state
|
return state
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"exploration,plugins", [(False, False), (True, False), (False, True), (True, True)]
|
||||||
|
)
|
||||||
|
async def test_injected_prefix_and_compression_preserve_tool_pairs(
|
||||||
|
monkeypatch, exploration, plugins
|
||||||
|
):
|
||||||
|
from langchain_openai import ChatOpenAI
|
||||||
|
|
||||||
|
ctx = _transcript_ctx()
|
||||||
|
ctx.exploration_id = "test-exploration" if exploration else None
|
||||||
|
ctx.plugins = SimpleNamespace(prompt=lambda: "Plugin instructions") if plugins else None
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"artemis.exploration.runtime.exploration_messages",
|
||||||
|
lambda _: [
|
||||||
|
SystemMessage(content="Exploration instructions"),
|
||||||
|
HumanMessage(content="User guidance"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
monkeypatch.setattr("artemis.agents.operator.operator.get_llm", lambda **_: MagicMock())
|
||||||
|
node = OperatorNode(ctx, transcript_config=MemoryTranscriptConfig(enabled=True))
|
||||||
|
requests = []
|
||||||
|
wire_model = ChatOpenAI(model="test", api_key="test", use_responses_api=True, store=False)
|
||||||
|
|
||||||
|
async def tool_loop(current_messages, **_kwargs):
|
||||||
|
messages = current_messages
|
||||||
|
payload = wire_model._get_request_payload(messages)
|
||||||
|
requests.append(payload["input"])
|
||||||
|
call_id = f"call_{len(requests)}"
|
||||||
|
messages.extend(
|
||||||
|
[
|
||||||
|
AIMessage(
|
||||||
|
content="", tool_calls=[{"name": "click", "args": {"target": 1}, "id": call_id}]
|
||||||
|
),
|
||||||
|
ToolMessage(content="Action Recorded", tool_call_id=call_id),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
return [{"action": "tap"}], None, None, False
|
||||||
|
|
||||||
|
monkeypatch.setattr(node, "_invoke_llm_loop", tool_loop)
|
||||||
|
for turn in range(8):
|
||||||
|
await node(_transcript_state(current_step_id=f"step-{turn}"))
|
||||||
|
if turn == 4:
|
||||||
|
ctx.transcript_ledger.freeze_turns(2, [HumanMessage(content="Summarized first turns")])
|
||||||
|
|
||||||
|
for items in requests:
|
||||||
|
pending, seen = set(), set()
|
||||||
|
for item in items:
|
||||||
|
if item.get("type") == "function_call":
|
||||||
|
assert item["call_id"] not in seen
|
||||||
|
pending.add(item["call_id"])
|
||||||
|
seen.add(item["call_id"])
|
||||||
|
elif item.get("type") == "function_call_output":
|
||||||
|
assert item["call_id"] in pending
|
||||||
|
pending.remove(item["call_id"])
|
||||||
|
assert not pending
|
||||||
|
assert {item["call_id"] for item in requests[-1] if item.get("type") == "function_call"} == {
|
||||||
|
f"call_{n}" for n in range(3, 8)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _action_llm(captured: list):
|
def _action_llm(captured: list):
|
||||||
mock_llm = MagicMock()
|
mock_llm = MagicMock()
|
||||||
mock_response = MagicMock()
|
mock_response = MagicMock()
|
||||||
|
|||||||
@@ -215,6 +215,40 @@ async def test_with_fallback_refuses_to_hide_bad_requests():
|
|||||||
assert calls["fallback"] == 0
|
assert calls["fallback"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("method", ["ainvoke", "complete"])
|
||||||
|
async def test_streamed_tool_pairing_error_does_not_retry_pause_or_fallback(monkeypatch, method):
|
||||||
|
import httpx
|
||||||
|
from openai import APIError
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
|
error = APIError(
|
||||||
|
"No tool call found for function call output with call_id call_test.",
|
||||||
|
request=httpx.Request("POST", "https://example.test/responses"),
|
||||||
|
body=None,
|
||||||
|
)
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
class BrokenHistory:
|
||||||
|
async def ainvoke(self, *_args, **_kwargs):
|
||||||
|
calls.append("invoke")
|
||||||
|
raise error
|
||||||
|
|
||||||
|
async def astream(self, *_args, **_kwargs):
|
||||||
|
calls.append("stream")
|
||||||
|
raise error
|
||||||
|
yield # Make this a stream whose error arrives before any chunks.
|
||||||
|
|
||||||
|
pause, fallback = AsyncMock(), AsyncMock()
|
||||||
|
monkeypatch.setattr(llm_service, "_wait_for_resume", pause)
|
||||||
|
wrapper = RobustChatModelWrapper(BrokenHistory())
|
||||||
|
with pytest.raises(LLMPermanentError):
|
||||||
|
await with_fallback(lambda: getattr(wrapper, method)([]), fallback)
|
||||||
|
assert len(calls) == 1
|
||||||
|
pause.assert_not_called()
|
||||||
|
fallback.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_acomplete_structured_corrective_reask():
|
async def test_acomplete_structured_corrective_reask():
|
||||||
class CorrectsItself:
|
class CorrectsItself:
|
||||||
|
|||||||
@@ -19,6 +19,16 @@ import pytest
|
|||||||
from artemis.config import AgentGlobalConfig, load_agent_config
|
from artemis.config import AgentGlobalConfig, load_agent_config
|
||||||
|
|
||||||
|
|
||||||
|
def test_transcript_budget_cannot_exceed_300k_context_ceiling():
|
||||||
|
from pydantic import ValidationError
|
||||||
|
from artemis.config.agent import MemoryTranscriptConfig
|
||||||
|
|
||||||
|
assert MemoryTranscriptConfig().context_budget_tokens == 80_000
|
||||||
|
assert MemoryTranscriptConfig(context_budget_tokens=300_000).context_budget_tokens == 300_000
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
MemoryTranscriptConfig(context_budget_tokens=300_001)
|
||||||
|
|
||||||
|
|
||||||
def test_memory_defaults():
|
def test_memory_defaults():
|
||||||
cfg = AgentGlobalConfig.model_validate({})
|
cfg = AgentGlobalConfig.model_validate({})
|
||||||
assert cfg.flash.step_summarizer.timeout_seconds == 120.0
|
assert cfg.flash.step_summarizer.timeout_seconds == 120.0
|
||||||
|
|||||||
Reference in new issue
Block a user