feat: add tool-call fallback and resilient batch exports
This commit is contained in:
1 parent
ab066733b8
commit
d31eb2a19d
32 files changed
+538
-72
No files matched your search
@@ -1071,6 +1071,8 @@ class FlashRunner:
|
||||
pending_notices: list[str] = []
|
||||
no_tool_attempts = 0
|
||||
no_tool_limit = settings.ARTEMIS_FLASH_MAX_NO_PROGRESS_ATTEMPTS
|
||||
fallback_llm = None
|
||||
using_tool_fallback = False
|
||||
|
||||
while limit is None or turns < limit:
|
||||
control = getattr(self.ctx, "task_control", None)
|
||||
@@ -1080,6 +1082,9 @@ class FlashRunner:
|
||||
on_resume=getattr(self.ctx, "on_task_resume", None),
|
||||
)
|
||||
if resumed:
|
||||
using_tool_fallback = False
|
||||
no_tool_attempts = 0
|
||||
self._llm = llm
|
||||
(
|
||||
shot_path,
|
||||
current_pre_screenshot_bytes,
|
||||
@@ -1131,14 +1136,18 @@ class FlashRunner:
|
||||
|
||||
if isinstance(control, TaskControl):
|
||||
interrupted, response = await control.run_interruptibly(
|
||||
self._invoke_model(llm, current_tools, messages),
|
||||
self._invoke_model(
|
||||
fallback_llm if using_tool_fallback else llm, current_tools, messages
|
||||
),
|
||||
on_pause=getattr(self.ctx, "on_task_pause", None),
|
||||
on_resume=getattr(self.ctx, "on_task_resume", None),
|
||||
)
|
||||
else:
|
||||
interrupted, response = (
|
||||
False,
|
||||
await self._invoke_model(llm, current_tools, messages),
|
||||
await self._invoke_model(
|
||||
fallback_llm if using_tool_fallback else llm, current_tools, messages
|
||||
),
|
||||
)
|
||||
|
||||
if interrupted or (
|
||||
@@ -1146,6 +1155,9 @@ class FlashRunner:
|
||||
):
|
||||
# Do not dispatch a stale action or final answer from a prompt
|
||||
# assembled before the instruction arrived.
|
||||
using_tool_fallback = False
|
||||
no_tool_attempts = 0
|
||||
self._llm = llm
|
||||
(
|
||||
shot_path,
|
||||
current_pre_screenshot_bytes,
|
||||
@@ -1186,6 +1198,9 @@ class FlashRunner:
|
||||
|
||||
tool_calls = self._resolve_tool_calls(response, raw_text)
|
||||
if isinstance(control, TaskControl) and has_pending_instruction(control.directory):
|
||||
using_tool_fallback = False
|
||||
no_tool_attempts = 0
|
||||
self._llm = llm
|
||||
(
|
||||
shot_path,
|
||||
current_pre_screenshot_bytes,
|
||||
@@ -1207,15 +1222,27 @@ class FlashRunner:
|
||||
final_report = {"status": "failed", "explanation": raw_text}
|
||||
break
|
||||
no_tool_attempts += 1
|
||||
if no_tool_attempts >= no_tool_limit:
|
||||
final_report = {
|
||||
"status": "failed",
|
||||
"explanation": (
|
||||
f"模型连续 {no_tool_attempts} 次未调用工具,"
|
||||
"已停止任务以避免空转和重复计费;未执行新的设备动作。"
|
||||
),
|
||||
}
|
||||
break
|
||||
active_limit = (
|
||||
settings.ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS
|
||||
if using_tool_fallback
|
||||
else no_tool_limit
|
||||
)
|
||||
if no_tool_attempts >= active_limit:
|
||||
if using_tool_fallback:
|
||||
final_report = {
|
||||
"status": "failed",
|
||||
"explanation": (
|
||||
f"首选模型连续 {no_tool_limit} 次、兜底模型连续 "
|
||||
f"{no_tool_attempts} 次未调用工具,已停止任务;"
|
||||
"未执行新的设备动作。"
|
||||
),
|
||||
}
|
||||
break
|
||||
fallback_llm = get_llm(self.ctx, name="operator", use_fallback=True)
|
||||
using_tool_fallback = True
|
||||
no_tool_attempts = 0
|
||||
logger.warning("Flash 首选模型未调用工具,当前步骤切换到兜底模型")
|
||||
pending_notices.append("首选模型连续未调用工具;本步骤改用兜底模型。")
|
||||
pending_notices.append(_NO_TOOL_CALL_NOTICE)
|
||||
# Keep the observation and any injected user instruction, but
|
||||
# do not feed the model its own unproductive prose on retry.
|
||||
@@ -1224,6 +1251,10 @@ class FlashRunner:
|
||||
continue
|
||||
|
||||
no_tool_attempts = 0
|
||||
if using_tool_fallback:
|
||||
# A tool-producing turn completes the fallback's scope. The next
|
||||
# observation goes back to the configured primary model.
|
||||
self._llm = fallback_llm
|
||||
# Process tool calls
|
||||
turn = _TurnRecord()
|
||||
(
|
||||
@@ -1244,6 +1275,10 @@ class FlashRunner:
|
||||
injected=injected,
|
||||
native_text=native_text,
|
||||
)
|
||||
if using_tool_fallback and any(status == "success" for _, status, _ in turn.actions):
|
||||
using_tool_fallback = False
|
||||
self._llm = llm
|
||||
pending_notices.append("兜底模型已完成本步骤;后续步骤恢复首选模型。")
|
||||
ledger.stage_turn(messages[turn_base:])
|
||||
previous_turn = turn
|
||||
if final_report_from_calls is not None:
|
||||
|
||||
@@ -508,6 +508,9 @@ class OperatorNode:
|
||||
plan_gate_bounced = False
|
||||
no_progress_attempts = 0
|
||||
no_progress_limit = settings.ARTEMIS_OPERATOR_MAX_NO_PROGRESS_ATTEMPTS
|
||||
fallback_limit = settings.ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS
|
||||
active_llm = base_llm
|
||||
using_tool_fallback = False
|
||||
|
||||
for iteration in range(max_iterations):
|
||||
control = getattr(self.ctx, "task_control", None)
|
||||
@@ -532,7 +535,7 @@ class OperatorNode:
|
||||
)
|
||||
)
|
||||
|
||||
bound_llm = base_llm.bind_tools(tools=traced_tools)
|
||||
bound_llm = active_llm.bind_tools(tools=traced_tools)
|
||||
interrupted, response = await self._await_or_pause(
|
||||
invoke_llm_with_timeout_message(acomplete(bound_llm, current_messages))
|
||||
)
|
||||
@@ -607,11 +610,25 @@ class OperatorNode:
|
||||
|
||||
if not response.tool_calls:
|
||||
no_progress_attempts += 1
|
||||
if no_progress_attempts >= no_progress_limit:
|
||||
raise OperatorNoProgressError(
|
||||
f"模型连续 {no_progress_attempts} 次未提供可执行的工具调用,"
|
||||
"已停止任务以避免空转;未执行新的设备动作。"
|
||||
if no_progress_attempts >= (
|
||||
fallback_limit if using_tool_fallback else no_progress_limit
|
||||
):
|
||||
if using_tool_fallback:
|
||||
raise OperatorNoProgressError(
|
||||
f"首选模型连续 {no_progress_limit} 次、兜底模型连续 "
|
||||
f"{no_progress_attempts} 次未提供可执行的工具调用;"
|
||||
"已停止任务以避免空转。"
|
||||
)
|
||||
active_llm = get_llm(ctx=self.ctx, name="operator", use_fallback=True)
|
||||
using_tool_fallback = True
|
||||
no_progress_attempts = 0
|
||||
logger.warning("Operator 首选模型未调用工具,当前步骤切换到兜底模型")
|
||||
current_messages.append(
|
||||
SystemMessage(
|
||||
content="本步骤已切换兜底模型。请使用工具完成操作或明确报告阻碍。"
|
||||
)
|
||||
)
|
||||
continue
|
||||
logger.warning("LLM returned no tool call; requesting a concrete tool call.")
|
||||
current_messages.append(response)
|
||||
current_messages.append(
|
||||
@@ -931,11 +948,25 @@ class OperatorNode:
|
||||
|
||||
if validation_errors:
|
||||
no_progress_attempts += 1
|
||||
if no_progress_attempts >= no_progress_limit:
|
||||
raise OperatorNoProgressError(
|
||||
f"模型连续 {no_progress_attempts} 次未提供可执行的工具调用,"
|
||||
"工具参数校验仍失败;已停止任务,未执行被拒绝的设备动作。"
|
||||
if no_progress_attempts >= (
|
||||
fallback_limit if using_tool_fallback else no_progress_limit
|
||||
):
|
||||
if using_tool_fallback:
|
||||
raise OperatorNoProgressError(
|
||||
f"首选模型连续 {no_progress_limit} 次、兜底模型连续 "
|
||||
f"{no_progress_attempts} 次未提供可执行的工具调用,"
|
||||
"工具参数校验仍失败;已停止任务。"
|
||||
)
|
||||
active_llm = get_llm(ctx=self.ctx, name="operator", use_fallback=True)
|
||||
using_tool_fallback = True
|
||||
no_progress_attempts = 0
|
||||
logger.warning("Operator 首选模型工具调用校验失败,当前步骤切换到兜底模型")
|
||||
current_messages.append(
|
||||
SystemMessage(
|
||||
content="本步骤已切换兜底模型。请修正上面的工具参数后再调用。"
|
||||
)
|
||||
)
|
||||
continue
|
||||
logger.warning("Some action calls failed validation. Feeding back to LLM.")
|
||||
continue
|
||||
no_progress_attempts = 0
|
||||
|
||||
@@ -100,6 +100,11 @@ class Settings(BaseSettings):
|
||||
OPENAI_TIMEOUT_SECONDS: float = Field(default=300.0, gt=0)
|
||||
OPENAI_MAX_RETRIES: int = Field(default=5, ge=0, le=10)
|
||||
OPENAI_RETRY_INTERVAL_SECONDS: float = Field(default=10.0, ge=0, le=300)
|
||||
OPENAI_RATE_LIMIT_RETRY_SECONDS: float = Field(default=60.0, ge=0, le=3600)
|
||||
OPENAI_TOOL_FALLBACK_MODEL: str = Field(default="gpt-6-luna", min_length=1)
|
||||
OPENAI_TOOL_FALLBACK_REASONING_EFFORT: str = "high"
|
||||
OPENAI_TOOL_FALLBACK_MAX_RETRIES: int = Field(default=3, ge=0, le=10)
|
||||
ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS: int = Field(default=3, ge=1, le=10)
|
||||
OPENAI_REASONING_EFFORT: str = "high"
|
||||
ARTEMIS_CREDENTIAL_TIMEOUT_SECONDS: float = Field(default=12.0, gt=0)
|
||||
ARTEMIS_MODEL_CACHE_TTL_SECONDS: float = Field(default=10.0, ge=0)
|
||||
|
||||
@@ -117,6 +117,11 @@ def init_command() -> None:
|
||||
"OPENAI_TIMEOUT_SECONDS=300",
|
||||
"OPENAI_MAX_RETRIES=5",
|
||||
"OPENAI_RETRY_INTERVAL_SECONDS=10",
|
||||
"OPENAI_RATE_LIMIT_RETRY_SECONDS=60",
|
||||
"OPENAI_TOOL_FALLBACK_MODEL=gpt-6-luna",
|
||||
"OPENAI_TOOL_FALLBACK_REASONING_EFFORT=high",
|
||||
"OPENAI_TOOL_FALLBACK_MAX_RETRIES=3",
|
||||
"ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS=3",
|
||||
"ARTEMIS_VIDEO_ENABLED=false",
|
||||
"ARTEMIS_TRACES_DIR=./traces",
|
||||
"",
|
||||
|
||||
@@ -75,7 +75,7 @@ def extract_status_code(error: BaseException) -> int | None:
|
||||
return None
|
||||
|
||||
|
||||
_RATE_LIMIT_MARKERS = ("rate limit", "quota", "resource_exhausted")
|
||||
_RATE_LIMIT_MARKERS = ("rate limit", "too many requests", "quota", "resource_exhausted")
|
||||
_UNAVAILABLE_MARKERS = ("unavailable", "overloaded", "high demand", "service unavailable")
|
||||
_AUTH_MARKERS = ("unauthorized", "forbidden", "api key", "credential")
|
||||
# "deadline exceeded"/"deadline_exceeded" cover google-genai SDK timeouts that
|
||||
@@ -196,6 +196,10 @@ def retry_policy_for(category: FailureCategory | str) -> RetryPolicy:
|
||||
return _DEFAULT_RETRY_POLICIES[category]
|
||||
from artemis.config.settings import settings
|
||||
|
||||
if category is FailureCategory.RATE_LIMIT:
|
||||
return _configured_policy(
|
||||
settings.OPENAI_MAX_RETRIES, settings.OPENAI_RATE_LIMIT_RETRY_SECONDS
|
||||
)
|
||||
return _configured_policy(settings.OPENAI_MAX_RETRIES, settings.OPENAI_RETRY_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
|
||||
+24
-13
@@ -15,25 +15,36 @@
|
||||
"""Backward compatibility entrypoint for artemis CLI."""
|
||||
|
||||
import sys
|
||||
|
||||
import typer
|
||||
|
||||
from artemis.interfaces.cli.main import app
|
||||
from artemis.interfaces.cli.main import app as _typer_app
|
||||
|
||||
|
||||
def _root_tokens() -> frozenset[str]:
|
||||
"""Read command names and root flags from the registered CLI."""
|
||||
group = typer.main.get_command(_typer_app)
|
||||
tokens: set[str] = {"-h"}
|
||||
tokens.update(getattr(group, "commands", {}) or {})
|
||||
tokens.update(group.get_help_option_names(group.context_class(group)))
|
||||
for param in group.params:
|
||||
tokens.update(getattr(param, "opts", ()))
|
||||
tokens.update(getattr(param, "secondary_opts", ()))
|
||||
return frozenset(tokens)
|
||||
|
||||
|
||||
def normalize_argv(argv: list[str]) -> list[str]:
|
||||
"""Keep implicit ``run`` for a goal while preserving registered commands."""
|
||||
if len(argv) > 1 and argv[1] not in _root_tokens():
|
||||
return [argv[0], "run", *argv[1:]]
|
||||
return list(argv)
|
||||
|
||||
|
||||
def cli():
|
||||
# If invoked directly as python -m artemis.main without subcommand 'run',
|
||||
# check if first argument is a goal rather than a subcommand
|
||||
if len(sys.argv) > 1 and sys.argv[1] not in (
|
||||
"run",
|
||||
"init",
|
||||
"doctor",
|
||||
"batch",
|
||||
"bench",
|
||||
"mcp",
|
||||
"server",
|
||||
"trace",
|
||||
"--help",
|
||||
"-h",
|
||||
):
|
||||
sys.argv.insert(1, "run")
|
||||
sys.argv[:] = normalize_argv(sys.argv)
|
||||
app()
|
||||
|
||||
|
||||
|
||||
@@ -13,7 +13,14 @@
|
||||
"presets": {},
|
||||
"nodes": {
|
||||
"planner": {},
|
||||
"operator": {},
|
||||
"operator": {
|
||||
"fallback": {
|
||||
"provider": "openai",
|
||||
"model": "${env:OPENAI_TOOL_FALLBACK_MODEL:-gpt-6-luna}",
|
||||
"reasoning_effort": "${env:OPENAI_TOOL_FALLBACK_REASONING_EFFORT:-high}",
|
||||
"max_retries": "${env:OPENAI_TOOL_FALLBACK_MAX_RETRIES:-3}"
|
||||
}
|
||||
},
|
||||
"video_analyzer": {},
|
||||
"object_detector": {},
|
||||
"explorer": {},
|
||||
|
||||
@@ -66,7 +66,7 @@ def is_daemon_running(
|
||||
def is_artemis_daemon(
|
||||
host: str = DEFAULT_DAEMON_HOST,
|
||||
port: int = DEFAULT_DAEMON_PORT,
|
||||
timeout: float = 0.5,
|
||||
timeout: float = 2.0,
|
||||
) -> bool:
|
||||
"""Identity probe: is the process on ``host:port`` the Artemis daemon?
|
||||
|
||||
|
||||
@@ -615,9 +615,9 @@ async def render_timeline_clip(
|
||||
)
|
||||
labels.append(f"[{label}]")
|
||||
if len(labels) == 1:
|
||||
filter_parts.append(f"{labels[0]}null[outv]")
|
||||
filter_parts.append(f"{labels[0]}fps={fps}[outv]")
|
||||
else:
|
||||
filter_parts.append(f"{''.join(labels)}concat=n={len(labels)}:v=1:a=0[outv]")
|
||||
filter_parts.append(f"{''.join(labels)}concat=n={len(labels)}:v=1:a=0,fps={fps}[outv]")
|
||||
|
||||
command.extend(
|
||||
[
|
||||
|
||||
Reference in new issue
Block a user