feat: add tool-call fallback and resilient batch exports

This commit is contained in:
chenmingxuan committed 2026-09-29 19:53:20 +08:00
1 parent ab066733b8
commit d31eb2a19d
32 files changed
+538 -72

No files matched your search

+46 -11
View File
@@ -1071,6 +1071,8 @@ class FlashRunner:
pending_notices: list[str] = []
no_tool_attempts = 0
no_tool_limit = settings.ARTEMIS_FLASH_MAX_NO_PROGRESS_ATTEMPTS
fallback_llm = None
using_tool_fallback = False
while limit is None or turns < limit:
control = getattr(self.ctx, "task_control", None)
@@ -1080,6 +1082,9 @@ class FlashRunner:
on_resume=getattr(self.ctx, "on_task_resume", None),
)
if resumed:
using_tool_fallback = False
no_tool_attempts = 0
self._llm = llm
(
shot_path,
current_pre_screenshot_bytes,
@@ -1131,14 +1136,18 @@ class FlashRunner:
if isinstance(control, TaskControl):
interrupted, response = await control.run_interruptibly(
self._invoke_model(llm, current_tools, messages),
self._invoke_model(
fallback_llm if using_tool_fallback else llm, current_tools, messages
),
on_pause=getattr(self.ctx, "on_task_pause", None),
on_resume=getattr(self.ctx, "on_task_resume", None),
)
else:
interrupted, response = (
False,
await self._invoke_model(llm, current_tools, messages),
await self._invoke_model(
fallback_llm if using_tool_fallback else llm, current_tools, messages
),
)
if interrupted or (
@@ -1146,6 +1155,9 @@ class FlashRunner:
):
# Do not dispatch a stale action or final answer from a prompt
# assembled before the instruction arrived.
using_tool_fallback = False
no_tool_attempts = 0
self._llm = llm
(
shot_path,
current_pre_screenshot_bytes,
@@ -1186,6 +1198,9 @@ class FlashRunner:
tool_calls = self._resolve_tool_calls(response, raw_text)
if isinstance(control, TaskControl) and has_pending_instruction(control.directory):
using_tool_fallback = False
no_tool_attempts = 0
self._llm = llm
(
shot_path,
current_pre_screenshot_bytes,
@@ -1207,15 +1222,27 @@ class FlashRunner:
final_report = {"status": "failed", "explanation": raw_text}
break
no_tool_attempts += 1
if no_tool_attempts >= no_tool_limit:
final_report = {
"status": "failed",
"explanation": (
f"模型连续 {no_tool_attempts} 次未调用工具,"
"已停止任务以避免空转和重复计费;未执行新的设备动作。"
),
}
break
active_limit = (
settings.ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS
if using_tool_fallback
else no_tool_limit
)
if no_tool_attempts >= active_limit:
if using_tool_fallback:
final_report = {
"status": "failed",
"explanation": (
f"首选模型连续 {no_tool_limit} 次、兜底模型连续 "
f"{no_tool_attempts} 次未调用工具,已停止任务;"
"未执行新的设备动作。"
),
}
break
fallback_llm = get_llm(self.ctx, name="operator", use_fallback=True)
using_tool_fallback = True
no_tool_attempts = 0
logger.warning("Flash 首选模型未调用工具,当前步骤切换到兜底模型")
pending_notices.append("首选模型连续未调用工具;本步骤改用兜底模型。")
pending_notices.append(_NO_TOOL_CALL_NOTICE)
# Keep the observation and any injected user instruction, but
# do not feed the model its own unproductive prose on retry.
@@ -1224,6 +1251,10 @@ class FlashRunner:
continue
no_tool_attempts = 0
if using_tool_fallback:
# A tool-producing turn completes the fallback's scope. The next
# observation goes back to the configured primary model.
self._llm = fallback_llm
# Process tool calls
turn = _TurnRecord()
(
@@ -1244,6 +1275,10 @@ class FlashRunner:
injected=injected,
native_text=native_text,
)
if using_tool_fallback and any(status == "success" for _, status, _ in turn.actions):
using_tool_fallback = False
self._llm = llm
pending_notices.append("兜底模型已完成本步骤;后续步骤恢复首选模型。")
ledger.stage_turn(messages[turn_base:])
previous_turn = turn
if final_report_from_calls is not None:
+40 -9
View File
@@ -508,6 +508,9 @@ class OperatorNode:
plan_gate_bounced = False
no_progress_attempts = 0
no_progress_limit = settings.ARTEMIS_OPERATOR_MAX_NO_PROGRESS_ATTEMPTS
fallback_limit = settings.ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS
active_llm = base_llm
using_tool_fallback = False
for iteration in range(max_iterations):
control = getattr(self.ctx, "task_control", None)
@@ -532,7 +535,7 @@ class OperatorNode:
)
)
bound_llm = base_llm.bind_tools(tools=traced_tools)
bound_llm = active_llm.bind_tools(tools=traced_tools)
interrupted, response = await self._await_or_pause(
invoke_llm_with_timeout_message(acomplete(bound_llm, current_messages))
)
@@ -607,11 +610,25 @@ class OperatorNode:
if not response.tool_calls:
no_progress_attempts += 1
if no_progress_attempts >= no_progress_limit:
raise OperatorNoProgressError(
f"模型连续 {no_progress_attempts} 次未提供可执行的工具调用,"
"已停止任务以避免空转;未执行新的设备动作。"
if no_progress_attempts >= (
fallback_limit if using_tool_fallback else no_progress_limit
):
if using_tool_fallback:
raise OperatorNoProgressError(
f"首选模型连续 {no_progress_limit} 次、兜底模型连续 "
f"{no_progress_attempts} 次未提供可执行的工具调用;"
"已停止任务以避免空转。"
)
active_llm = get_llm(ctx=self.ctx, name="operator", use_fallback=True)
using_tool_fallback = True
no_progress_attempts = 0
logger.warning("Operator 首选模型未调用工具,当前步骤切换到兜底模型")
current_messages.append(
SystemMessage(
content="本步骤已切换兜底模型。请使用工具完成操作或明确报告阻碍。"
)
)
continue
logger.warning("LLM returned no tool call; requesting a concrete tool call.")
current_messages.append(response)
current_messages.append(
@@ -931,11 +948,25 @@ class OperatorNode:
if validation_errors:
no_progress_attempts += 1
if no_progress_attempts >= no_progress_limit:
raise OperatorNoProgressError(
f"模型连续 {no_progress_attempts} 次未提供可执行的工具调用,"
"工具参数校验仍失败;已停止任务,未执行被拒绝的设备动作。"
if no_progress_attempts >= (
fallback_limit if using_tool_fallback else no_progress_limit
):
if using_tool_fallback:
raise OperatorNoProgressError(
f"首选模型连续 {no_progress_limit} 次、兜底模型连续 "
f"{no_progress_attempts} 次未提供可执行的工具调用,"
"工具参数校验仍失败;已停止任务。"
)
active_llm = get_llm(ctx=self.ctx, name="operator", use_fallback=True)
using_tool_fallback = True
no_progress_attempts = 0
logger.warning("Operator 首选模型工具调用校验失败,当前步骤切换到兜底模型")
current_messages.append(
SystemMessage(
content="本步骤已切换兜底模型。请修正上面的工具参数后再调用。"
)
)
continue
logger.warning("Some action calls failed validation. Feeding back to LLM.")
continue
no_progress_attempts = 0
+5
View File
@@ -100,6 +100,11 @@ class Settings(BaseSettings):
OPENAI_TIMEOUT_SECONDS: float = Field(default=300.0, gt=0)
OPENAI_MAX_RETRIES: int = Field(default=5, ge=0, le=10)
OPENAI_RETRY_INTERVAL_SECONDS: float = Field(default=10.0, ge=0, le=300)
OPENAI_RATE_LIMIT_RETRY_SECONDS: float = Field(default=60.0, ge=0, le=3600)
OPENAI_TOOL_FALLBACK_MODEL: str = Field(default="gpt-6-luna", min_length=1)
OPENAI_TOOL_FALLBACK_REASONING_EFFORT: str = "high"
OPENAI_TOOL_FALLBACK_MAX_RETRIES: int = Field(default=3, ge=0, le=10)
ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS: int = Field(default=3, ge=1, le=10)
OPENAI_REASONING_EFFORT: str = "high"
ARTEMIS_CREDENTIAL_TIMEOUT_SECONDS: float = Field(default=12.0, gt=0)
ARTEMIS_MODEL_CACHE_TTL_SECONDS: float = Field(default=10.0, ge=0)
+5
View File
@@ -117,6 +117,11 @@ def init_command() -> None:
"OPENAI_TIMEOUT_SECONDS=300",
"OPENAI_MAX_RETRIES=5",
"OPENAI_RETRY_INTERVAL_SECONDS=10",
"OPENAI_RATE_LIMIT_RETRY_SECONDS=60",
"OPENAI_TOOL_FALLBACK_MODEL=gpt-6-luna",
"OPENAI_TOOL_FALLBACK_REASONING_EFFORT=high",
"OPENAI_TOOL_FALLBACK_MAX_RETRIES=3",
"ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS=3",
"ARTEMIS_VIDEO_ENABLED=false",
"ARTEMIS_TRACES_DIR=./traces",
"",
+5 -1
View File
@@ -75,7 +75,7 @@ def extract_status_code(error: BaseException) -> int | None:
return None
_RATE_LIMIT_MARKERS = ("rate limit", "quota", "resource_exhausted")
_RATE_LIMIT_MARKERS = ("rate limit", "too many requests", "quota", "resource_exhausted")
_UNAVAILABLE_MARKERS = ("unavailable", "overloaded", "high demand", "service unavailable")
_AUTH_MARKERS = ("unauthorized", "forbidden", "api key", "credential")
# "deadline exceeded"/"deadline_exceeded" cover google-genai SDK timeouts that
@@ -196,6 +196,10 @@ def retry_policy_for(category: FailureCategory | str) -> RetryPolicy:
return _DEFAULT_RETRY_POLICIES[category]
from artemis.config.settings import settings
if category is FailureCategory.RATE_LIMIT:
return _configured_policy(
settings.OPENAI_MAX_RETRIES, settings.OPENAI_RATE_LIMIT_RETRY_SECONDS
)
return _configured_policy(settings.OPENAI_MAX_RETRIES, settings.OPENAI_RETRY_INTERVAL_SECONDS)
+24 -13
View File
@@ -15,25 +15,36 @@
"""Backward compatibility entrypoint for artemis CLI."""
import sys
import typer
from artemis.interfaces.cli.main import app
from artemis.interfaces.cli.main import app as _typer_app
def _root_tokens() -> frozenset[str]:
"""Read command names and root flags from the registered CLI."""
group = typer.main.get_command(_typer_app)
tokens: set[str] = {"-h"}
tokens.update(getattr(group, "commands", {}) or {})
tokens.update(group.get_help_option_names(group.context_class(group)))
for param in group.params:
tokens.update(getattr(param, "opts", ()))
tokens.update(getattr(param, "secondary_opts", ()))
return frozenset(tokens)
def normalize_argv(argv: list[str]) -> list[str]:
"""Keep implicit ``run`` for a goal while preserving registered commands."""
if len(argv) > 1 and argv[1] not in _root_tokens():
return [argv[0], "run", *argv[1:]]
return list(argv)
def cli():
# If invoked directly as python -m artemis.main without subcommand 'run',
# check if first argument is a goal rather than a subcommand
if len(sys.argv) > 1 and sys.argv[1] not in (
"run",
"init",
"doctor",
"batch",
"bench",
"mcp",
"server",
"trace",
"--help",
"-h",
):
sys.argv.insert(1, "run")
sys.argv[:] = normalize_argv(sys.argv)
app()
+8 -1
View File
@@ -13,7 +13,14 @@
"presets": {},
"nodes": {
"planner": {},
"operator": {},
"operator": {
"fallback": {
"provider": "openai",
"model": "${env:OPENAI_TOOL_FALLBACK_MODEL:-gpt-6-luna}",
"reasoning_effort": "${env:OPENAI_TOOL_FALLBACK_REASONING_EFFORT:-high}",
"max_retries": "${env:OPENAI_TOOL_FALLBACK_MAX_RETRIES:-3}"
}
},
"video_analyzer": {},
"object_detector": {},
"explorer": {},
+1 -1
View File
@@ -66,7 +66,7 @@ def is_daemon_running(
def is_artemis_daemon(
host: str = DEFAULT_DAEMON_HOST,
port: int = DEFAULT_DAEMON_PORT,
timeout: float = 0.5,
timeout: float = 2.0,
) -> bool:
"""Identity probe: is the process on ``host:port`` the Artemis daemon?
+2 -2
View File
@@ -615,9 +615,9 @@ async def render_timeline_clip(
)
labels.append(f"[{label}]")
if len(labels) == 1:
filter_parts.append(f"{labels[0]}null[outv]")
filter_parts.append(f"{labels[0]}fps={fps}[outv]")
else:
filter_parts.append(f"{''.join(labels)}concat=n={len(labels)}:v=1:a=0[outv]")
filter_parts.append(f"{''.join(labels)}concat=n={len(labels)}:v=1:a=0,fps={fps}[outv]")
command.extend(
[