feat: add tool-call fallback and resilient batch exports
This commit is contained in:
1 parent
ab066733b8
commit
d31eb2a19d
32 files changed
+538
-72
No files matched your search
@@ -384,7 +384,7 @@ def _make_runner(mock_context, responses, max_turns=None):
|
||||
pending = list(responses)
|
||||
|
||||
async def _invoke(llm, tools, messages):
|
||||
runner.calls.append({"tools": list(tools), "messages": list(messages)})
|
||||
runner.calls.append({"llm": llm, "tools": list(tools), "messages": list(messages)})
|
||||
return pending.pop(0)
|
||||
|
||||
runner._invoke_model = AsyncMock(side_effect=_invoke)
|
||||
@@ -647,9 +647,11 @@ async def test_flash_stops_repeated_no_tool_responses(mock_context, monkeypatch,
|
||||
from artemis.config.settings import settings
|
||||
|
||||
monkeypatch.setattr(settings, "ARTEMIS_FLASH_MAX_NO_PROGRESS_ATTEMPTS", attempts)
|
||||
responses = [AIMessage(content="I will click now.", tool_calls=[]) for _ in range(attempts)]
|
||||
monkeypatch.setattr(settings, "ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS", 3)
|
||||
responses = [AIMessage(content="I will click now.", tool_calls=[]) for _ in range(attempts + 3)]
|
||||
with (
|
||||
patch("artemis.controllers.unified_controller.get_driver"),
|
||||
patch("artemis.agents.flash.runner.get_llm", return_value=Mock()) as get_fallback,
|
||||
patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(return_value=_INITIAL_OBSERVATION),
|
||||
@@ -659,11 +661,55 @@ async def test_flash_stops_repeated_no_tool_responses(mock_context, monkeypatch,
|
||||
result = await runner.run(State(initial_goal="Open Settings"))
|
||||
|
||||
assert result["status"] == "failed"
|
||||
assert f"连续 {attempts} 次未调用工具" in result["explanation"]
|
||||
assert runner._invoke_model.await_count == attempts
|
||||
assert f"连续 {attempts} 次" in result["explanation"]
|
||||
assert "兜底模型连续 3 次" in result["explanation"]
|
||||
assert runner._invoke_model.await_count == attempts + 3
|
||||
get_fallback.assert_called_once_with(mock_context, name="operator", use_fallback=True)
|
||||
runner.executor.execute.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flash_uses_fallback_for_one_tool_turn_then_primary(mock_context):
|
||||
no_call = AIMessage(content="I will click now.")
|
||||
observe = AIMessage(
|
||||
content="Check the target.",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "ask_explorer",
|
||||
"args": {"question": "Find Settings"},
|
||||
"id": "fallback-observe",
|
||||
}
|
||||
],
|
||||
)
|
||||
click = AIMessage(
|
||||
content="Tap Settings.",
|
||||
tool_calls=[{"name": "click", "args": {"target": 1}, "id": "fallback-click"}],
|
||||
)
|
||||
fallback = Mock()
|
||||
responses = [no_call, no_call, no_call, observe, click, _report()]
|
||||
with (
|
||||
patch("artemis.controllers.unified_controller.get_driver"),
|
||||
patch("artemis.agents.flash.runner.get_llm", return_value=fallback),
|
||||
patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(return_value=_INITIAL_OBSERVATION),
|
||||
),
|
||||
):
|
||||
runner = _make_runner(mock_context, responses, max_turns=0)
|
||||
result = await runner.run(State(initial_goal="Open Settings"))
|
||||
|
||||
primary = runner.calls[0]["llm"]
|
||||
assert [call["llm"] for call in runner.calls] == [
|
||||
primary,
|
||||
primary,
|
||||
primary,
|
||||
fallback,
|
||||
fallback,
|
||||
primary,
|
||||
]
|
||||
assert result["status"] == "completed"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _TurnRecord.result: the turn's execution result in the validator-report shape
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -107,23 +107,56 @@ async def test_invalid_target_can_be_corrected_before_limit(operator, monkeypatc
|
||||
@pytest.mark.parametrize("limit", [1, 3])
|
||||
async def test_empty_responses_fail_at_configured_limit(operator, monkeypatch, limit):
|
||||
monkeypatch.setattr(settings, "ARTEMIS_OPERATOR_MAX_NO_PROGRESS_ATTEMPTS", limit)
|
||||
monkeypatch.setattr(settings, "ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS", 3)
|
||||
fallback = Mock()
|
||||
get_fallback = Mock(return_value=fallback)
|
||||
monkeypatch.setattr("artemis.agents.operator.operator.get_llm", get_fallback)
|
||||
invoke = AsyncMock(return_value=AIMessage(content="I will click now."))
|
||||
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
|
||||
node, state = operator
|
||||
with pytest.raises(OperatorNoProgressError, match="未提供可执行的工具调用"):
|
||||
await node._invoke_llm_loop(Mock(), [], [], [], state)
|
||||
assert invoke.await_count == limit
|
||||
assert invoke.await_count == limit + 3
|
||||
get_fallback.assert_called_once_with(ctx=node.ctx, name="operator", use_fallback=True)
|
||||
assert invoke.call_args_list[-1].args[0] is fallback.bind_tools.return_value
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_fallback_is_scoped_to_current_step(operator, monkeypatch):
|
||||
node, state = operator
|
||||
primary = Mock()
|
||||
fallback = Mock()
|
||||
get_fallback = Mock(return_value=fallback)
|
||||
monkeypatch.setattr("artemis.agents.operator.operator.get_llm", get_fallback)
|
||||
invoke = AsyncMock(side_effect=[AIMessage(content="no tool")] * 3 + [click(), click()])
|
||||
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
|
||||
|
||||
first = await node._invoke_llm_loop(primary, [], [], [], state)
|
||||
second = await node._invoke_llm_loop(primary, [], [], [], state)
|
||||
|
||||
assert first[0] and second[0]
|
||||
assert get_fallback.call_count == 1
|
||||
assert [call.args[0] for call in invoke.call_args_list] == [
|
||||
primary.bind_tools.return_value,
|
||||
primary.bind_tools.return_value,
|
||||
primary.bind_tools.return_value,
|
||||
fallback.bind_tools.return_value,
|
||||
primary.bind_tools.return_value,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_calls_and_prose_share_bounded_recovery(operator, monkeypatch):
|
||||
invoke = AsyncMock(side_effect=[click("missing"), AIMessage(content="Retry."), click("14")])
|
||||
invoke = AsyncMock(
|
||||
side_effect=[click("missing"), AIMessage(content="Retry."), click("14")] + [click("14")] * 3
|
||||
)
|
||||
monkeypatch.setattr("artemis.agents.operator.operator.get_llm", Mock(return_value=Mock()))
|
||||
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
|
||||
node, state = operator
|
||||
messages = []
|
||||
with pytest.raises(OperatorNoProgressError, match="校验仍失败"):
|
||||
await node._invoke_llm_loop(Mock(), messages, [], [], state)
|
||||
assert invoke.await_count == 3
|
||||
assert invoke.await_count == 6
|
||||
assert any(isinstance(m, ToolMessage) and m.status == "error" for m in messages)
|
||||
assert not any(isinstance(m, ToolMessage) and m.content == "Action Recorded" for m in messages)
|
||||
|
||||
|
||||
Reference in new issue
Block a user