fix(runtime): recover awake ownership and batch model failures
This commit is contained in:
1 parent
6b2c0f199d
commit
baf069854b
17 files changed
+477
-32
No files matched your search
@@ -429,6 +429,44 @@ def _report(status="completed", tc_id="tc-report", **extra):
|
||||
_INITIAL_OBSERVATION = ("shot0.png", b"IMG0", "[1] Settings")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generation_failure_uses_fallback_without_repeating_device_actions(mock_context):
|
||||
from artemis.llm.response_guard import ResponseIntegrityError
|
||||
from artemis.services import llm as gateway
|
||||
|
||||
click = AIMessage(
|
||||
content="tap", tool_calls=[{"name": "click", "args": {"index": 1}, "id": "click"}]
|
||||
)
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = _make_runner(mock_context, [])
|
||||
primary = runner._init_llm.return_value
|
||||
fallback = Mock()
|
||||
calls = []
|
||||
|
||||
async def invoke(model, tools, messages):
|
||||
calls.append(model)
|
||||
if len(calls) == 1:
|
||||
assert gateway._FALLBACK_AVAILABLE.get()
|
||||
raise ResponseIntegrityError("missing completed response")
|
||||
if model is fallback:
|
||||
assert not gateway._FALLBACK_AVAILABLE.get()
|
||||
return click
|
||||
return _report()
|
||||
|
||||
runner._invoke_model = AsyncMock(side_effect=invoke)
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.flash.runner.capture_screenshot_and_parse_ui",
|
||||
AsyncMock(return_value=_INITIAL_OBSERVATION),
|
||||
),
|
||||
patch("artemis.agents.flash.runner.get_llm", return_value=fallback),
|
||||
):
|
||||
result = await runner.run(State(initial_goal="Open Settings"))
|
||||
assert result["status"] == "completed"
|
||||
assert calls == [primary, fallback, primary]
|
||||
assert runner.executor.execute.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("fallback_succeeds", [True, False])
|
||||
async def test_failed_actions_have_finite_fallback_and_restore_primary(
|
||||
|
||||
@@ -82,6 +82,31 @@ async def run_loop(operator, monkeypatch, responses):
|
||||
return result, messages, invoke
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generation_failure_hands_over_before_pausing(operator, monkeypatch):
|
||||
from artemis.llm.response_guard import ResponseIntegrityError
|
||||
from artemis.services import llm as gateway
|
||||
|
||||
node, state = operator
|
||||
primary, fallback = Mock(), Mock()
|
||||
contexts = []
|
||||
|
||||
async def invoke(model, messages):
|
||||
contexts.append(gateway._FALLBACK_AVAILABLE.get())
|
||||
if model is primary.bind_tools.return_value:
|
||||
raise ResponseIntegrityError("missing completed response")
|
||||
assert model is fallback.bind_tools.return_value
|
||||
return click()
|
||||
|
||||
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
|
||||
monkeypatch.setattr("artemis.agents.operator.operator.get_llm", Mock(return_value=fallback))
|
||||
messages = []
|
||||
result = await node._invoke_llm_loop(primary, messages, [], [], state)
|
||||
assert contexts == [True, False]
|
||||
assert len(result[0]) == 1
|
||||
assert len([m for m in messages if isinstance(m, AIMessage)]) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prose_retry_keeps_context_and_recovers_once(operator, monkeypatch):
|
||||
prose = AIMessage(content="I will click the model menu.")
|
||||
|
||||
Reference in new issue
Block a user