289 lines
12 KiB
Python
289 lines
12 KiB
Python
"""Regression coverage for operator recovery and task pause handling."""
|
|
|
|
import asyncio
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, Mock
|
|
|
|
from langchain_core.messages import AIMessage, SystemMessage, ToolMessage
|
|
import pytest
|
|
|
|
from artemis.agents.operator.operator import OperatorNode, OperatorNoProgressError
|
|
from artemis.config.settings import settings
|
|
from artemis.runtime import task_control
|
|
|
|
|
|
@pytest.fixture
|
|
def operator(monkeypatch):
|
|
node = object.__new__(OperatorNode)
|
|
node.ctx = SimpleNamespace(transcript_ledger=None)
|
|
node._available_device_actions = lambda: {"click", "long_press", "input_text"}
|
|
node._plan_changed_this_turn = lambda: True
|
|
node._max_burst_actions = lambda: 4
|
|
state = SimpleNamespace(
|
|
operator_raw_data={"width": 1280, "height": 2856},
|
|
indexed_elements=[{"center": [310, 2610], "text": "model menu", "resource_id": "model"}]
|
|
* 13,
|
|
open_incident=None,
|
|
)
|
|
monkeypatch.setattr(settings, "ARTEMIS_OPERATOR_MAX_NO_PROGRESS_ATTEMPTS", 3)
|
|
return node, state
|
|
|
|
|
|
@pytest.mark.parametrize("name", ["click", "long_press", "input_text"])
|
|
@pytest.mark.parametrize("target", ["13", "[240, 914]", "240, 914"])
|
|
def test_quoted_targets_keep_same_action_and_grounding(operator, name, target):
|
|
node, state = operator
|
|
expected_target = 13 if target == "13" else [240, 914]
|
|
args = {"target": target, "target_description": "model menu", "text": "test"}
|
|
actions, error = node._translate_and_validate_tool({"name": name, "args": args}, state)
|
|
expected, _ = node._translate_and_validate_tool(
|
|
{"name": name, "args": {**args, "target": expected_target}}, state
|
|
)
|
|
assert error is None
|
|
assert actions == expected
|
|
if target == "13":
|
|
assert actions[0]["target_resource_id"] == "model"
|
|
else:
|
|
assert actions[0]["target_description"] == "model menu"
|
|
|
|
|
|
@pytest.mark.parametrize("target", ["0", "14", "model menu", True, "true"])
|
|
def test_invalid_index_is_not_guessed_or_dispatched(operator, target):
|
|
node, state = operator
|
|
actions, error = node._translate_and_validate_tool(
|
|
{"name": "click", "args": {"target": target}}, state
|
|
)
|
|
assert not actions
|
|
assert error
|
|
|
|
|
|
def test_quoted_coordinates_still_need_description(operator):
|
|
node, state = operator
|
|
actions, error = node._translate_and_validate_tool(
|
|
{"name": "click", "args": {"target": "[240, 914]"}}, state
|
|
)
|
|
assert not actions
|
|
assert "target_description" in error
|
|
|
|
|
|
def click(target="13"):
|
|
return AIMessage(
|
|
content="Open the model menu.",
|
|
tool_calls=[{"name": "click", "args": {"target": target}, "id": "call_click"}],
|
|
)
|
|
|
|
|
|
async def run_loop(operator, monkeypatch, responses):
|
|
node, state = operator
|
|
invoke = AsyncMock(side_effect=responses)
|
|
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
|
|
messages = []
|
|
result = await node._invoke_llm_loop(Mock(), messages, [], [], state)
|
|
return result, messages, invoke
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prose_retry_keeps_context_and_recovers_once(operator, monkeypatch):
|
|
prose = AIMessage(content="I will click the model menu.")
|
|
result, messages, invoke = await run_loop(operator, monkeypatch, [prose, click()])
|
|
assert invoke.await_count == 2
|
|
assert len(result[0]) == 1
|
|
assert result[0][0]["coordinates"] == [310, 2610]
|
|
assert prose in messages
|
|
assert any(isinstance(m, SystemMessage) and "No tool call" in m.content for m in messages)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"scenario", ["complete", "pending", "unfinished", "unchanged", "failed", "continuous", "pro"]
|
|
)
|
|
async def test_completed_exploration_plan_does_not_force_an_extra_action(
|
|
operator, monkeypatch, scenario
|
|
):
|
|
node, state = operator
|
|
node.ctx.exploration_id = None if scenario == "pro" else "exploration"
|
|
node.ctx.data_engine = None
|
|
plan = {"text": "- [ ] Return home"}
|
|
if scenario == "unchanged":
|
|
plan["text"] = "- [x] Return home"
|
|
node._read_task_plan_text = lambda: plan["text"]
|
|
monkeypatch.setattr("artemis.exploration.runtime.publish_response", Mock())
|
|
monkeypatch.setattr(
|
|
"artemis.exploration.runtime.needs_exploration_turn", lambda ctx: scenario == "pending"
|
|
)
|
|
|
|
async def save_note(**kwargs):
|
|
plan["text"] = "- [ ] Return home" if scenario == "unfinished" else "- [x] Return home"
|
|
if scenario == "continuous":
|
|
plan["text"] += " [Loop:continuous]"
|
|
return ToolMessage(
|
|
content="Saved plan",
|
|
tool_call_id="plan",
|
|
status="error" if scenario == "failed" else "success",
|
|
)
|
|
|
|
monkeypatch.setattr("artemis.agents.operator.operator.invoke_tool_with_injection", save_note)
|
|
response = AIMessage(
|
|
content="Confirmed home; recorded evidence.",
|
|
tool_calls=[{"name": "save_note", "args": {}, "id": "plan"}],
|
|
)
|
|
invoke = AsyncMock(side_effect=[response, click()])
|
|
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
|
|
messages = []
|
|
result = await node._invoke_llm_loop(
|
|
Mock(), messages, [SimpleNamespace(name="save_note")], [], state
|
|
)
|
|
if scenario == "complete":
|
|
assert invoke.await_count == 1
|
|
assert result[0] is None
|
|
assert result[1] == "Confirmed home; recorded evidence."
|
|
assert result[3] is False
|
|
else:
|
|
assert invoke.await_count == 2
|
|
assert result[0]
|
|
assert any(isinstance(m, ToolMessage) and m.tool_call_id == "plan" for m in messages)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_invalid_target_can_be_corrected_before_limit(operator, monkeypatch):
|
|
result, messages, invoke = await run_loop(operator, monkeypatch, [click("14"), click("13")])
|
|
assert invoke.await_count == 2
|
|
assert len(result[0]) == 1
|
|
assert result[0][0]["coordinates"] == [310, 2610]
|
|
outputs = [m for m in messages if isinstance(m, ToolMessage)]
|
|
assert [m.status for m in outputs] == ["error", "success"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("limit", [1, 3])
|
|
async def test_empty_responses_fail_at_configured_limit(operator, monkeypatch, limit):
|
|
monkeypatch.setattr(settings, "ARTEMIS_OPERATOR_MAX_NO_PROGRESS_ATTEMPTS", limit)
|
|
monkeypatch.setattr(settings, "ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS", 3)
|
|
fallback = Mock()
|
|
get_fallback = Mock(return_value=fallback)
|
|
monkeypatch.setattr("artemis.agents.operator.operator.get_llm", get_fallback)
|
|
invoke = AsyncMock(return_value=AIMessage(content="I will click now."))
|
|
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
|
|
node, state = operator
|
|
with pytest.raises(OperatorNoProgressError, match="未提供可执行的工具调用"):
|
|
await node._invoke_llm_loop(Mock(), [], [], [], state)
|
|
assert invoke.await_count == limit + 3
|
|
get_fallback.assert_called_once_with(ctx=node.ctx, name="operator", use_fallback=True)
|
|
assert invoke.call_args_list[-1].args[0] is fallback.bind_tools.return_value
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_operator_fallback_is_scoped_to_current_step(operator, monkeypatch):
|
|
node, state = operator
|
|
primary = Mock()
|
|
fallback = Mock()
|
|
get_fallback = Mock(return_value=fallback)
|
|
monkeypatch.setattr("artemis.agents.operator.operator.get_llm", get_fallback)
|
|
invoke = AsyncMock(side_effect=[AIMessage(content="no tool")] * 3 + [click(), click()])
|
|
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
|
|
|
|
first = await node._invoke_llm_loop(primary, [], [], [], state)
|
|
second = await node._invoke_llm_loop(primary, [], [], [], state)
|
|
|
|
assert first[0] and second[0]
|
|
assert get_fallback.call_count == 1
|
|
assert [call.args[0] for call in invoke.call_args_list] == [
|
|
primary.bind_tools.return_value,
|
|
primary.bind_tools.return_value,
|
|
primary.bind_tools.return_value,
|
|
fallback.bind_tools.return_value,
|
|
primary.bind_tools.return_value,
|
|
]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_invalid_calls_and_prose_share_bounded_recovery(operator, monkeypatch):
|
|
invoke = AsyncMock(
|
|
side_effect=[click("missing"), AIMessage(content="Retry."), click("14")] + [click("14")] * 3
|
|
)
|
|
monkeypatch.setattr("artemis.agents.operator.operator.get_llm", Mock(return_value=Mock()))
|
|
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
|
|
node, state = operator
|
|
messages = []
|
|
with pytest.raises(OperatorNoProgressError, match="校验仍失败"):
|
|
await node._invoke_llm_loop(Mock(), messages, [], [], state)
|
|
assert invoke.await_count == 6
|
|
assert any(isinstance(m, ToolMessage) and m.status == "error" for m in messages)
|
|
assert not any(isinstance(m, ToolMessage) and m.content == "Action Recorded" for m in messages)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_iteration_cap_cannot_restart_no_progress_next_turn(operator, monkeypatch):
|
|
monkeypatch.setattr("artemis.agents.operator.operator.OPERATOR_MAX_TOOL_ITERATIONS", 2)
|
|
invoke = AsyncMock(return_value=AIMessage(content="I will click now."))
|
|
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
|
|
node, state = operator
|
|
with pytest.raises(OperatorNoProgressError, match="预算内未恢复"):
|
|
await node._invoke_llm_loop(Mock(), [], [], [], state)
|
|
assert invoke.await_count == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pause_during_diagnosis_returns_to_fresh_observation(operator, monkeypatch, tmp_path):
|
|
node, state = operator
|
|
monkeypatch.setattr(task_control, "TRACES_PATH", tmp_path)
|
|
started = asyncio.Event()
|
|
cancelled = asyncio.Event()
|
|
events = []
|
|
|
|
class Lock:
|
|
def __init__(self, device_id, **kwargs):
|
|
self.device_id = device_id
|
|
self.description = "test"
|
|
self.max_concurrency = None
|
|
self.concurrency_mode = "per_device"
|
|
self.ingress = "web"
|
|
self.lock_scope = "test"
|
|
|
|
def release(self):
|
|
events.append("released")
|
|
|
|
def acquire(self, cancel_event=None):
|
|
events.append("acquired")
|
|
|
|
monkeypatch.setattr(task_control, "DeviceExecutionLock", Lock)
|
|
node.ctx.data_engine = None
|
|
node.ctx.task_control = task_control.TaskControl("pro-task", Lock("device"))
|
|
node.ctx.on_task_pause = AsyncMock()
|
|
node.ctx.on_task_resume = AsyncMock()
|
|
|
|
async def diagnose(**kwargs):
|
|
started.set()
|
|
try:
|
|
await asyncio.Event().wait()
|
|
finally:
|
|
cancelled.set()
|
|
|
|
monkeypatch.setattr("artemis.agents.operator.operator.invoke_tool_with_injection", diagnose)
|
|
response = AIMessage(
|
|
content="Inspect the failure",
|
|
tool_calls=[
|
|
{"name": "ask_diagnoser", "args": {"query": "why"}, "id": "diagnosis"},
|
|
{"name": "click", "args": {"target": 13}, "id": "tap"},
|
|
],
|
|
)
|
|
invoke = AsyncMock(return_value=response)
|
|
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
|
|
tool = SimpleNamespace(name="ask_diagnoser")
|
|
messages = []
|
|
execution = asyncio.create_task(node._invoke_llm_loop(Mock(), messages, [tool], [], state))
|
|
await asyncio.wait_for(started.wait(), 1)
|
|
task_control.request_pause("pro-task")
|
|
for _ in range(40):
|
|
if task_control.read_state("pro-task").get("status") == "task_paused":
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert task_control.read_state("pro-task")["status"] == "task_paused"
|
|
assert cancelled.is_set()
|
|
assert events == ["released"]
|
|
task_control.request_resume("pro-task")
|
|
assert (await asyncio.wait_for(execution, 2))[0] is None
|
|
assert events == ["released", "acquired"]
|
|
assert invoke.await_count == 1
|
|
assert [m.status for m in messages if isinstance(m, ToolMessage)] == ["error", "error"]
|