feat: improve operator progress and recording accuracy
This commit is contained in:
1 parent
58210106d7
commit
4bc79fdec8
15 files changed
+773
-224
No files matched your search
@@ -1188,140 +1188,6 @@ async def test_operator_background_tasks_prompt_injection():
|
||||
adb_registry.background.pop("task_99999", None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_task_plan_warning_prompt_injection():
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.context import ArtemisContext
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.execution_setup = None
|
||||
mock_ctx.data_engine = None
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.initial_goal = "Test goal"
|
||||
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=")
|
||||
mock_device_data = MagicMock()
|
||||
mock_device_data.elements = []
|
||||
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.tool_calls = []
|
||||
|
||||
mock_llm.ainvoke = AsyncMock(return_value=mock_response)
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
# Case 1: task_plan updated in the last step (using the "key" argument)
|
||||
steps_updated = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"tool_calls": [{"name": "click", "args": {"target": 1}}],
|
||||
},
|
||||
{
|
||||
"step_id": "step_2",
|
||||
"step_number": 2,
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "update_note",
|
||||
"args": {"key": "task_plan", "target": "a", "replacement": "b"},
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
captured_messages_1 = []
|
||||
|
||||
async def mock_invoke_llm_loop_1(*args, **kwargs):
|
||||
messages = await node._build_prompt(
|
||||
state=mock_state,
|
||||
latest_screenshot_b64="YmFzZTY0X3NjcmVlbnNob3Q=",
|
||||
fused_xml=[],
|
||||
minimal_list="",
|
||||
current_step_num=3,
|
||||
steps=steps_updated,
|
||||
task_plan="",
|
||||
)
|
||||
captured_messages_1.extend(messages)
|
||||
return [], None, None, False
|
||||
|
||||
with (
|
||||
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
|
||||
patch(
|
||||
"artemis.agents.operator.operator.OperatorNode._invoke_llm_loop",
|
||||
side_effect=mock_invoke_llm_loop_1,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
full_content_1 = ""
|
||||
for m in captured_messages_1:
|
||||
if isinstance(m.content, list):
|
||||
for part in m.content:
|
||||
if isinstance(part, dict) and "text" in part:
|
||||
full_content_1 += part["text"] + "\n"
|
||||
elif isinstance(m.content, str):
|
||||
full_content_1 += m.content + "\n"
|
||||
|
||||
assert "Reminder: your last turn executed a Turn-Ending Action" not in full_content_1
|
||||
|
||||
# Case 2: an action turn that did NOT update task_plan
|
||||
steps_not_updated = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"tool_calls": [{"name": "click", "args": {"target": 1}}],
|
||||
},
|
||||
{
|
||||
"step_id": "step_2",
|
||||
"step_number": 2,
|
||||
"tool_calls": [{"name": "click", "args": {"target": 2}}],
|
||||
"action_taken": {"action": "click", "target": 2},
|
||||
},
|
||||
]
|
||||
|
||||
captured_messages_2 = []
|
||||
|
||||
async def mock_invoke_llm_loop_2(*args, **kwargs):
|
||||
messages = await node._build_prompt(
|
||||
state=mock_state,
|
||||
latest_screenshot_b64="YmFzZTY0X3NjcmVlbnNob3Q=",
|
||||
fused_xml=[],
|
||||
minimal_list="",
|
||||
current_step_num=3,
|
||||
steps=steps_not_updated,
|
||||
task_plan="",
|
||||
)
|
||||
captured_messages_2.extend(messages)
|
||||
return [], None, None, False
|
||||
|
||||
with (
|
||||
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
|
||||
patch(
|
||||
"artemis.agents.operator.operator.OperatorNode._invoke_llm_loop",
|
||||
side_effect=mock_invoke_llm_loop_2,
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
full_content_2 = ""
|
||||
for m in captured_messages_2:
|
||||
if isinstance(m.content, list):
|
||||
for part in m.content:
|
||||
if isinstance(part, dict) and "text" in part:
|
||||
full_content_2 += part["text"] + "\n"
|
||||
elif isinstance(m.content, str):
|
||||
full_content_2 += m.content + "\n"
|
||||
|
||||
assert "Reminder: your last turn executed a Turn-Ending Action" in full_content_2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_tool_limit_exceeded_warning():
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
|
||||
Reference in new issue
Block a user