feat: improve operator progress and recording accuracy

This commit is contained in:
somew1nd committed 2026-09-03 10:10:22 -07:00
1 parent 58210106d7
commit 4bc79fdec8
15 files changed
+773 -224

No files matched your search

-134
View File
@@ -1188,140 +1188,6 @@ async def test_operator_background_tasks_prompt_injection():
adb_registry.background.pop("task_99999", None)
@pytest.mark.asyncio
async def test_operator_task_plan_warning_prompt_injection():
from artemis.agents.operator.operator import OperatorNode
from artemis.context import ArtemisContext
from unittest.mock import AsyncMock, MagicMock, patch
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.execution_setup = None
mock_ctx.data_engine = None
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.initial_goal = "Test goal"
mock_controller = MagicMock()
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=")
mock_device_data = MagicMock()
mock_device_data.elements = []
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.tool_calls = []
mock_llm.ainvoke = AsyncMock(return_value=mock_response)
mock_llm.bind_tools.return_value = mock_llm
# Case 1: task_plan updated in the last step (using the "key" argument)
steps_updated = [
{
"step_id": "step_1",
"step_number": 1,
"tool_calls": [{"name": "click", "args": {"target": 1}}],
},
{
"step_id": "step_2",
"step_number": 2,
"tool_calls": [
{
"name": "update_note",
"args": {"key": "task_plan", "target": "a", "replacement": "b"},
}
],
},
]
captured_messages_1 = []
async def mock_invoke_llm_loop_1(*args, **kwargs):
messages = await node._build_prompt(
state=mock_state,
latest_screenshot_b64="YmFzZTY0X3NjcmVlbnNob3Q=",
fused_xml=[],
minimal_list="",
current_step_num=3,
steps=steps_updated,
task_plan="",
)
captured_messages_1.extend(messages)
return [], None, None, False
with (
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
patch(
"artemis.agents.operator.operator.OperatorNode._invoke_llm_loop",
side_effect=mock_invoke_llm_loop_1,
),
):
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
full_content_1 = ""
for m in captured_messages_1:
if isinstance(m.content, list):
for part in m.content:
if isinstance(part, dict) and "text" in part:
full_content_1 += part["text"] + "\n"
elif isinstance(m.content, str):
full_content_1 += m.content + "\n"
assert "Reminder: your last turn executed a Turn-Ending Action" not in full_content_1
# Case 2: an action turn that did NOT update task_plan
steps_not_updated = [
{
"step_id": "step_1",
"step_number": 1,
"tool_calls": [{"name": "click", "args": {"target": 1}}],
},
{
"step_id": "step_2",
"step_number": 2,
"tool_calls": [{"name": "click", "args": {"target": 2}}],
"action_taken": {"action": "click", "target": 2},
},
]
captured_messages_2 = []
async def mock_invoke_llm_loop_2(*args, **kwargs):
messages = await node._build_prompt(
state=mock_state,
latest_screenshot_b64="YmFzZTY0X3NjcmVlbnNob3Q=",
fused_xml=[],
minimal_list="",
current_step_num=3,
steps=steps_not_updated,
task_plan="",
)
captured_messages_2.extend(messages)
return [], None, None, False
with (
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
patch(
"artemis.agents.operator.operator.OperatorNode._invoke_llm_loop",
side_effect=mock_invoke_llm_loop_2,
),
):
node = OperatorNode(mock_ctx, transcript_config=LEGACY_TRANSCRIPT)
node_update = await node(mock_state)
full_content_2 = ""
for m in captured_messages_2:
if isinstance(m.content, list):
for part in m.content:
if isinstance(part, dict) and "text" in part:
full_content_2 += part["text"] + "\n"
elif isinstance(m.content, str):
full_content_2 += m.content + "\n"
assert "Reminder: your last turn executed a Turn-Ending Action" in full_content_2
@pytest.mark.asyncio
async def test_operator_tool_limit_exceeded_warning():
from artemis.agents.operator.operator import OperatorNode