feat: harden agent verification and runtime contracts
Introduce two-gate checker settlement, explicit graph visibility and checkpoints, structured LLM reliability, centralized action specifications, and corresponding tests and design documentation.
This commit is contained in:
1 parent
ddfa420945
commit
cbe8f4f0c6
106 files changed
+11542
-4445
No files matched your search
@@ -13,9 +13,8 @@
|
||||
# limitations under the License.
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
@@ -28,7 +27,7 @@ from artemis.tools.scratchpad import (
|
||||
append_note,
|
||||
append_note_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -108,11 +107,9 @@ async def test_append_note_callable_execution(mock_ctx, tmp_path):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_note_with_state_command(mock_ctx, tmp_path):
|
||||
"""Verify AppendNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
|
||||
async def test_append_note_with_state_tool_message(mock_ctx, tmp_path):
|
||||
"""Verify AppendNoteTool.execute with state returns a ToolMessage directly."""
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.current_agent = "operator"
|
||||
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
|
||||
|
||||
result = await append_note.execute(
|
||||
ctx=mock_ctx,
|
||||
@@ -122,15 +119,10 @@ async def test_append_note_with_state_command(mock_ctx, tmp_path):
|
||||
tool_call_id="call_append_999",
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in result.update
|
||||
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0].tool_call_id == "call_append_999"
|
||||
assert tool_messages[0].content == "Successfully appended note to state_append_note.md."
|
||||
assert tool_messages[0].status == "success"
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert result.tool_call_id == "call_append_999"
|
||||
assert result.content == "Successfully appended note to state_append_note.md."
|
||||
assert result.status == "success"
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "state_append_note.md"
|
||||
assert note_path.exists()
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
@@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import (
|
||||
click_wrapper,
|
||||
get_click_tool,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -50,13 +49,7 @@ def mock_driver():
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
state = MagicMock(spec=State)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
return MagicMock(spec=State)
|
||||
|
||||
|
||||
def test_click_tool_subclass_and_registry():
|
||||
@@ -108,7 +101,7 @@ async def test_click_direct_execution_with_ctx(mock_ctx):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify ClickTool returns Command when state is provided."""
|
||||
"""Verify ClickTool returns ToolMessage when state is provided."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
|
||||
@@ -120,13 +113,10 @@ async def test_click_with_state_command(mock_ctx, mock_state):
|
||||
tool_call_id="call_click_1",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_click_1"
|
||||
assert messages[0].status == "success"
|
||||
assert "Clicked at [100, 200]" in messages[0].content
|
||||
assert isinstance(cmd, ToolMessage)
|
||||
assert cmd.tool_call_id == "call_click_1"
|
||||
assert cmd.status == "success"
|
||||
assert "Clicked at [100, 200]" in cmd.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -62,8 +62,6 @@ class TestCommitteeTool(unittest.IsolatedAsyncioTestCase):
|
||||
mock_state.device_date = None
|
||||
mock_state.structured_decisions = None
|
||||
mock_state.complete_subgoals_by_ids = []
|
||||
mock_state.validator_messages = []
|
||||
mock_state.operator_replan_reason = "Mock Immediate Replan Reason"
|
||||
|
||||
# Mock LLM responses for the members
|
||||
mock_llm_pl = MagicMock()
|
||||
@@ -165,9 +163,6 @@ class TestCommitteeTool(unittest.IsolatedAsyncioTestCase):
|
||||
"Timeout waiting for element", content
|
||||
) # Check failed history is in blackboard
|
||||
self.assertIn("Test Goal", content) # Check initial goal is in blackboard
|
||||
self.assertIn(
|
||||
"Mock Immediate Replan Reason", content
|
||||
) # Check current replan reason is in blackboard
|
||||
|
||||
finally:
|
||||
# Clean up temp directory
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
@@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import (
|
||||
input_text,
|
||||
input_text_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -51,13 +50,7 @@ def mock_driver():
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
state = MagicMock(spec=State)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
return MagicMock(spec=State)
|
||||
|
||||
|
||||
def test_input_text_tool_subclass_and_registry():
|
||||
@@ -120,7 +113,7 @@ async def test_input_text_direct_execution_with_ctx(mock_ctx):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_text_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify InputTextTool returns Command when state is provided."""
|
||||
"""Verify InputTextTool returns ToolMessage when state is provided."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
|
||||
@@ -135,13 +128,10 @@ async def test_input_text_with_state_command(mock_ctx, mock_state):
|
||||
tool_call_id="call_it_1",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_it_1"
|
||||
assert messages[0].status == "success"
|
||||
assert "Typed 'query'" in messages[0].content
|
||||
assert isinstance(cmd, ToolMessage)
|
||||
assert cmd.tool_call_id == "call_it_1"
|
||||
assert cmd.status == "success"
|
||||
assert "Typed 'query'" in cmd.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
@@ -27,7 +26,7 @@ from artemis.tools.mobile.launch_app import (
|
||||
launch_app,
|
||||
launch_app_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -47,13 +46,7 @@ def mock_driver():
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
state = MagicMock(spec=State)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
return MagicMock(spec=State)
|
||||
|
||||
|
||||
def test_launch_app_tool_subclass_and_registry():
|
||||
@@ -144,7 +137,7 @@ async def test_launch_app_execution_failure(mock_ctx):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_launch_app_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify LaunchAppTool returns Command when state is provided."""
|
||||
"""Verify LaunchAppTool returns ToolMessage when state is provided."""
|
||||
with (
|
||||
patch(
|
||||
"artemis.tools.mobile.launch_app.find_package",
|
||||
@@ -163,13 +156,10 @@ async def test_launch_app_with_state_command(mock_ctx, mock_state):
|
||||
tool_call_id="call_la_1",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_la_1"
|
||||
assert messages[0].status == "success"
|
||||
assert "App 'State App' launched successfully." in messages[0].content
|
||||
assert isinstance(cmd, ToolMessage)
|
||||
assert cmd.tool_call_id == "call_la_1"
|
||||
assert cmd.status == "success"
|
||||
assert "App 'State App' launched successfully." in cmd.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -12,9 +12,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
@@ -28,7 +27,7 @@ from artemis.tools.scratchpad import (
|
||||
list_notes_wrapper,
|
||||
)
|
||||
from artemis.utils.notes import save_note_content
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -95,13 +94,11 @@ async def test_list_notes_callable_execution(mock_ctx, tmp_path):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_notes_with_state_command(mock_ctx, tmp_path):
|
||||
"""Verify ListNotesTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
|
||||
async def test_list_notes_with_state_tool_message(mock_ctx, tmp_path):
|
||||
"""Verify ListNotesTool.execute with state returns a ToolMessage directly."""
|
||||
save_note_content(tmp_path, "state_note", "content")
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.current_agent = "operator"
|
||||
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
|
||||
|
||||
result = await list_notes.execute(
|
||||
ctx=mock_ctx,
|
||||
@@ -109,16 +106,11 @@ async def test_list_notes_with_state_command(mock_ctx, tmp_path):
|
||||
tool_call_id="call_list_999",
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in result.update
|
||||
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0].tool_call_id == "call_list_999"
|
||||
assert "Here are all the notes:" in tool_messages[0].content
|
||||
assert "- state_note (1 lines)" in tool_messages[0].content
|
||||
assert tool_messages[0].status == "success"
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert result.tool_call_id == "call_list_999"
|
||||
assert "Here are all the notes:" in result.content
|
||||
assert "- state_note (1 lines)" in result.content
|
||||
assert result.status == "success"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
@@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import (
|
||||
long_press,
|
||||
long_press_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -50,13 +49,7 @@ def mock_driver():
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
state = MagicMock(spec=State)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
return MagicMock(spec=State)
|
||||
|
||||
|
||||
def test_long_press_tool_subclass_and_registry():
|
||||
@@ -111,7 +104,7 @@ async def test_long_press_direct_execution_with_ctx(mock_ctx):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_press_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify LongPressTool returns Command when state is provided."""
|
||||
"""Verify LongPressTool returns ToolMessage when state is provided."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
|
||||
@@ -124,13 +117,10 @@ async def test_long_press_with_state_command(mock_ctx, mock_state):
|
||||
tool_call_id="call_lp_1",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_lp_1"
|
||||
assert messages[0].status == "success"
|
||||
assert "Long pressed at [100, 200]" in messages[0].content
|
||||
assert isinstance(cmd, ToolMessage)
|
||||
assert cmd.tool_call_id == "call_lp_1"
|
||||
assert cmd.status == "success"
|
||||
assert "Long pressed at [100, 200]" in cmd.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from langchain_core.messages import ToolMessage
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
@@ -110,13 +112,12 @@ async def test_object_detection_direct_execution_success():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_object_detection_direct_execution_with_state():
|
||||
"""Verify operator_object_detection reads screenshot from state."""
|
||||
"""Verify operator_object_detection reads screenshot from state and returns a ToolMessage."""
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = None
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/path/to/state_screen.jpg"
|
||||
mock_state.asanitize_update = None # Non-langgraph direct call
|
||||
|
||||
expected_result = {
|
||||
"detected": [{"label": "search bar", "point": [500, 100]}],
|
||||
@@ -137,7 +138,13 @@ async def test_operator_object_detection_direct_execution_with_state():
|
||||
# Verify it used the latest_screenshot from state
|
||||
call_args = mock_run.call_args[0]
|
||||
assert call_args[1] == "/path/to/state_screen.jpg"
|
||||
assert json.loads(result) == expected_result
|
||||
|
||||
# With state, the tool returns a ToolMessage directly
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert result.status == "success"
|
||||
assert "Object Detection successful." in result.content
|
||||
json_payload = result.content.split("Results:\n", 1)[1]
|
||||
assert json.loads(json_payload) == expected_result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -27,7 +27,6 @@ async def test_object_detection_trace():
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.latest_screenshot = "/path/to/screenshot.jpg"
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
|
||||
# We need to mock the decorator in object_detection_tool.py if we want to avoid it calling real get_llm
|
||||
# Or we can just mock get_llm.
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
import base64
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
@@ -30,7 +29,7 @@ from artemis.tools.mobile.ocr import (
|
||||
ocr_recognition,
|
||||
ocr_recognition_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -61,11 +60,6 @@ def mock_state(tmp_path):
|
||||
|
||||
state = MagicMock(spec=State)
|
||||
state.latest_screenshot = str(screenshot_file)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
|
||||
|
||||
@@ -125,8 +119,9 @@ async def test_ocr_direct_execution_with_state(mock_ctx, tmp_path):
|
||||
return_value=mock_results,
|
||||
):
|
||||
result = await ocr_recognition.execute(ctx=mock_ctx, state=simple_state)
|
||||
assert "OCR Recognition successful." in result
|
||||
assert "Submit" in result
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert "OCR Recognition successful." in result.content
|
||||
assert "Submit" in result.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -164,12 +159,13 @@ async def test_ocr_no_text_detected(mock_ctx, tmp_path):
|
||||
return_value=[],
|
||||
):
|
||||
result = await ocr_recognition.execute(ctx=mock_ctx, state=simple_state)
|
||||
assert "No text detected on the screen." in result
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert "No text detected on the screen." in result.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ocr_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify OcrRecognitionTool returns Command when state is provided."""
|
||||
"""Verify OcrRecognitionTool returns ToolMessage when state is provided."""
|
||||
mock_results = [{"text": "Hello", "position": [{"x": 100, "y": 100}]}]
|
||||
with patch(
|
||||
"artemis.tools.mobile.ocr.perform_ocr",
|
||||
@@ -181,13 +177,10 @@ async def test_ocr_with_state_command(mock_ctx, mock_state):
|
||||
tool_call_id="call_ocr_1",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_ocr_1"
|
||||
assert messages[0].status == "success"
|
||||
assert "Hello" in messages[0].content
|
||||
assert isinstance(cmd, ToolMessage)
|
||||
assert cmd.tool_call_id == "call_ocr_1"
|
||||
assert cmd.status == "success"
|
||||
assert "Hello" in cmd.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -195,17 +188,15 @@ async def test_ocr_failure_missing_screenshot(mock_ctx):
|
||||
"""Verify error handling when screenshot path is missing."""
|
||||
empty_state = MagicMock(spec=State)
|
||||
empty_state.latest_screenshot = None
|
||||
empty_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
|
||||
|
||||
cmd = await ocr_recognition.execute(
|
||||
ctx=mock_ctx,
|
||||
tool_call_id="call_ocr_err",
|
||||
state=empty_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert messages[0].status == "error"
|
||||
assert "OCR Recognition failed" in messages[0].content
|
||||
assert isinstance(cmd, ToolMessage)
|
||||
assert cmd.status == "error"
|
||||
assert "OCR Recognition failed" in cmd.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
@@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import (
|
||||
press_key,
|
||||
press_key_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -47,13 +46,7 @@ def mock_driver():
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
state = MagicMock(spec=State)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
return MagicMock(spec=State)
|
||||
|
||||
|
||||
def test_press_key_tool_subclass_and_registry():
|
||||
@@ -124,7 +117,7 @@ async def test_press_key_direct_execution_with_ctx(mock_ctx):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_press_key_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify PressKeyTool returns Command when state is provided."""
|
||||
"""Verify PressKeyTool returns ToolMessage when state is provided."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.go_home = AsyncMock(return_value=True)
|
||||
@@ -136,13 +129,10 @@ async def test_press_key_with_state_command(mock_ctx, mock_state):
|
||||
tool_call_id="call_pk_1",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_pk_1"
|
||||
assert messages[0].status == "success"
|
||||
assert "Pressed key 'HOME' successfully." in messages[0].content
|
||||
assert isinstance(cmd, ToolMessage)
|
||||
assert cmd.tool_call_id == "call_pk_1"
|
||||
assert cmd.status == "success"
|
||||
assert "Pressed key 'HOME' successfully." in cmd.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -12,9 +12,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
@@ -28,7 +27,7 @@ from artemis.tools.scratchpad import (
|
||||
read_note_wrapper,
|
||||
)
|
||||
from artemis.utils.notes import save_note_content
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -106,13 +105,11 @@ async def test_read_note_callable_execution(mock_ctx, tmp_path):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_with_state_command(mock_ctx, tmp_path):
|
||||
"""Verify ReadNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
|
||||
async def test_read_note_with_state_tool_message(mock_ctx, tmp_path):
|
||||
"""Verify ReadNoteTool.execute with state returns a ToolMessage directly."""
|
||||
save_note_content(tmp_path, "state_sample", "state note content")
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.current_agent = "operator"
|
||||
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
|
||||
|
||||
result = await read_note.execute(
|
||||
ctx=mock_ctx,
|
||||
@@ -121,16 +118,11 @@ async def test_read_note_with_state_command(mock_ctx, tmp_path):
|
||||
tool_call_id="call_read_999",
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in result.update
|
||||
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0].tool_call_id == "call_read_999"
|
||||
assert "Successfully read note 'state_sample'." in tool_messages[0].content
|
||||
assert "state note content" in tool_messages[0].content
|
||||
assert tool_messages[0].status == "success"
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert result.tool_call_id == "call_read_999"
|
||||
assert "Successfully read note 'state_sample'." in result.content
|
||||
assert "state note content" in result.content
|
||||
assert result.status == "success"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -13,9 +13,8 @@
|
||||
# limitations under the License.
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
@@ -28,7 +27,7 @@ from artemis.tools.scratchpad import (
|
||||
save_note,
|
||||
save_note_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -98,11 +97,9 @@ async def test_save_note_callable_execution(mock_ctx, tmp_path):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_save_note_with_state_command(mock_ctx, tmp_path):
|
||||
"""Verify SaveNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
|
||||
async def test_save_note_with_state_tool_message(mock_ctx, tmp_path):
|
||||
"""Verify SaveNoteTool.execute with state returns a ToolMessage directly."""
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.current_agent = "operator"
|
||||
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
|
||||
|
||||
result = await save_note.execute(
|
||||
ctx=mock_ctx,
|
||||
@@ -112,15 +109,10 @@ async def test_save_note_with_state_command(mock_ctx, tmp_path):
|
||||
tool_call_id="call_save_999",
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in result.update
|
||||
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0].tool_call_id == "call_save_999"
|
||||
assert tool_messages[0].content == "Successfully saved note to state_note.md."
|
||||
assert tool_messages[0].status == "success"
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert result.tool_call_id == "call_save_999"
|
||||
assert result.content == "Successfully saved note to state_note.md."
|
||||
assert result.status == "success"
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "state_note.md"
|
||||
assert note_path.exists()
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
@@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import (
|
||||
swipe,
|
||||
swipe_wrapper,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -51,13 +50,7 @@ def mock_driver():
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
state = MagicMock(spec=State)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
return MagicMock(spec=State)
|
||||
|
||||
|
||||
def test_swipe_tool_subclass_and_registry():
|
||||
@@ -131,7 +124,7 @@ async def test_swipe_direct_execution_with_ctx_coords(mock_ctx):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify SwipeTool returns Command when state is provided."""
|
||||
"""Verify SwipeTool returns ToolMessage when state is provided."""
|
||||
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
|
||||
mock_controller_inst = MagicMock()
|
||||
mock_controller_inst.swipe_coords = AsyncMock(return_value=None)
|
||||
@@ -143,13 +136,10 @@ async def test_swipe_with_state_command(mock_ctx, mock_state):
|
||||
tool_call_id="call_sw_1",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_sw_1"
|
||||
assert messages[0].status == "success"
|
||||
assert "Swipe completed successfully." in messages[0].content
|
||||
assert isinstance(cmd, ToolMessage)
|
||||
assert cmd.tool_call_id == "call_sw_1"
|
||||
assert cmd.status == "success"
|
||||
assert "Swipe completed successfully." in cmd.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -13,9 +13,8 @@
|
||||
# limitations under the License.
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
@@ -29,7 +28,7 @@ from artemis.tools.scratchpad import (
|
||||
update_note_wrapper,
|
||||
)
|
||||
from artemis.utils.notes import save_note_content
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -123,13 +122,11 @@ async def test_update_note_callable_execution(mock_ctx, tmp_path):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_note_with_state_command(mock_ctx, tmp_path):
|
||||
"""Verify UpdateNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
|
||||
async def test_update_note_with_state_tool_message(mock_ctx, tmp_path):
|
||||
"""Verify UpdateNoteTool.execute with state returns a ToolMessage directly."""
|
||||
save_note_content(tmp_path, "state_plan", "- [ ] Step 1")
|
||||
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.current_agent = "operator"
|
||||
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
|
||||
|
||||
result = await update_note.execute(
|
||||
ctx=mock_ctx,
|
||||
@@ -140,15 +137,10 @@ async def test_update_note_with_state_command(mock_ctx, tmp_path):
|
||||
tool_call_id="call_update_999",
|
||||
)
|
||||
|
||||
assert isinstance(result, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in result.update
|
||||
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0].tool_call_id == "call_update_999"
|
||||
assert tool_messages[0].content == "Successfully updated note 'state_plan'."
|
||||
assert tool_messages[0].status == "success"
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
assert isinstance(result, ToolMessage)
|
||||
assert result.tool_call_id == "call_update_999"
|
||||
assert result.content == "Successfully updated note 'state_plan'."
|
||||
assert result.status == "success"
|
||||
|
||||
note_path = Path(tmp_path) / "notes" / "state_plan.md"
|
||||
assert note_path.read_text(encoding="utf-8") == "- [x] Step 1"
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
@@ -27,7 +26,7 @@ from artemis.tools.video_tool import (
|
||||
get_video_analyzer_tool_pure,
|
||||
video_analyzer,
|
||||
)
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import ToolMessage
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -38,14 +37,8 @@ def mock_ctx():
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state(mock_ctx):
|
||||
state = MagicMock(spec=State)
|
||||
|
||||
async def _mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
|
||||
return state
|
||||
def mock_state():
|
||||
return MagicMock(spec=State)
|
||||
|
||||
|
||||
def test_video_analyzer_tool_subclass_and_registry():
|
||||
@@ -104,7 +97,7 @@ async def test_video_analyzer_direct_execution_failed(mock_ctx):
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_analyzer_with_state_command(mock_ctx, mock_state):
|
||||
"""Verify execute returns a Command with ToolMessage when state is provided."""
|
||||
"""Verify execute returns a ToolMessage when state is provided."""
|
||||
with patch(
|
||||
"artemis.tools.video_tool.VideoAnalyzer.run",
|
||||
new_callable=AsyncMock,
|
||||
@@ -117,13 +110,10 @@ async def test_video_analyzer_with_state_command(mock_ctx, mock_state):
|
||||
tool_call_id="call_video_123",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_call_id == "call_video_123"
|
||||
assert messages[0].content == "Action verified in video"
|
||||
assert messages[0].status == "success"
|
||||
assert isinstance(cmd, ToolMessage)
|
||||
assert cmd.tool_call_id == "call_video_123"
|
||||
assert cmd.content == "Action verified in video"
|
||||
assert cmd.status == "success"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -150,11 +140,9 @@ async def test_video_analyzer_exception_handling(mock_ctx, mock_state):
|
||||
tool_call_id="call_err",
|
||||
state=mock_state,
|
||||
)
|
||||
assert isinstance(cmd, Command)
|
||||
assert VALIDATOR_MESSAGES_KEY in cmd.update
|
||||
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
|
||||
assert messages[0].status == "error"
|
||||
assert "Error running video analyzer" in messages[0].content
|
||||
assert isinstance(cmd, ToolMessage)
|
||||
assert cmd.status == "error"
|
||||
assert "Error running video analyzer" in cmd.content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Reference in new issue
Block a user