feat: harden agent verification and runtime contracts

Introduce two-gate checker settlement, explicit graph visibility and checkpoints, structured LLM reliability, centralized action specifications, and corresponding tests and design documentation.
This commit is contained in:
somew1nd committed 2026-08-31 20:42:24 -07:00
1 parent ddfa420945
commit cbe8f4f0c6
106 files changed
+11542 -4445

No files matched your search

+8 -16
View File
@@ -13,9 +13,8 @@
# limitations under the License.
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
@@ -28,7 +27,7 @@ from artemis.tools.scratchpad import (
append_note,
append_note_wrapper,
)
from langgraph.types import Command
from langchain_core.messages import ToolMessage
import pytest
@@ -108,11 +107,9 @@ async def test_append_note_callable_execution(mock_ctx, tmp_path):
@pytest.mark.asyncio
async def test_append_note_with_state_command(mock_ctx, tmp_path):
"""Verify AppendNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
async def test_append_note_with_state_tool_message(mock_ctx, tmp_path):
"""Verify AppendNoteTool.execute with state returns a ToolMessage directly."""
mock_state = MagicMock(spec=State)
mock_state.current_agent = "operator"
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
result = await append_note.execute(
ctx=mock_ctx,
@@ -122,15 +119,10 @@ async def test_append_note_with_state_command(mock_ctx, tmp_path):
tool_call_id="call_append_999",
)
assert isinstance(result, Command)
assert VALIDATOR_MESSAGES_KEY in result.update
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
assert len(tool_messages) == 1
assert tool_messages[0].tool_call_id == "call_append_999"
assert tool_messages[0].content == "Successfully appended note to state_append_note.md."
assert tool_messages[0].status == "success"
mock_state.asanitize_update.assert_called_once()
assert isinstance(result, ToolMessage)
assert result.tool_call_id == "call_append_999"
assert result.content == "Successfully appended note to state_append_note.md."
assert result.status == "success"
note_path = Path(tmp_path) / "notes" / "state_append_note.md"
assert note_path.exists()
+7 -17
View File
@@ -14,7 +14,6 @@
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
@@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import (
click_wrapper,
get_click_tool,
)
from langgraph.types import Command
from langchain_core.messages import ToolMessage
import pytest
@@ -50,13 +49,7 @@ def mock_driver():
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
return MagicMock(spec=State)
def test_click_tool_subclass_and_registry():
@@ -108,7 +101,7 @@ async def test_click_direct_execution_with_ctx(mock_ctx):
@pytest.mark.asyncio
async def test_click_with_state_command(mock_ctx, mock_state):
"""Verify ClickTool returns Command when state is provided."""
"""Verify ClickTool returns ToolMessage when state is provided."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
@@ -120,13 +113,10 @@ async def test_click_with_state_command(mock_ctx, mock_state):
tool_call_id="call_click_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_click_1"
assert messages[0].status == "success"
assert "Clicked at [100, 200]" in messages[0].content
assert isinstance(cmd, ToolMessage)
assert cmd.tool_call_id == "call_click_1"
assert cmd.status == "success"
assert "Clicked at [100, 200]" in cmd.content
@pytest.mark.asyncio
-5
View File
@@ -62,8 +62,6 @@ class TestCommitteeTool(unittest.IsolatedAsyncioTestCase):
mock_state.device_date = None
mock_state.structured_decisions = None
mock_state.complete_subgoals_by_ids = []
mock_state.validator_messages = []
mock_state.operator_replan_reason = "Mock Immediate Replan Reason"
# Mock LLM responses for the members
mock_llm_pl = MagicMock()
@@ -165,9 +163,6 @@ class TestCommitteeTool(unittest.IsolatedAsyncioTestCase):
"Timeout waiting for element", content
) # Check failed history is in blackboard
self.assertIn("Test Goal", content) # Check initial goal is in blackboard
self.assertIn(
"Mock Immediate Replan Reason", content
) # Check current replan reason is in blackboard
finally:
# Clean up temp directory
+7 -17
View File
@@ -14,7 +14,6 @@
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
@@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import (
input_text,
input_text_wrapper,
)
from langgraph.types import Command
from langchain_core.messages import ToolMessage
import pytest
@@ -51,13 +50,7 @@ def mock_driver():
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
return MagicMock(spec=State)
def test_input_text_tool_subclass_and_registry():
@@ -120,7 +113,7 @@ async def test_input_text_direct_execution_with_ctx(mock_ctx):
@pytest.mark.asyncio
async def test_input_text_with_state_command(mock_ctx, mock_state):
"""Verify InputTextTool returns Command when state is provided."""
"""Verify InputTextTool returns ToolMessage when state is provided."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
@@ -135,13 +128,10 @@ async def test_input_text_with_state_command(mock_ctx, mock_state):
tool_call_id="call_it_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_it_1"
assert messages[0].status == "success"
assert "Typed 'query'" in messages[0].content
assert isinstance(cmd, ToolMessage)
assert cmd.tool_call_id == "call_it_1"
assert cmd.status == "success"
assert "Typed 'query'" in cmd.content
@pytest.mark.asyncio
+7 -17
View File
@@ -14,7 +14,6 @@
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
@@ -27,7 +26,7 @@ from artemis.tools.mobile.launch_app import (
launch_app,
launch_app_wrapper,
)
from langgraph.types import Command
from langchain_core.messages import ToolMessage
import pytest
@@ -47,13 +46,7 @@ def mock_driver():
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
return MagicMock(spec=State)
def test_launch_app_tool_subclass_and_registry():
@@ -144,7 +137,7 @@ async def test_launch_app_execution_failure(mock_ctx):
@pytest.mark.asyncio
async def test_launch_app_with_state_command(mock_ctx, mock_state):
"""Verify LaunchAppTool returns Command when state is provided."""
"""Verify LaunchAppTool returns ToolMessage when state is provided."""
with (
patch(
"artemis.tools.mobile.launch_app.find_package",
@@ -163,13 +156,10 @@ async def test_launch_app_with_state_command(mock_ctx, mock_state):
tool_call_id="call_la_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_la_1"
assert messages[0].status == "success"
assert "App 'State App' launched successfully." in messages[0].content
assert isinstance(cmd, ToolMessage)
assert cmd.tool_call_id == "call_la_1"
assert cmd.status == "success"
assert "App 'State App' launched successfully." in cmd.content
@pytest.mark.asyncio
+9 -17
View File
@@ -12,9 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
@@ -28,7 +27,7 @@ from artemis.tools.scratchpad import (
list_notes_wrapper,
)
from artemis.utils.notes import save_note_content
from langgraph.types import Command
from langchain_core.messages import ToolMessage
import pytest
@@ -95,13 +94,11 @@ async def test_list_notes_callable_execution(mock_ctx, tmp_path):
@pytest.mark.asyncio
async def test_list_notes_with_state_command(mock_ctx, tmp_path):
"""Verify ListNotesTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
async def test_list_notes_with_state_tool_message(mock_ctx, tmp_path):
"""Verify ListNotesTool.execute with state returns a ToolMessage directly."""
save_note_content(tmp_path, "state_note", "content")
mock_state = MagicMock(spec=State)
mock_state.current_agent = "operator"
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
result = await list_notes.execute(
ctx=mock_ctx,
@@ -109,16 +106,11 @@ async def test_list_notes_with_state_command(mock_ctx, tmp_path):
tool_call_id="call_list_999",
)
assert isinstance(result, Command)
assert VALIDATOR_MESSAGES_KEY in result.update
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
assert len(tool_messages) == 1
assert tool_messages[0].tool_call_id == "call_list_999"
assert "Here are all the notes:" in tool_messages[0].content
assert "- state_note (1 lines)" in tool_messages[0].content
assert tool_messages[0].status == "success"
mock_state.asanitize_update.assert_called_once()
assert isinstance(result, ToolMessage)
assert result.tool_call_id == "call_list_999"
assert "Here are all the notes:" in result.content
assert "- state_note (1 lines)" in result.content
assert result.status == "success"
@pytest.mark.asyncio
+7 -17
View File
@@ -14,7 +14,6 @@
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
@@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import (
long_press,
long_press_wrapper,
)
from langgraph.types import Command
from langchain_core.messages import ToolMessage
import pytest
@@ -50,13 +49,7 @@ def mock_driver():
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
return MagicMock(spec=State)
def test_long_press_tool_subclass_and_registry():
@@ -111,7 +104,7 @@ async def test_long_press_direct_execution_with_ctx(mock_ctx):
@pytest.mark.asyncio
async def test_long_press_with_state_command(mock_ctx, mock_state):
"""Verify LongPressTool returns Command when state is provided."""
"""Verify LongPressTool returns ToolMessage when state is provided."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
@@ -124,13 +117,10 @@ async def test_long_press_with_state_command(mock_ctx, mock_state):
tool_call_id="call_lp_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_lp_1"
assert messages[0].status == "success"
assert "Long pressed at [100, 200]" in messages[0].content
assert isinstance(cmd, ToolMessage)
assert cmd.tool_call_id == "call_lp_1"
assert cmd.status == "success"
assert "Long pressed at [100, 200]" in cmd.content
@pytest.mark.asyncio
+10 -3
View File
@@ -17,6 +17,8 @@
import json
from unittest.mock import AsyncMock, MagicMock, patch
from langchain_core.messages import ToolMessage
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
@@ -110,13 +112,12 @@ async def test_object_detection_direct_execution_success():
@pytest.mark.asyncio
async def test_operator_object_detection_direct_execution_with_state():
"""Verify operator_object_detection reads screenshot from state."""
"""Verify operator_object_detection reads screenshot from state and returns a ToolMessage."""
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/path/to/state_screen.jpg"
mock_state.asanitize_update = None # Non-langgraph direct call
expected_result = {
"detected": [{"label": "search bar", "point": [500, 100]}],
@@ -137,7 +138,13 @@ async def test_operator_object_detection_direct_execution_with_state():
# Verify it used the latest_screenshot from state
call_args = mock_run.call_args[0]
assert call_args[1] == "/path/to/state_screen.jpg"
assert json.loads(result) == expected_result
# With state, the tool returns a ToolMessage directly
assert isinstance(result, ToolMessage)
assert result.status == "success"
assert "Object Detection successful." in result.content
json_payload = result.content.split("Results:\n", 1)[1]
assert json.loads(json_payload) == expected_result
@pytest.mark.asyncio
@@ -27,7 +27,6 @@ async def test_object_detection_trace():
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/path/to/screenshot.jpg"
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
# We need to mock the decorator in object_detection_tool.py if we want to avoid it calling real get_llm
# Or we can just mock get_llm.
+14 -23
View File
@@ -15,7 +15,6 @@
import base64
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
@@ -30,7 +29,7 @@ from artemis.tools.mobile.ocr import (
ocr_recognition,
ocr_recognition_wrapper,
)
from langgraph.types import Command
from langchain_core.messages import ToolMessage
import pytest
@@ -61,11 +60,6 @@ def mock_state(tmp_path):
state = MagicMock(spec=State)
state.latest_screenshot = str(screenshot_file)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
@@ -125,8 +119,9 @@ async def test_ocr_direct_execution_with_state(mock_ctx, tmp_path):
return_value=mock_results,
):
result = await ocr_recognition.execute(ctx=mock_ctx, state=simple_state)
assert "OCR Recognition successful." in result
assert "Submit" in result
assert isinstance(result, ToolMessage)
assert "OCR Recognition successful." in result.content
assert "Submit" in result.content
@pytest.mark.asyncio
@@ -164,12 +159,13 @@ async def test_ocr_no_text_detected(mock_ctx, tmp_path):
return_value=[],
):
result = await ocr_recognition.execute(ctx=mock_ctx, state=simple_state)
assert "No text detected on the screen." in result
assert isinstance(result, ToolMessage)
assert "No text detected on the screen." in result.content
@pytest.mark.asyncio
async def test_ocr_with_state_command(mock_ctx, mock_state):
"""Verify OcrRecognitionTool returns Command when state is provided."""
"""Verify OcrRecognitionTool returns ToolMessage when state is provided."""
mock_results = [{"text": "Hello", "position": [{"x": 100, "y": 100}]}]
with patch(
"artemis.tools.mobile.ocr.perform_ocr",
@@ -181,13 +177,10 @@ async def test_ocr_with_state_command(mock_ctx, mock_state):
tool_call_id="call_ocr_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_ocr_1"
assert messages[0].status == "success"
assert "Hello" in messages[0].content
assert isinstance(cmd, ToolMessage)
assert cmd.tool_call_id == "call_ocr_1"
assert cmd.status == "success"
assert "Hello" in cmd.content
@pytest.mark.asyncio
@@ -195,17 +188,15 @@ async def test_ocr_failure_missing_screenshot(mock_ctx):
"""Verify error handling when screenshot path is missing."""
empty_state = MagicMock(spec=State)
empty_state.latest_screenshot = None
empty_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
cmd = await ocr_recognition.execute(
ctx=mock_ctx,
tool_call_id="call_ocr_err",
state=empty_state,
)
assert isinstance(cmd, Command)
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert messages[0].status == "error"
assert "OCR Recognition failed" in messages[0].content
assert isinstance(cmd, ToolMessage)
assert cmd.status == "error"
assert "OCR Recognition failed" in cmd.content
@pytest.mark.asyncio
+7 -17
View File
@@ -14,7 +14,6 @@
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
@@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import (
press_key,
press_key_wrapper,
)
from langgraph.types import Command
from langchain_core.messages import ToolMessage
import pytest
@@ -47,13 +46,7 @@ def mock_driver():
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
return MagicMock(spec=State)
def test_press_key_tool_subclass_and_registry():
@@ -124,7 +117,7 @@ async def test_press_key_direct_execution_with_ctx(mock_ctx):
@pytest.mark.asyncio
async def test_press_key_with_state_command(mock_ctx, mock_state):
"""Verify PressKeyTool returns Command when state is provided."""
"""Verify PressKeyTool returns ToolMessage when state is provided."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.go_home = AsyncMock(return_value=True)
@@ -136,13 +129,10 @@ async def test_press_key_with_state_command(mock_ctx, mock_state):
tool_call_id="call_pk_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_pk_1"
assert messages[0].status == "success"
assert "Pressed key 'HOME' successfully." in messages[0].content
assert isinstance(cmd, ToolMessage)
assert cmd.tool_call_id == "call_pk_1"
assert cmd.status == "success"
assert "Pressed key 'HOME' successfully." in cmd.content
@pytest.mark.asyncio
+9 -17
View File
@@ -12,9 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
@@ -28,7 +27,7 @@ from artemis.tools.scratchpad import (
read_note_wrapper,
)
from artemis.utils.notes import save_note_content
from langgraph.types import Command
from langchain_core.messages import ToolMessage
import pytest
@@ -106,13 +105,11 @@ async def test_read_note_callable_execution(mock_ctx, tmp_path):
@pytest.mark.asyncio
async def test_read_note_with_state_command(mock_ctx, tmp_path):
"""Verify ReadNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
async def test_read_note_with_state_tool_message(mock_ctx, tmp_path):
"""Verify ReadNoteTool.execute with state returns a ToolMessage directly."""
save_note_content(tmp_path, "state_sample", "state note content")
mock_state = MagicMock(spec=State)
mock_state.current_agent = "operator"
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
result = await read_note.execute(
ctx=mock_ctx,
@@ -121,16 +118,11 @@ async def test_read_note_with_state_command(mock_ctx, tmp_path):
tool_call_id="call_read_999",
)
assert isinstance(result, Command)
assert VALIDATOR_MESSAGES_KEY in result.update
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
assert len(tool_messages) == 1
assert tool_messages[0].tool_call_id == "call_read_999"
assert "Successfully read note 'state_sample'." in tool_messages[0].content
assert "state note content" in tool_messages[0].content
assert tool_messages[0].status == "success"
mock_state.asanitize_update.assert_called_once()
assert isinstance(result, ToolMessage)
assert result.tool_call_id == "call_read_999"
assert "Successfully read note 'state_sample'." in result.content
assert "state note content" in result.content
assert result.status == "success"
@pytest.mark.asyncio
+8 -16
View File
@@ -13,9 +13,8 @@
# limitations under the License.
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
@@ -28,7 +27,7 @@ from artemis.tools.scratchpad import (
save_note,
save_note_wrapper,
)
from langgraph.types import Command
from langchain_core.messages import ToolMessage
import pytest
@@ -98,11 +97,9 @@ async def test_save_note_callable_execution(mock_ctx, tmp_path):
@pytest.mark.asyncio
async def test_save_note_with_state_command(mock_ctx, tmp_path):
"""Verify SaveNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
async def test_save_note_with_state_tool_message(mock_ctx, tmp_path):
"""Verify SaveNoteTool.execute with state returns a ToolMessage directly."""
mock_state = MagicMock(spec=State)
mock_state.current_agent = "operator"
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
result = await save_note.execute(
ctx=mock_ctx,
@@ -112,15 +109,10 @@ async def test_save_note_with_state_command(mock_ctx, tmp_path):
tool_call_id="call_save_999",
)
assert isinstance(result, Command)
assert VALIDATOR_MESSAGES_KEY in result.update
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
assert len(tool_messages) == 1
assert tool_messages[0].tool_call_id == "call_save_999"
assert tool_messages[0].content == "Successfully saved note to state_note.md."
assert tool_messages[0].status == "success"
mock_state.asanitize_update.assert_called_once()
assert isinstance(result, ToolMessage)
assert result.tool_call_id == "call_save_999"
assert result.content == "Successfully saved note to state_note.md."
assert result.status == "success"
note_path = Path(tmp_path) / "notes" / "state_note.md"
assert note_path.exists()
+7 -17
View File
@@ -14,7 +14,6 @@
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
@@ -27,7 +26,7 @@ from artemis.tools.mobile.exec_tools import (
swipe,
swipe_wrapper,
)
from langgraph.types import Command
from langchain_core.messages import ToolMessage
import pytest
@@ -51,13 +50,7 @@ def mock_driver():
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
return MagicMock(spec=State)
def test_swipe_tool_subclass_and_registry():
@@ -131,7 +124,7 @@ async def test_swipe_direct_execution_with_ctx_coords(mock_ctx):
@pytest.mark.asyncio
async def test_swipe_with_state_command(mock_ctx, mock_state):
"""Verify SwipeTool returns Command when state is provided."""
"""Verify SwipeTool returns ToolMessage when state is provided."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.swipe_coords = AsyncMock(return_value=None)
@@ -143,13 +136,10 @@ async def test_swipe_with_state_command(mock_ctx, mock_state):
tool_call_id="call_sw_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_sw_1"
assert messages[0].status == "success"
assert "Swipe completed successfully." in messages[0].content
assert isinstance(cmd, ToolMessage)
assert cmd.tool_call_id == "call_sw_1"
assert cmd.status == "success"
assert "Swipe completed successfully." in cmd.content
@pytest.mark.asyncio
+8 -16
View File
@@ -13,9 +13,8 @@
# limitations under the License.
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
@@ -29,7 +28,7 @@ from artemis.tools.scratchpad import (
update_note_wrapper,
)
from artemis.utils.notes import save_note_content
from langgraph.types import Command
from langchain_core.messages import ToolMessage
import pytest
@@ -123,13 +122,11 @@ async def test_update_note_callable_execution(mock_ctx, tmp_path):
@pytest.mark.asyncio
async def test_update_note_with_state_command(mock_ctx, tmp_path):
"""Verify UpdateNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
async def test_update_note_with_state_tool_message(mock_ctx, tmp_path):
"""Verify UpdateNoteTool.execute with state returns a ToolMessage directly."""
save_note_content(tmp_path, "state_plan", "- [ ] Step 1")
mock_state = MagicMock(spec=State)
mock_state.current_agent = "operator"
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
result = await update_note.execute(
ctx=mock_ctx,
@@ -140,15 +137,10 @@ async def test_update_note_with_state_command(mock_ctx, tmp_path):
tool_call_id="call_update_999",
)
assert isinstance(result, Command)
assert VALIDATOR_MESSAGES_KEY in result.update
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
assert len(tool_messages) == 1
assert tool_messages[0].tool_call_id == "call_update_999"
assert tool_messages[0].content == "Successfully updated note 'state_plan'."
assert tool_messages[0].status == "success"
mock_state.asanitize_update.assert_called_once()
assert isinstance(result, ToolMessage)
assert result.tool_call_id == "call_update_999"
assert result.content == "Successfully updated note 'state_plan'."
assert result.status == "success"
note_path = Path(tmp_path) / "notes" / "state_plan.md"
assert note_path.read_text(encoding="utf-8") == "- [x] Step 1"
+11 -23
View File
@@ -14,7 +14,6 @@
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
@@ -27,7 +26,7 @@ from artemis.tools.video_tool import (
get_video_analyzer_tool_pure,
video_analyzer,
)
from langgraph.types import Command
from langchain_core.messages import ToolMessage
import pytest
@@ -38,14 +37,8 @@ def mock_ctx():
@pytest.fixture
def mock_state(mock_ctx):
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def mock_state():
return MagicMock(spec=State)
def test_video_analyzer_tool_subclass_and_registry():
@@ -104,7 +97,7 @@ async def test_video_analyzer_direct_execution_failed(mock_ctx):
@pytest.mark.asyncio
async def test_video_analyzer_with_state_command(mock_ctx, mock_state):
"""Verify execute returns a Command with ToolMessage when state is provided."""
"""Verify execute returns a ToolMessage when state is provided."""
with patch(
"artemis.tools.video_tool.VideoAnalyzer.run",
new_callable=AsyncMock,
@@ -117,13 +110,10 @@ async def test_video_analyzer_with_state_command(mock_ctx, mock_state):
tool_call_id="call_video_123",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_video_123"
assert messages[0].content == "Action verified in video"
assert messages[0].status == "success"
assert isinstance(cmd, ToolMessage)
assert cmd.tool_call_id == "call_video_123"
assert cmd.content == "Action verified in video"
assert cmd.status == "success"
@pytest.mark.asyncio
@@ -150,11 +140,9 @@ async def test_video_analyzer_exception_handling(mock_ctx, mock_state):
tool_call_id="call_err",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert messages[0].status == "error"
assert "Error running video analyzer" in messages[0].content
assert isinstance(cmd, ToolMessage)
assert cmd.status == "error"
assert "Error running video analyzer" in cmd.content
@pytest.mark.asyncio