feat: harden agent verification and runtime contracts

Introduce two-gate checker settlement, explicit graph visibility and checkpoints, structured LLM reliability, centralized action specifications, and corresponding tests and design documentation.
This commit is contained in:
somew1nd committed 2026-08-31 20:42:24 -07:00
1 parent ddfa420945
commit cbe8f4f0c6
106 files changed
+11542 -4445

No files matched your search

@@ -0,0 +1,176 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""§1.3 assembly discipline: prompts are functions of configuration x scenario.
Both check gates off => zero context pollution; a plan carrying check lines
gets the behavior explainer regardless of switches."""
import json
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from artemis.agents.operator.prompts import (
CheckItemsExplainerPromptComponent,
FeedbackPromptComponent,
PromptBuilder,
TemplatePromptComponent,
)
from artemis.agents.planner.planner import build_planner_system_blocks
from artemis.context import ArtemisContext, ExecutionSetup
from artemis.graph.state import State
PLANNER_JSON = Path(__file__).resolve().parents[3] / "artemis/agents/planner/planner.json"
OPERATOR_JSON = Path(__file__).resolve().parents[3] / "artemis/agents/operator/operator.json"
def _planner_data():
return json.loads(PLANNER_JSON.read_text(encoding="utf-8"))
def test_planner_blocks_without_checks_are_byte_identical_to_legacy():
data = _planner_data()
baseline = list(data["modes"]["initial_plan"]["system"])
assert build_planner_system_blocks(data, "initial_plan", include_checks=False) == baseline
# And the disabled assembly never references the check blocks
rendered = "\n\n".join(
data["blocks"][b]
for b in build_planner_system_blocks(data, "initial_plan", include_checks=False)
)
assert "- verify:" not in rendered
assert "- assert:" not in rendered
def test_planner_blocks_with_checks_mount_generation_and_audit():
data = _planner_data()
initial = build_planner_system_blocks(data, "initial_plan", include_checks=True)
assert initial[-1] == "check_generation"
validator = build_planner_system_blocks(data, "validator", include_checks=True)
assert validator[-1] == "check_audit"
generation = data["blocks"]["check_generation"]
# Restraint, honesty about post-hoc auditing, and no silent ordering promises
assert "restraint" in generation.lower() or "Do NOT attach" in generation
assert "post-hoc" in generation.lower()
assert "Never silently promise" in generation
audit = data["blocks"]["check_audit"]
assert "REJECT" in audit
def _mock_ctx(tmp_path=None, plan: str | None = None, **setup_kwargs):
ctx = MagicMock(spec=ArtemisContext)
ctx.execution_setup = ExecutionSetup(**setup_kwargs) if setup_kwargs is not None else None
if plan is not None and tmp_path is not None:
notes = tmp_path / "notes"
notes.mkdir(parents=True, exist_ok=True)
(notes / "task_plan.md").write_text(plan, encoding="utf-8")
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = tmp_path
else:
ctx.data_engine = None
return ctx
def _state(**overrides):
state = MagicMock(spec=State)
state.operator_feedback = None
for k, v in overrides.items():
setattr(state, k, v)
return state
@pytest.mark.asyncio
async def test_explainer_renders_iff_plan_has_check_lines(tmp_path):
component = CheckItemsExplainerPromptComponent()
# Plan without check lines -> nothing rendered
builder = PromptBuilder()
ctx = _mock_ctx(tmp_path, plan="- [ ] G\n")
await component(builder, _state(), ctx)
assert builder.human_parts == []
# Plan with check lines -> explainer rendered
builder2 = PromptBuilder()
ctx2 = _mock_ctx(tmp_path, plan="- [ ] G\n - verify: V\n")
await component(builder2, _state(), ctx2)
joined = "\n".join(p for p in builder2.human_parts if isinstance(p, str))
assert "check lines" in joined
assert "independent Checker" in joined
assert "automatically restored" in joined
@pytest.mark.asyncio
async def test_explainer_is_content_driven_not_switch_driven(tmp_path):
"""A resumed old plan with check lines still gets the explanation even when
every check switch is off."""
component = CheckItemsExplainerPromptComponent()
builder = PromptBuilder()
ctx = _mock_ctx(tmp_path, plan="- [ ] G\n - assert: A\n", disable_checker=True)
await component(builder, _state(), ctx)
joined = "\n".join(p for p in builder.human_parts if isinstance(p, str))
assert "check lines" in joined
async def _render_main_template(**setup_kwargs) -> str:
component = TemplatePromptComponent()
builder = PromptBuilder()
ctx = _mock_ctx(**setup_kwargs)
ctx.actuator = None
state = _state(initial_goal="G")
prompts = json.loads(OPERATOR_JSON.read_text(encoding="utf-8"))
await component(builder, state, ctx, prompts=prompts, plan_and_history="")
return "".join(builder.system_parts) + (builder.human_footer or "")
@pytest.mark.asyncio
async def test_main_template_diagnosis_trigger_gated_on_verification():
"""The rejection/finding diagnosis trigger is a dead instruction unless a
mechanism that can produce rejections or findings is active (§1.3)."""
# Factory default: checks off (master alias) AND planner validation off.
default = await _render_main_template(disable_checker=True, disable_planner_validation=True)
assert "Verification Finding" not in default
assert "Ambiguous Validation Rejection" not in default
# Checks on -> the trigger is real and must be documented.
with_checks = await _render_main_template(
disable_checker=False, disable_planner_validation=True
)
assert "Ambiguous Validation Rejection or Verification Finding" in with_checks
# Planner validation alone can also produce rejections.
with_validation = await _render_main_template(
disable_checker=True, disable_planner_validation=False
)
assert "Ambiguous Validation Rejection or Verification Finding" in with_validation
@pytest.mark.asyncio
async def test_check_feedback_component_appends_findings_only_when_present():
component = FeedbackPromptComponent()
builder = PromptBuilder()
await component(builder, _state(operator_feedback=None), _mock_ctx())
assert builder.human_parts == []
builder2 = PromptBuilder()
await component(
builder2,
_state(operator_feedback=["[verify failed] 'X': missing"]),
_mock_ctx(),
)
joined = "\n".join(p for p in builder2.human_parts if isinstance(p, str))
assert "Verification Findings" in joined
assert "[verify failed] 'X': missing" in joined
+235 -68
View File
@@ -14,85 +14,252 @@
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.checker.checker import CheckerResult, run_async_check
from artemis.context import ArtemisContext
import pytest
from artemis.agents.checker.checker import (
CheckReport,
CheckVerdict,
_normalize_report,
assemble_checker_prompt_segments,
build_checker_tools,
build_probe_argv,
run_checkpoint_check,
run_final_check,
verdicts_allow_release,
)
from artemis.context import ArtemisContext, ExecutionSetup
from artemis.graph.checkpoints import EvidenceAnchor
from artemis.utils.plan_grammar import CheckItem
def _ci(kind="verify", when="on_complete", text="expected state", parent="p"):
return CheckItem(kind=kind, when=when, text=text, parent_key=parent)
def _mock_ctx(**setup_kwargs):
ctx = MagicMock(spec=ArtemisContext)
ctx.execution_setup = ExecutionSetup(**setup_kwargs)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = "unused"
ctx.data_engine.get_agent_friendly_steps.return_value = []
ctx.data_engine.get_step_number.return_value = 7
return ctx
# --- probe_device: enumerated table, programmatic argv (§8 item 12) ------------------
def test_probe_argv_enumerated_kinds_allowed():
assert build_probe_argv("alarms") == ["dumpsys", "alarm"]
assert build_probe_argv("battery") == ["dumpsys", "battery"]
assert build_probe_argv("foreground") == ["dumpsys", "activity", "activities"]
assert build_probe_argv("packages") == ["pm", "list", "packages"]
assert build_probe_argv("setting", {"namespace": "system", "key": "screen_brightness"}) == [
"settings",
"get",
"system",
"screen_brightness",
]
assert build_probe_argv("content", {"uri": "content://settings/system"}) == [
"content",
"query",
"--uri",
"content://settings/system",
]
assert build_probe_argv("prop", {"key": "ro.build.version.sdk"}) == [
"getprop",
"ro.build.version.sdk",
]
def test_probe_argv_rejects_unknown_kind():
with pytest.raises(ValueError):
build_probe_argv("shell")
with pytest.raises(ValueError):
build_probe_argv("dumpsys")
def test_probe_argv_rejects_whitespace_and_metachars():
for bad in ("a key", "key;rm", "key|x", "key$", "key`x`", "key&&y", "a\nb"):
with pytest.raises(ValueError):
build_probe_argv("prop", {"key": bad})
with pytest.raises(ValueError):
build_probe_argv("setting", {"namespace": "system", "key": "a b"})
with pytest.raises(ValueError):
build_probe_argv("content", {"uri": "content://a; rm -rf /"})
def test_probe_argv_battery_set_inexpressible():
"""Mutating dumpsys subcommands cannot be expressed: no-parameter probes
reject every parameter."""
with pytest.raises(ValueError):
build_probe_argv("battery", {"extra": "set level 100"})
with pytest.raises(ValueError):
build_probe_argv("alarms", {"args": "anything"})
# And the setting namespace is a closed set
with pytest.raises(ValueError):
build_probe_argv("setting", {"namespace": "battery", "key": "level"})
# --- Release decision is node-side and never rewrites verdicts -----------------------
def test_verdicts_allow_release_semantics():
ok = CheckReport(
verdicts=[
CheckVerdict(item_text="a", kind="verify", status="passed", evidence="e"),
CheckVerdict(item_text="b", kind="verify", status="inconclusive", evidence="e"),
CheckVerdict(item_text="c", kind="assert", status="failed", evidence="e"),
]
)
# Assert failures never block release
assert verdicts_allow_release(ok)
blocked = CheckReport(
verdicts=[
CheckVerdict(item_text="a", kind="verify", status="failed", evidence="e"),
]
)
assert not verdicts_allow_release(blocked)
def test_normalize_report_downgrades_vague_failures_and_fills_missing():
items = [_ci(kind="verify", text="v1"), _ci(kind="assert", text="a1")]
report = CheckReport(
verdicts=[CheckVerdict(item_text="v1", kind="verify", status="failed", evidence=" ")]
)
normalized = _normalize_report(report, items)
by_text = {(v.kind, v.item_text): v for v in normalized.verdicts}
# Vague failed -> inconclusive (verdict value hygiene, not release logic)
assert by_text[("verify", "v1")].status == "inconclusive"
# Missing item gets an inconclusive verdict, never a silent pass
assert by_text[("assert", "a1")].status == "inconclusive"
# --- Assembly: prompt segments x entry x content x config (§8 item 18) ---------------
def test_prompt_segments_verify_only_has_no_assert_section():
prompts = {
"base_rules": "BASE",
"verify_semantics": "VERIFY-SEG",
"assert_semantics": "ASSERT-SEG",
"anchor_guide": "ANCHOR-SEG",
"final_guide": "FINAL-SEG",
"probe_guide": "PROBE-SEG",
}
text = assemble_checker_prompt_segments(
"checkpoint", [_ci(kind="verify")], probe_tool_registered=True, prompts=prompts
)
assert "VERIFY-SEG" in text
assert "ASSERT-SEG" not in text
assert "ANCHOR-SEG" in text
assert "FINAL-SEG" not in text
assert "PROBE-SEG" in text
text2 = assemble_checker_prompt_segments(
"final", [_ci(kind="assert")], probe_tool_registered=False, prompts=prompts
)
assert "ASSERT-SEG" in text2
assert "VERIFY-SEG" not in text2
assert "FINAL-SEG" in text2
assert "ANCHOR-SEG" not in text2
# Probe disabled -> zero mention of probing anywhere in the prompt
assert "PROBE-SEG" not in text2
def test_tool_table_probe_gate():
ctx = _mock_ctx(disable_device_probes=False)
names = {t.name for t in build_checker_tools(ctx, "checkpoint")}
assert "probe_device" in names
assert "read_note" in names
assert "get_step_detail" in names
assert "get_step_screenshot" in names
# Never any device action, note write, or sub-agent
assert not names & {
"save_note",
"update_note",
"append_note",
"click",
"swipe",
"ask_diagnoser",
"ask_explorer",
"run_adb_command",
}
ctx2 = _mock_ctx(disable_device_probes=True)
names2 = {t.name for t in build_checker_tools(ctx2, "checkpoint")}
assert "probe_device" not in names2
@pytest.mark.asyncio
async def test_run_async_check(tmp_path):
notes_dir = tmp_path / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
# Put a dummy task plan file
(notes_dir / "task_plan.md").write_text("Plan")
# Put a dummy verification_chat file
import hashlib
subgoal_text = "Open WhatsApp"
s_hash = hashlib.md5(subgoal_text.encode("utf-8")).hexdigest()
(notes_dir / f"verification_chat_{s_hash}.json").write_text("[]")
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = str(tmp_path)
mock_ctx.data_engine.get_agent_friendly_steps.return_value = []
mock_ctx.execution_setup = None
# Mock controller for screen data
mock_controller = MagicMock()
mock_controller.take_screenshot = AsyncMock(return_value="dummy_screenshot_base64")
from artemis.controllers.device_controller import ScreenDataResponse
mock_screen_data = ScreenDataResponse(
base64="dummy_screenshot_base64",
elements=[],
width=1080,
height=2400,
platform="android",
async def test_checkpoint_entry_never_touches_live_screen():
"""Evidence discipline: the checkpoint entry must not capture the current
screen — its evidence is anchored history plus persistent probes."""
ctx = _mock_ctx()
report = CheckReport(
verdicts=[CheckVerdict(item_text="x", kind="verify", status="passed", evidence="e")]
)
mock_controller.get_screen_data = AsyncMock(return_value=mock_screen_data)
# Mock LLM response with with_structured_output
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.content = "Task completed successfully."
mock_response.tool_calls = []
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
mock_llm.bind_tools.return_value = mock_llm
mock_structured_llm = MagicMock()
mock_structured_llm.ainvoke = AsyncMock(
return_value=CheckerResult(success=True, reason="Task completed successfully.")
)
mock_llm.with_structured_output.return_value = mock_structured_llm
structured = MagicMock()
structured.ainvoke = AsyncMock(return_value=report)
llm = MagicMock()
llm.with_structured_output.return_value = structured
response = MagicMock()
response.tool_calls = []
with (
patch("artemis.agents.checker.checker.get_llm", return_value=llm),
patch(
"artemis.agents.checker.checker.UnifiedMobileController",
return_value=mock_controller,
"artemis.agents.checker.checker.acomplete",
new=AsyncMock(return_value=response),
),
patch("artemis.agents.checker.checker.get_llm", return_value=mock_llm),
patch(
"artemis.utils.task_tree.build_plan_and_history",
return_value="Task tree",
),
"artemis.agents.checker.checker._capture_final_screen",
new=AsyncMock(return_value=(None, "SHOULD NOT BE CALLED")),
) as capture_mock,
):
result = await run_async_check(
mock_ctx,
subgoal_text=subgoal_text,
subgoal_hash=s_hash,
raw_perception_data={"screenshot_b64": "dummy", "width": 1080, "height": 2400},
latest_ui_hierarchy=[],
result = await run_checkpoint_check(
ctx,
check_items=[_ci(text="x")],
anchor=EvidenceAnchor(anchor_step_id="sid", trigger_ts=0.0, plan_text="- [x] G"),
goal="the goal",
subgoal_text="G",
)
assert result["status"] == "success"
assert result["reason"] == "Task completed successfully."
assert mock_llm.astream.called
assert mock_llm.with_structured_output.called
capture_mock.assert_not_awaited()
assert result.verdicts[0].status == "passed"
@pytest.mark.asyncio
async def test_final_entry_captures_live_screen():
ctx = _mock_ctx()
report = CheckReport(verdicts=[])
structured = MagicMock()
structured.ainvoke = AsyncMock(return_value=report)
llm = MagicMock()
llm.with_structured_output.return_value = structured
response = MagicMock()
response.tool_calls = []
with (
patch("artemis.agents.checker.checker.get_llm", return_value=llm),
patch(
"artemis.agents.checker.checker.acomplete",
new=AsyncMock(return_value=response),
),
patch(
"artemis.agents.checker.checker._capture_final_screen",
new=AsyncMock(return_value=("b64img", "elements")),
) as capture_mock,
):
await run_final_check(
ctx,
goal="the goal",
plan_text="- [x] G",
ledger=[],
check_items=[_ci(kind="assert", when="at_end", text="no crash", parent=None)],
)
capture_mock.assert_awaited_once()
+2 -15
View File
@@ -14,9 +14,7 @@
from unittest.mock import AsyncMock, Mock, patch
from langchain_core.messages import AIMessage, ToolMessage
from langgraph.types import Command
from artemis.agents.diagnoser.diagnoser import Diagnoser
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
import pytest
@@ -148,14 +146,9 @@ async def test_diagnostic_agent_tool_calls(
"purpose": None,
"state": None,
}
mock_command = Command(
update={
VALIDATOR_MESSAGES_KEY: [
ToolMessage(content="Video shows UI loading.", tool_call_id="call_2")
]
}
mock_video_tool.ainvoke.return_value = ToolMessage(
content="Video shows UI loading.", tool_call_id="call_2"
)
mock_video_tool.ainvoke.return_value = mock_command
mock_get_video_tool.return_value = mock_video_tool
agent = Diagnoser(mock_context)
@@ -273,12 +266,6 @@ async def test_diagnoser_background_job_lifecycle(
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock asanitize_update on mock_state
async def mock_asanitize_update(ctx, update, agent):
return update
mock_state.asanitize_update = mock_asanitize_update
# Instantiate the Diagnoser
agent = Diagnoser(ctx)
+31 -167
View File
@@ -30,10 +30,7 @@ async def test_operator_node_fast_path():
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_state.subgoal_plan = []
mock_state.initial_goal = "Test goal"
mock_state.operator_tactical_plan = []
# Mock controller and data
mock_controller = MagicMock()
@@ -62,18 +59,15 @@ async def test_operator_node_fast_path():
# Patch dependencies
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
await node(mock_state)
node_update = await node(mock_state)
# Verify state was updated with action
mock_state.asanitize_update.assert_called_once()
args, kwargs = mock_state.asanitize_update.call_args
update_dict = kwargs["update"]
update_dict = node_update
assert "structured_decisions" in update_dict
decisions = json.loads(update_dict["structured_decisions"])
assert decisions[0]["action"] == "tap"
assert decisions[0]["coordinates"] == [54, 120]
assert update_dict["complete_subgoals_by_ids"] == []
@pytest.mark.asyncio
@@ -241,8 +235,6 @@ async def test_operator_node_multiple_actions():
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_state.subgoal_plan = []
mock_state.initial_goal = "Test goal"
mock_controller = MagicMock()
@@ -267,11 +259,9 @@ async def test_operator_node_multiple_actions():
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
await node(mock_state)
node_update = await node(mock_state)
mock_state.asanitize_update.assert_called_once()
args, kwargs = mock_state.asanitize_update.call_args
update_dict = kwargs["update"]
update_dict = node_update
assert "structured_decisions" in update_dict
decisions = json.loads(update_dict["structured_decisions"])
@@ -296,7 +286,6 @@ async def test_operator_node_no_record_step():
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_state.initial_goal = "Test goal"
mock_controller = MagicMock()
@@ -314,13 +303,11 @@ async def test_operator_node_no_record_step():
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
await node(mock_state)
node_update = await node(mock_state)
mock_ctx.data_engine.record_step.assert_not_called()
mock_state.asanitize_update.assert_called_once()
args, kwargs = mock_state.asanitize_update.call_args
update_dict = kwargs["update"]
update_dict = node_update
assert "structured_decisions" in update_dict
@@ -336,7 +323,6 @@ async def test_operator_dynamic_prompt():
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.initial_goal = "Test goal"
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
class MockComponent(PromptComponent):
async def __call__(self, builder, state, ctx, **kwargs):
@@ -367,7 +353,7 @@ async def test_operator_dynamic_prompt():
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx, prompt_components=[MockComponent()])
await node(mock_state)
node_update = await node(mock_state)
@pytest.mark.asyncio
@@ -443,7 +429,6 @@ async def test_operator_history_compression():
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.initial_goal = "Test goal"
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_controller = MagicMock()
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=")
@@ -465,10 +450,6 @@ async def test_operator_history_compression():
"Goals: {{ initial_goal }}\nPlan:\n{{ plan_and_history"
" }}\n# CURRENT OBSERVATION"
),
"troubleshooter_template": (
"Goals: {{ initial_goal }}\nPlan:\n{{ plan_and_history"
" }}\n# CURRENT OBSERVATION"
),
}
)
elif "task_plan.md" in path_str:
@@ -508,7 +489,7 @@ async def test_operator_history_compression():
patch("pathlib.Path.read_text", mock_read_text),
):
node = OperatorNode(mock_ctx, last_n_detailed=5)
await node(mock_state)
node_update = await node(mock_state)
@pytest.mark.asyncio
@@ -528,7 +509,7 @@ async def test_operator_loop_detection():
node = OperatorNode(mock_ctx)
with pytest.raises(RuntimeError) as exc_info:
await node(mock_state)
node_update = await node(mock_state)
assert "Infinite loop detected" in str(exc_info.value)
@@ -546,7 +527,6 @@ async def test_operator_tracks_subagent_calls():
mock_state = MagicMock()
mock_state.subagent_calls = ["ask_explorer"]
mock_state.initial_goal = "Test goal"
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_controller = MagicMock()
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=")
@@ -589,11 +569,9 @@ async def test_operator_tracks_subagent_calls():
),
):
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool])
await node(mock_state)
node_update = await node(mock_state)
mock_state.asanitize_update.assert_called_once()
args, kwargs = mock_state.asanitize_update.call_args
update_dict = kwargs["update"]
update_dict = node_update
assert "subagent_calls" in update_dict
assert update_dict["subagent_calls"] == [
@@ -615,7 +593,6 @@ async def test_operator_defer_action_for_gathering():
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_state.initial_goal = "Test goal"
mock_controller = MagicMock()
@@ -684,15 +661,13 @@ async def test_operator_defer_action_for_gathering():
),
):
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool])
await node(mock_state)
node_update = await node(mock_state)
# Subagent tool should have been executed
mock_call_receiver.assert_called_once_with(query="check something", state=mock_state)
# Verify final state has the action execution from the second response
mock_state.asanitize_update.assert_called_once()
args, kwargs = mock_state.asanitize_update.call_args
update_dict = kwargs["update"]
update_dict = node_update
assert "structured_decisions" in update_dict
decisions = json.loads(update_dict["structured_decisions"])
@@ -701,100 +676,6 @@ async def test_operator_defer_action_for_gathering():
assert decisions[0]["coordinates"] == [54, 120]
@pytest.mark.asyncio
async def test_operator_defer_reply_to_checker_for_gathering():
from artemis.agents.operator.operator import OperatorNode
from artemis.context import ArtemisContext
from unittest.mock import AsyncMock, MagicMock, patch
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.execution_setup = None
from pathlib import Path
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = Path("/tmp/test_session")
mock_ctx.data_engine.get_agent_friendly_steps.return_value = []
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_state.initial_goal = "Test goal"
mock_llm = MagicMock()
# First response calls ask_diagnoser and reply_to_checker
mock_response_1 = MagicMock()
mock_response_1.tool_calls = [
{"name": "ask_diagnoser", "args": {"query": "test"}, "id": "call_diag"},
{
"name": "reply_to_checker",
"args": {"reasoning": "I solved it"},
"id": "call_reply",
},
]
# Second response stops without action tools
mock_response_2 = MagicMock()
mock_response_2.tool_calls = []
responses = [mock_response_1, mock_response_2]
call_count = 0
async def mock_ainvoke(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
res = responses[call_count]
call_count += 1
return res
mock_llm.ainvoke.side_effect = mock_ainvoke
mock_llm.bind_tools.return_value = mock_llm
from langgraph.prebuilt import InjectedState
from typing import Annotated
mock_diagnoser_tool = MagicMock()
mock_diagnoser_tool.name = "ask_diagnoser"
mock_diagnoser_tool.args = {"state": None}
mock_call_receiver = Mock()
async def dummy_diagnoser_func(query: str, state: Annotated[MagicMock, InjectedState]):
mock_call_receiver(query=query, state=state)
return "Diagnostics completed"
mock_diagnoser_tool.coroutine = dummy_diagnoser_func
mock_diagnoser_tool.func = None
with (
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
patch(
"artemis.agents.operator.operator.trace_langchain_tool",
side_effect=lambda t, ctx: t,
),
patch(
"artemis.agents.operator.operator.OperatorNode._has_checker_feedback",
return_value=True,
),
patch(
"artemis.agents.operator.operator.OperatorNode._get_verification_chat_rounds",
return_value=(1, 1),
),
):
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool])
await node(mock_state)
# Subagent was called
mock_call_receiver.assert_called_once()
# Verify reply to checker was NOT executed / flagged
mock_state.asanitize_update.assert_called_once()
args, kwargs = mock_state.asanitize_update.call_args
update_dict = kwargs["update"]
assert update_dict.get("operator_replied") is False
@pytest.mark.asyncio
async def test_wait_actions_translation():
from artemis.agents.operator.operator import OperatorNode
@@ -907,7 +788,6 @@ async def test_operator_no_defer_for_note_updating():
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_state.initial_goal = "Test goal"
mock_controller = MagicMock()
@@ -965,15 +845,13 @@ async def test_operator_no_defer_for_note_updating():
),
):
node = OperatorNode(mock_ctx, tools=[mock_update_note_tool])
await node(mock_state)
node_update = await node(mock_state)
# The update_note tool should have been executed immediately
mock_call_receiver.assert_called_once_with(key="progress", content="step 1")
# The click action should NOT be deferred, so it should be in structured_decisions
mock_state.asanitize_update.assert_called_once()
args, kwargs = mock_state.asanitize_update.call_args
update_dict = kwargs["update"]
update_dict = node_update
assert "structured_decisions" in update_dict
decisions = json.loads(update_dict["structured_decisions"])
@@ -995,7 +873,6 @@ async def test_operator_defer_for_note_reading():
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_state.initial_goal = "Test goal"
mock_controller = MagicMock()
@@ -1058,7 +935,7 @@ async def test_operator_defer_for_note_reading():
),
):
node = OperatorNode(mock_ctx, tools=[mock_read_note_tool])
await node(mock_state)
node_update = await node(mock_state)
# The read_note tool should have been executed
mock_call_receiver.assert_called_once_with(key="progress")
@@ -1068,9 +945,7 @@ async def test_operator_defer_for_note_reading():
# Total call_count should be 2.
assert call_count == 2
mock_state.asanitize_update.assert_called_once()
args, kwargs = mock_state.asanitize_update.call_args
update_dict = kwargs["update"]
update_dict = node_update
assert "structured_decisions" in update_dict
decisions = json.loads(update_dict["structured_decisions"])
@@ -1092,7 +967,6 @@ async def test_operator_no_defer_for_note_appending():
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_state.initial_goal = "Test goal"
mock_controller = MagicMock()
@@ -1149,15 +1023,13 @@ async def test_operator_no_defer_for_note_appending():
),
):
node = OperatorNode(mock_ctx, tools=[mock_append_note_tool])
await node(mock_state)
node_update = await node(mock_state)
# The append_note tool should have been executed immediately
mock_call_receiver.assert_called_once_with(key="progress", content="step 1")
# The click action should NOT be deferred, so it should be in structured_decisions
mock_state.asanitize_update.assert_called_once()
args, kwargs = mock_state.asanitize_update.call_args
update_dict = kwargs["update"]
update_dict = node_update
assert "structured_decisions" in update_dict
decisions = json.loads(update_dict["structured_decisions"])
@@ -1180,7 +1052,6 @@ async def test_operator_background_tasks_prompt_injection():
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.initial_goal = "Test goal"
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_controller = MagicMock()
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=")
@@ -1231,7 +1102,7 @@ async def test_operator_background_tasks_prompt_injection():
active_background_tasks=active_background_tasks,
)
captured_messages.extend(messages)
return [], False, None, None, False
return [], None, None, False
with (
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
@@ -1241,7 +1112,7 @@ async def test_operator_background_tasks_prompt_injection():
),
):
node = OperatorNode(mock_ctx)
await node(mock_state)
node_update = await node(mock_state)
# Assert that the captured messages (prompts) contain the background tasks section
full_content = ""
@@ -1276,7 +1147,6 @@ async def test_operator_task_plan_warning_prompt_injection():
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.initial_goal = "Test goal"
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_controller = MagicMock()
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=")
@@ -1323,7 +1193,7 @@ async def test_operator_task_plan_warning_prompt_injection():
task_plan="",
)
captured_messages_1.extend(messages)
return [], False, None, None, False
return [], None, None, False
with (
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
@@ -1333,7 +1203,7 @@ async def test_operator_task_plan_warning_prompt_injection():
),
):
node = OperatorNode(mock_ctx)
await node(mock_state)
node_update = await node(mock_state)
full_content_1 = ""
for m in captured_messages_1:
@@ -1376,7 +1246,7 @@ async def test_operator_task_plan_warning_prompt_injection():
task_plan="",
)
captured_messages_2.extend(messages)
return [], False, None, None, False
return [], None, None, False
with (
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
@@ -1386,7 +1256,7 @@ async def test_operator_task_plan_warning_prompt_injection():
),
):
node = OperatorNode(mock_ctx)
await node(mock_state)
node_update = await node(mock_state)
full_content_2 = ""
for m in captured_messages_2:
@@ -1418,7 +1288,6 @@ async def test_operator_tool_limit_exceeded_warning():
# Set the flag to True to simulate tool call limit exceeded in previous turn
mock_state.operator_tool_limit_exceeded = True
mock_state.initial_goal = "Test goal"
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_controller = MagicMock()
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=")
@@ -1443,8 +1312,8 @@ async def test_operator_tool_limit_exceeded_warning():
task_plan="",
)
captured_messages.extend(messages)
# Return 5 elements: action_result, requested_argue, raw_thinking, native_thinking, tool_limit_exceeded
return [], False, None, None, False
# Return 4 elements: action_result, raw_thinking, native_thinking, tool_limit_exceeded
return [], None, None, False
with (
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
@@ -1454,7 +1323,7 @@ async def test_operator_tool_limit_exceeded_warning():
),
):
node = OperatorNode(mock_ctx)
await node(mock_state)
node_update = await node(mock_state)
full_content = ""
for m in captured_messages:
@@ -1469,9 +1338,7 @@ async def test_operator_tool_limit_exceeded_warning():
"You did not execute any screen interaction actions in your last turn" in full_content
)
mock_state.asanitize_update.assert_called_once()
_, kwargs = mock_state.asanitize_update.call_args
assert kwargs["update"].get("operator_tool_limit_exceeded") is False
assert node_update.get("operator_tool_limit_exceeded") is False
@pytest.mark.asyncio
@@ -1488,7 +1355,6 @@ async def test_operator_fallback_function_call_parsing():
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_state.initial_goal = "Test goal"
mock_controller = MagicMock()
@@ -1515,12 +1381,10 @@ async def test_operator_fallback_function_call_parsing():
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
await node(mock_state)
node_update = await node(mock_state)
# Verify state was updated with parsed click action
mock_state.asanitize_update.assert_called_once()
_, kwargs = mock_state.asanitize_update.call_args
update_dict = kwargs["update"]
update_dict = node_update
assert "structured_decisions" in update_dict
decisions = json.loads(update_dict["structured_decisions"])
+8 -9
View File
@@ -29,15 +29,14 @@ def test_operator_prompt_contract_matches_note_runtime_semantics():
assert "memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`" not in prompt
def test_troubleshooter_prompt_has_mutually_exclusive_recovery_branches():
prompt = apply_operator_prompt_contract(load_operator_prompts()["troubleshooter_template"])
assert "invoke `reply_to_checker` directly" in prompt
assert "This branch takes precedence over diagnosis" in prompt
assert "perform that correction directly" in prompt
assert "also takes precedence over generic diagnosis triggers" in prompt
assert "A Checker rejection alone is not a mandatory diagnosis trigger" in prompt
assert "**Validation/Checker Rejection**" not in prompt
def test_operator_prompt_is_single_template_without_checker_dialogue():
"""The prompt is never switched by verification results: main_template is
the only template, and no checker-dialogue machinery is advertised."""
prompts = load_operator_prompts()
assert set(prompts) == {"main_template"}
prompt = apply_operator_prompt_contract(prompts["main_template"])
assert "reply_to_checker" not in prompt
assert "Checker" not in prompt
def test_operator_prompt_omits_environment_trust_and_explorer_directives():
@@ -12,10 +12,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock
from artemis.graph.graph import check_plan_mutation_rejections
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from artemis.context import ArtemisContext, ExecutionSetup
from artemis.graph.graph import (
check_plan_mutation_rejections,
check_unintended_rewrite,
wrap_note_tool,
)
from artemis.graph.state import State
CONTINUOUS_PLAN = (
"- [x] Open app\n"
"- [/] [Loop:continuous] Periodically monitor for new emails "
"(Interval: every 5 minutes)\n"
" - [x] Polling Check #1: Done"
)
def test_check_plan_mutation_incomplete_nested():
content_before = "- [/] Main task\n - [ ] Subtask 1"
@@ -26,57 +41,52 @@ def test_check_plan_mutation_incomplete_nested():
def test_check_plan_mutation_delete_continuous_loop():
content_before = (
"- [x] Open app\n"
"- [/] [Loop] Periodically monitor for new emails "
"(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n"
" - [x] Polling Check #1: Done"
)
content_after = (
"- [x] Open app\n"
"- [x] Monitor for new emails\n"
" - [x] Baseline established\n"
" - [x] Polling Check #1: Done"
)
res = check_plan_mutation_rejections(content_before, content_after)
res = check_plan_mutation_rejections(CONTINUOUS_PLAN, content_after)
assert res is not None
assert "cannot delete an active [Loop]" in res
assert "cannot delete an active [Loop:continuous]" in res
def test_check_plan_mutation_mark_continuous_loop_completed():
content_before = (
"- [x] Open app\n"
"- [/] [Loop] Periodically monitor for new emails "
"(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n"
" - [x] Polling Check #1: Done"
)
content_after = (
"- [x] Open app\n"
"- [x] [Loop] Periodically monitor for new emails "
"(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n"
" - [x] Polling Check #1: Done"
)
res = check_plan_mutation_rejections(content_before, content_after)
content_after = CONTINUOUS_PLAN.replace("- [/] [Loop:continuous]", "- [x] [Loop:continuous]")
res = check_plan_mutation_rejections(CONTINUOUS_PLAN, content_after)
assert res is not None
assert "cannot be unilaterally marked as completed [x]" in res
assert "cannot be" in res and "marked as completed [x]" in res
def test_check_plan_mutation_user_stopped_allowed():
content_after = CONTINUOUS_PLAN.replace("- [/] [Loop:continuous]", "- [x] [Loop:continuous]")
state = MagicMock(spec=State)
state.user_stop_requested = True
res = check_plan_mutation_rejections(CONTINUOUS_PLAN, content_after, state=state)
assert res is None
def test_check_plan_mutation_stop_wording_is_not_a_signal():
"""Natural-language 'stop' phrasing must NOT unlock the continuous loop —
only the explicit release_loop signal (user_stop_requested) does."""
content_after = CONTINUOUS_PLAN.replace("- [/] [Loop:continuous]", "- [x] [Loop:continuous]")
state = MagicMock(spec=State)
state.user_stop_requested = False
state.injected_instruction = "Please stop the task now."
res = check_plan_mutation_rejections(CONTINUOUS_PLAN, content_after, state=state)
assert res is not None
def test_check_plan_mutation_bounded_loop_completion_allowed():
"""Plain [Loop] milestones are bounded: the model may complete them."""
content_before = (
"- [x] Open app\n"
"- [/] [Loop] Periodically monitor for new emails "
"(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n"
" - [x] Polling Check #1: Done"
"- [/] [Loop] Inspect candidate emails (Exit: all 5 candidates inspected)\n"
" - [x] Candidate #5: Done"
)
content_after = (
"- [x] Open app\n"
"- [x] [Loop] Periodically monitor for new emails "
"(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n"
" - [x] Polling Check #1: Done"
)
state = MagicMock(spec=State)
state.injected_instruction = "Please stop the task now."
res = check_plan_mutation_rejections(content_before, content_after, state=state)
content_after = content_before.replace("- [/] [Loop]", "- [x] [Loop]")
res = check_plan_mutation_rejections(content_before, content_after)
assert res is None
@@ -85,3 +95,209 @@ def test_check_plan_mutation_normal_bounded_task():
content_after = "- [x] Open Settings\n- [x] Toggle WiFi"
res = check_plan_mutation_rejections(content_before, content_after)
assert res is None
def test_check_unintended_rewrite_detects_hand_slip():
content_before = "- [x] Open Settings app\n- [/] Toggle the WiFi switch"
# Full rewrite: milestone 1's status unchanged but text drifted
content_after = "- [x] Open the Settings application\n- [/] Toggle the WiFi switch"
res = check_unintended_rewrite(content_before, content_after)
assert res is not None
assert "Open Settings app" in res
assert "update_note" in res
def test_check_unintended_rewrite_allows_status_flips_and_appends():
content_before = "- [x] Open Settings\n- [/] Toggle WiFi"
content_after = "- [x] Open Settings\n- [x] Toggle WiFi\n - [x] Confirmed enabled"
assert check_unintended_rewrite(content_before, content_after) is None
def test_check_unintended_rewrite_ignores_structural_replans():
# Milestone count changes are declared replans, judged by planner validation
content_before = "- [x] Open Settings\n- [/] Toggle WiFi"
content_after = "- [x] Open Settings\n- [/] Toggle WiFi\n- [ ] Verify connectivity"
assert check_unintended_rewrite(content_before, content_after) is None
# --- §4.3 deterministic check-line guard (independent of planner validation) ---------
GUARDED_PLAN = (
"- [/] Create the alarm\n"
" - verify: alarm exists in the list\n"
" - assert: a toast appeared\n"
"- [ ] Next milestone\n"
)
def _guard_ctx(tmp_path, **setup_kwargs):
# Planner validation OFF by default: the guard must hold on its own.
setup_kwargs.setdefault("disable_planner_validation", True)
setup_kwargs.setdefault("disable_checker", False)
ctx = MagicMock(spec=ArtemisContext)
ctx.execution_setup = ExecutionSetup(**setup_kwargs)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
ctx.planner_task = None
ctx.last_validated_plan = None
ctx.pending_validated_plan = None
ctx.task_plan_content_before = None
ctx.pending_checkpoints = []
ctx.checkpoint_tasks = {}
return ctx
def _guard_plan_path(tmp_path, content=GUARDED_PLAN):
notes_dir = tmp_path / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
path = notes_dir / "task_plan.md"
path.write_text(content, encoding="utf-8")
return path
def _save_invoke(path):
async def fake(tool, args, tool_call_id, state, record_trace=None):
path.write_text(args["content"], encoding="utf-8")
return "Success"
return fake
def _save_tool():
tool = MagicMock()
tool.name = "save_note"
tool.description = "save_note tool"
return tool
@pytest.mark.asyncio
async def test_operator_deleting_check_line_is_restored(tmp_path):
path = _guard_plan_path(tmp_path)
ctx = _guard_ctx(tmp_path)
rewritten = "- [/] Create the alarm\n - assert: a toast appeared\n- [ ] Next milestone\n"
with patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=_save_invoke(path),
):
wrapped = wrap_note_tool(ctx, _save_tool())
await wrapped.ainvoke({"key": "task_plan", "content": rewritten, "tool_call_id": "t"})
content = path.read_text(encoding="utf-8")
assert "- verify: alarm exists in the list" in content
assert "- assert: a toast appeared" in content
# No planner validation was needed or spawned (guard is deterministic)
assert ctx.planner_task is None
@pytest.mark.asyncio
async def test_operator_rewording_check_line_restores_original_keeps_new(tmp_path):
path = _guard_plan_path(tmp_path)
ctx = _guard_ctx(tmp_path)
rewritten = GUARDED_PLAN.replace(
" - verify: alarm exists in the list\n",
" - verify: alarm roughly configured\n",
)
with patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=_save_invoke(path),
):
wrapped = wrap_note_tool(ctx, _save_tool())
await wrapped.ainvoke({"key": "task_plan", "content": rewritten, "tool_call_id": "t"})
content = path.read_text(encoding="utf-8")
# Original standard restored; the Operator's new line survives as an addition
assert "- verify: alarm exists in the list" in content
assert "- verify: alarm roughly configured" in content
@pytest.mark.asyncio
async def test_deleted_parent_turns_check_into_task_level_at_end(tmp_path):
path = _guard_plan_path(tmp_path)
ctx = _guard_ctx(tmp_path)
# Whole parent subgoal removed
rewritten = "- [ ] Next milestone\n"
with patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=_save_invoke(path),
):
wrapped = wrap_note_tool(ctx, _save_tool())
await wrapped.ainvoke({"key": "task_plan", "content": rewritten, "tool_call_id": "t"})
content = path.read_text(encoding="utf-8")
assert "- verify@end: alarm exists in the list" in content
assert "- assert@end: a toast appeared" in content
@pytest.mark.asyncio
async def test_operator_adding_check_lines_is_allowed(tmp_path):
path = _guard_plan_path(tmp_path)
ctx = _guard_ctx(tmp_path)
extended = GUARDED_PLAN + " - verify: extra criterion\n"
with patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=_save_invoke(path),
):
wrapped = wrap_note_tool(ctx, _save_tool())
result = await wrapped.ainvoke(
{"key": "task_plan", "content": extended, "tool_call_id": "t"}
)
assert result == "Success"
assert "- verify: extra criterion" in path.read_text(encoding="utf-8")
@pytest.mark.asyncio
async def test_guard_holds_with_planner_validation_disabled_and_checks_off(tmp_path):
"""The guard is content-driven: it protects check lines even when every
check switch AND planner validation are disabled."""
path = _guard_plan_path(tmp_path)
ctx = _guard_ctx(tmp_path, disable_checker=True, disable_planner_validation=True)
with patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=_save_invoke(path),
):
wrapped = wrap_note_tool(ctx, _save_tool())
await wrapped.ainvoke(
{
"key": "task_plan",
"content": "- [/] Create the alarm\n- [ ] Next milestone\n",
"tool_call_id": "t",
}
)
content = path.read_text(encoding="utf-8")
assert "- verify: alarm exists in the list" in content
assert "- assert: a toast appeared" in content
@pytest.mark.asyncio
async def test_planner_node_writes_are_not_guarded():
"""The Planner keeps revision authority over check standards: its node is
wired with the UNWRAPPED note tools, so the guard never applies to it."""
from artemis.context import DeviceContext, DevicePlatform
from artemis.graph.graph import get_graph
device = DeviceContext(
host_platform="LINUX",
mobile_platform=DevicePlatform.ANDROID,
device_id="dummy",
device_width=1080,
device_height=2400,
)
ctx = ArtemisContext(device=device, execution_setup=ExecutionSetup())
graph = await get_graph(ctx)
planner_tools = {t.name: t for t in graph.nodes["planner"].bound.afunc.tools}
operator_tools = {t.name: t for t in graph.nodes["operator"].bound.afunc.tools}
# Operator's plan-writing tools go through the guard wrapper...
assert operator_tools["save_note"].coroutine.__name__ == "wrapped_note_tool"
assert operator_tools["update_note"].coroutine.__name__ == "wrapped_update_note"
# ...the Planner's do not.
assert getattr(planner_tools["save_note"].coroutine, "__name__", "") != "wrapped_note_tool"
assert getattr(planner_tools["update_note"].coroutine, "__name__", "") != "wrapped_update_note"
-3
View File
@@ -32,9 +32,6 @@ class DummyState:
self.operator_raw_data = operator_raw_data
self.operator_replan_reason = operator_replan_reason
async def asanitize_update(self, ctx, update, agent):
return update
@pytest.fixture
def mock_context():
-112
View File
@@ -1,112 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.operator.operator import OperatorNode
from artemis.context import ArtemisContext
from artemis.graph.graph import execution_check_node
from artemis.graph.state import State
import pytest
@pytest.mark.asyncio
async def test_snapshot_rollback_flow(tmp_path):
# Setup dummy directories
base_dir = tmp_path / "trace"
notes_dir = base_dir / "notes"
notes_dir.mkdir(parents=True)
task_plan_path = notes_dir / "task_plan.md"
task_plan_path.write_text("Initial Plan", encoding="utf-8")
# Mock Context
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = base_dir
mock_ctx.execution_setup = None
# Mock State
mock_state = MagicMock(spec=State)
mock_state.subagent_calls = []
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_state.initial_goal = "Test goal"
mock_state.short_term_memory = ""
mock_state.operator_raw_data = {
"screenshot_b64": "YmFzZTY0",
"xml_hierarchy": "<hierarchy></hierarchy>",
"ocr_results": [],
}
mock_state.current_step_id = "test-step-id"
# Mock Controller and LLM to let Operator run minimally
mock_controller = MagicMock()
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0")
mock_device_data = MagicMock()
mock_device_data.elements = []
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.tool_calls = []
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
mock_llm.bind_tools.return_value = mock_llm
# 1. Run Operator to trigger snapshot
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
await node(mock_state)
# Verify snapshot was created
snapshot_dir = notes_dir.with_name("notes_snapshot")
assert snapshot_dir.exists()
assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "Initial Plan"
assert mock_ctx.task_plan_snapshot == snapshot_dir
# 2. Simulate Operator optimistic modification
task_plan_path.write_text("Modified Plan", encoding="utf-8")
# 3. Run execution_check_node with failure to trigger rollback
future = asyncio.Future()
future.set_result({"status": "failed"})
mock_ctx.checker_task = future
await execution_check_node(mock_state, mock_ctx)
# Verify rollback restored the file
assert task_plan_path.read_text(encoding="utf-8") == "Initial Plan"
# 4. Run Operator again to create new snapshot (simulate next attempt)
# First modify it again to simulate operator working on restored plan
task_plan_path.write_text("Modified Plan 2", encoding="utf-8")
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
await node(mock_state)
assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "Modified Plan 2"
# 5. Run execution_check_node with success to trigger commit (cleanup)
future_success = asyncio.Future()
future_success.set_result({"status": "success"})
mock_ctx.checker_task = future_success
await execution_check_node(mock_state, mock_ctx)
# Verify snapshot was deleted
assert not snapshot_dir.exists()
+1 -5
View File
@@ -37,18 +37,13 @@ class DummyState:
def __init__(
self,
structured_decisions,
validator_messages=[],
current_step_id=None,
latest_screenshot=None,
):
self.structured_decisions = structured_decisions
self.validator_messages = validator_messages
self.current_step_id = current_step_id
self.latest_screenshot = latest_screenshot
async def asanitize_update(self, ctx, update, agent):
return update
class FakeActionSession:
"""Test double for artemis.mcp.action_session.ActionSession."""
@@ -146,6 +141,7 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
# entirely. Its hazard class (leaked AnyIO cancel scopes) is now pinned by
# tests/unit/mcp/test_action_session.py against the in-process ActionSession.
@pytest.mark.asyncio
async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode triggers failure analysis on error and handles