feat: harden agent verification and runtime contracts
Introduce two-gate checker settlement, explicit graph visibility and checkpoints, structured LLM reliability, centralized action specifications, and corresponding tests and design documentation.
This commit is contained in:
1 parent
ddfa420945
commit
cbe8f4f0c6
106 files changed
+11542
-4445
No files matched your search
@@ -0,0 +1,176 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""§1.3 assembly discipline: prompts are functions of configuration x scenario.
|
||||
Both check gates off => zero context pollution; a plan carrying check lines
|
||||
gets the behavior explainer regardless of switches."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.agents.operator.prompts import (
|
||||
CheckItemsExplainerPromptComponent,
|
||||
FeedbackPromptComponent,
|
||||
PromptBuilder,
|
||||
TemplatePromptComponent,
|
||||
)
|
||||
from artemis.agents.planner.planner import build_planner_system_blocks
|
||||
from artemis.context import ArtemisContext, ExecutionSetup
|
||||
from artemis.graph.state import State
|
||||
|
||||
PLANNER_JSON = Path(__file__).resolve().parents[3] / "artemis/agents/planner/planner.json"
|
||||
OPERATOR_JSON = Path(__file__).resolve().parents[3] / "artemis/agents/operator/operator.json"
|
||||
|
||||
|
||||
def _planner_data():
|
||||
return json.loads(PLANNER_JSON.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def test_planner_blocks_without_checks_are_byte_identical_to_legacy():
|
||||
data = _planner_data()
|
||||
baseline = list(data["modes"]["initial_plan"]["system"])
|
||||
assert build_planner_system_blocks(data, "initial_plan", include_checks=False) == baseline
|
||||
# And the disabled assembly never references the check blocks
|
||||
rendered = "\n\n".join(
|
||||
data["blocks"][b]
|
||||
for b in build_planner_system_blocks(data, "initial_plan", include_checks=False)
|
||||
)
|
||||
assert "- verify:" not in rendered
|
||||
assert "- assert:" not in rendered
|
||||
|
||||
|
||||
def test_planner_blocks_with_checks_mount_generation_and_audit():
|
||||
data = _planner_data()
|
||||
initial = build_planner_system_blocks(data, "initial_plan", include_checks=True)
|
||||
assert initial[-1] == "check_generation"
|
||||
validator = build_planner_system_blocks(data, "validator", include_checks=True)
|
||||
assert validator[-1] == "check_audit"
|
||||
|
||||
generation = data["blocks"]["check_generation"]
|
||||
# Restraint, honesty about post-hoc auditing, and no silent ordering promises
|
||||
assert "restraint" in generation.lower() or "Do NOT attach" in generation
|
||||
assert "post-hoc" in generation.lower()
|
||||
assert "Never silently promise" in generation
|
||||
|
||||
audit = data["blocks"]["check_audit"]
|
||||
assert "REJECT" in audit
|
||||
|
||||
|
||||
def _mock_ctx(tmp_path=None, plan: str | None = None, **setup_kwargs):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.execution_setup = ExecutionSetup(**setup_kwargs) if setup_kwargs is not None else None
|
||||
if plan is not None and tmp_path is not None:
|
||||
notes = tmp_path / "notes"
|
||||
notes.mkdir(parents=True, exist_ok=True)
|
||||
(notes / "task_plan.md").write_text(plan, encoding="utf-8")
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = tmp_path
|
||||
else:
|
||||
ctx.data_engine = None
|
||||
return ctx
|
||||
|
||||
|
||||
def _state(**overrides):
|
||||
state = MagicMock(spec=State)
|
||||
state.operator_feedback = None
|
||||
for k, v in overrides.items():
|
||||
setattr(state, k, v)
|
||||
return state
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explainer_renders_iff_plan_has_check_lines(tmp_path):
|
||||
component = CheckItemsExplainerPromptComponent()
|
||||
|
||||
# Plan without check lines -> nothing rendered
|
||||
builder = PromptBuilder()
|
||||
ctx = _mock_ctx(tmp_path, plan="- [ ] G\n")
|
||||
await component(builder, _state(), ctx)
|
||||
assert builder.human_parts == []
|
||||
|
||||
# Plan with check lines -> explainer rendered
|
||||
builder2 = PromptBuilder()
|
||||
ctx2 = _mock_ctx(tmp_path, plan="- [ ] G\n - verify: V\n")
|
||||
await component(builder2, _state(), ctx2)
|
||||
joined = "\n".join(p for p in builder2.human_parts if isinstance(p, str))
|
||||
assert "check lines" in joined
|
||||
assert "independent Checker" in joined
|
||||
assert "automatically restored" in joined
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_explainer_is_content_driven_not_switch_driven(tmp_path):
|
||||
"""A resumed old plan with check lines still gets the explanation even when
|
||||
every check switch is off."""
|
||||
component = CheckItemsExplainerPromptComponent()
|
||||
builder = PromptBuilder()
|
||||
ctx = _mock_ctx(tmp_path, plan="- [ ] G\n - assert: A\n", disable_checker=True)
|
||||
await component(builder, _state(), ctx)
|
||||
joined = "\n".join(p for p in builder.human_parts if isinstance(p, str))
|
||||
assert "check lines" in joined
|
||||
|
||||
|
||||
async def _render_main_template(**setup_kwargs) -> str:
|
||||
component = TemplatePromptComponent()
|
||||
builder = PromptBuilder()
|
||||
ctx = _mock_ctx(**setup_kwargs)
|
||||
ctx.actuator = None
|
||||
state = _state(initial_goal="G")
|
||||
prompts = json.loads(OPERATOR_JSON.read_text(encoding="utf-8"))
|
||||
await component(builder, state, ctx, prompts=prompts, plan_and_history="")
|
||||
return "".join(builder.system_parts) + (builder.human_footer or "")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_main_template_diagnosis_trigger_gated_on_verification():
|
||||
"""The rejection/finding diagnosis trigger is a dead instruction unless a
|
||||
mechanism that can produce rejections or findings is active (§1.3)."""
|
||||
# Factory default: checks off (master alias) AND planner validation off.
|
||||
default = await _render_main_template(disable_checker=True, disable_planner_validation=True)
|
||||
assert "Verification Finding" not in default
|
||||
assert "Ambiguous Validation Rejection" not in default
|
||||
|
||||
# Checks on -> the trigger is real and must be documented.
|
||||
with_checks = await _render_main_template(
|
||||
disable_checker=False, disable_planner_validation=True
|
||||
)
|
||||
assert "Ambiguous Validation Rejection or Verification Finding" in with_checks
|
||||
|
||||
# Planner validation alone can also produce rejections.
|
||||
with_validation = await _render_main_template(
|
||||
disable_checker=True, disable_planner_validation=False
|
||||
)
|
||||
assert "Ambiguous Validation Rejection or Verification Finding" in with_validation
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_feedback_component_appends_findings_only_when_present():
|
||||
component = FeedbackPromptComponent()
|
||||
|
||||
builder = PromptBuilder()
|
||||
await component(builder, _state(operator_feedback=None), _mock_ctx())
|
||||
assert builder.human_parts == []
|
||||
|
||||
builder2 = PromptBuilder()
|
||||
await component(
|
||||
builder2,
|
||||
_state(operator_feedback=["[verify failed] 'X': missing"]),
|
||||
_mock_ctx(),
|
||||
)
|
||||
joined = "\n".join(p for p in builder2.human_parts if isinstance(p, str))
|
||||
assert "Verification Findings" in joined
|
||||
assert "[verify failed] 'X': missing" in joined
|
||||
@@ -14,85 +14,252 @@
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.agents.checker.checker import CheckerResult, run_async_check
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
|
||||
from artemis.agents.checker.checker import (
|
||||
CheckReport,
|
||||
CheckVerdict,
|
||||
_normalize_report,
|
||||
assemble_checker_prompt_segments,
|
||||
build_checker_tools,
|
||||
build_probe_argv,
|
||||
run_checkpoint_check,
|
||||
run_final_check,
|
||||
verdicts_allow_release,
|
||||
)
|
||||
from artemis.context import ArtemisContext, ExecutionSetup
|
||||
from artemis.graph.checkpoints import EvidenceAnchor
|
||||
from artemis.utils.plan_grammar import CheckItem
|
||||
|
||||
|
||||
def _ci(kind="verify", when="on_complete", text="expected state", parent="p"):
|
||||
return CheckItem(kind=kind, when=when, text=text, parent_key=parent)
|
||||
|
||||
|
||||
def _mock_ctx(**setup_kwargs):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.execution_setup = ExecutionSetup(**setup_kwargs)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = "unused"
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
ctx.data_engine.get_step_number.return_value = 7
|
||||
return ctx
|
||||
|
||||
|
||||
# --- probe_device: enumerated table, programmatic argv (§8 item 12) ------------------
|
||||
|
||||
|
||||
def test_probe_argv_enumerated_kinds_allowed():
|
||||
assert build_probe_argv("alarms") == ["dumpsys", "alarm"]
|
||||
assert build_probe_argv("battery") == ["dumpsys", "battery"]
|
||||
assert build_probe_argv("foreground") == ["dumpsys", "activity", "activities"]
|
||||
assert build_probe_argv("packages") == ["pm", "list", "packages"]
|
||||
assert build_probe_argv("setting", {"namespace": "system", "key": "screen_brightness"}) == [
|
||||
"settings",
|
||||
"get",
|
||||
"system",
|
||||
"screen_brightness",
|
||||
]
|
||||
assert build_probe_argv("content", {"uri": "content://settings/system"}) == [
|
||||
"content",
|
||||
"query",
|
||||
"--uri",
|
||||
"content://settings/system",
|
||||
]
|
||||
assert build_probe_argv("prop", {"key": "ro.build.version.sdk"}) == [
|
||||
"getprop",
|
||||
"ro.build.version.sdk",
|
||||
]
|
||||
|
||||
|
||||
def test_probe_argv_rejects_unknown_kind():
|
||||
with pytest.raises(ValueError):
|
||||
build_probe_argv("shell")
|
||||
with pytest.raises(ValueError):
|
||||
build_probe_argv("dumpsys")
|
||||
|
||||
|
||||
def test_probe_argv_rejects_whitespace_and_metachars():
|
||||
for bad in ("a key", "key;rm", "key|x", "key$", "key`x`", "key&&y", "a\nb"):
|
||||
with pytest.raises(ValueError):
|
||||
build_probe_argv("prop", {"key": bad})
|
||||
with pytest.raises(ValueError):
|
||||
build_probe_argv("setting", {"namespace": "system", "key": "a b"})
|
||||
with pytest.raises(ValueError):
|
||||
build_probe_argv("content", {"uri": "content://a; rm -rf /"})
|
||||
|
||||
|
||||
def test_probe_argv_battery_set_inexpressible():
|
||||
"""Mutating dumpsys subcommands cannot be expressed: no-parameter probes
|
||||
reject every parameter."""
|
||||
with pytest.raises(ValueError):
|
||||
build_probe_argv("battery", {"extra": "set level 100"})
|
||||
with pytest.raises(ValueError):
|
||||
build_probe_argv("alarms", {"args": "anything"})
|
||||
# And the setting namespace is a closed set
|
||||
with pytest.raises(ValueError):
|
||||
build_probe_argv("setting", {"namespace": "battery", "key": "level"})
|
||||
|
||||
|
||||
# --- Release decision is node-side and never rewrites verdicts -----------------------
|
||||
|
||||
|
||||
def test_verdicts_allow_release_semantics():
|
||||
ok = CheckReport(
|
||||
verdicts=[
|
||||
CheckVerdict(item_text="a", kind="verify", status="passed", evidence="e"),
|
||||
CheckVerdict(item_text="b", kind="verify", status="inconclusive", evidence="e"),
|
||||
CheckVerdict(item_text="c", kind="assert", status="failed", evidence="e"),
|
||||
]
|
||||
)
|
||||
# Assert failures never block release
|
||||
assert verdicts_allow_release(ok)
|
||||
|
||||
blocked = CheckReport(
|
||||
verdicts=[
|
||||
CheckVerdict(item_text="a", kind="verify", status="failed", evidence="e"),
|
||||
]
|
||||
)
|
||||
assert not verdicts_allow_release(blocked)
|
||||
|
||||
|
||||
def test_normalize_report_downgrades_vague_failures_and_fills_missing():
|
||||
items = [_ci(kind="verify", text="v1"), _ci(kind="assert", text="a1")]
|
||||
report = CheckReport(
|
||||
verdicts=[CheckVerdict(item_text="v1", kind="verify", status="failed", evidence=" ")]
|
||||
)
|
||||
normalized = _normalize_report(report, items)
|
||||
by_text = {(v.kind, v.item_text): v for v in normalized.verdicts}
|
||||
# Vague failed -> inconclusive (verdict value hygiene, not release logic)
|
||||
assert by_text[("verify", "v1")].status == "inconclusive"
|
||||
# Missing item gets an inconclusive verdict, never a silent pass
|
||||
assert by_text[("assert", "a1")].status == "inconclusive"
|
||||
|
||||
|
||||
# --- Assembly: prompt segments x entry x content x config (§8 item 18) ---------------
|
||||
|
||||
|
||||
def test_prompt_segments_verify_only_has_no_assert_section():
|
||||
prompts = {
|
||||
"base_rules": "BASE",
|
||||
"verify_semantics": "VERIFY-SEG",
|
||||
"assert_semantics": "ASSERT-SEG",
|
||||
"anchor_guide": "ANCHOR-SEG",
|
||||
"final_guide": "FINAL-SEG",
|
||||
"probe_guide": "PROBE-SEG",
|
||||
}
|
||||
text = assemble_checker_prompt_segments(
|
||||
"checkpoint", [_ci(kind="verify")], probe_tool_registered=True, prompts=prompts
|
||||
)
|
||||
assert "VERIFY-SEG" in text
|
||||
assert "ASSERT-SEG" not in text
|
||||
assert "ANCHOR-SEG" in text
|
||||
assert "FINAL-SEG" not in text
|
||||
assert "PROBE-SEG" in text
|
||||
|
||||
text2 = assemble_checker_prompt_segments(
|
||||
"final", [_ci(kind="assert")], probe_tool_registered=False, prompts=prompts
|
||||
)
|
||||
assert "ASSERT-SEG" in text2
|
||||
assert "VERIFY-SEG" not in text2
|
||||
assert "FINAL-SEG" in text2
|
||||
assert "ANCHOR-SEG" not in text2
|
||||
# Probe disabled -> zero mention of probing anywhere in the prompt
|
||||
assert "PROBE-SEG" not in text2
|
||||
|
||||
|
||||
def test_tool_table_probe_gate():
|
||||
ctx = _mock_ctx(disable_device_probes=False)
|
||||
names = {t.name for t in build_checker_tools(ctx, "checkpoint")}
|
||||
assert "probe_device" in names
|
||||
assert "read_note" in names
|
||||
assert "get_step_detail" in names
|
||||
assert "get_step_screenshot" in names
|
||||
# Never any device action, note write, or sub-agent
|
||||
assert not names & {
|
||||
"save_note",
|
||||
"update_note",
|
||||
"append_note",
|
||||
"click",
|
||||
"swipe",
|
||||
"ask_diagnoser",
|
||||
"ask_explorer",
|
||||
"run_adb_command",
|
||||
}
|
||||
|
||||
ctx2 = _mock_ctx(disable_device_probes=True)
|
||||
names2 = {t.name for t in build_checker_tools(ctx2, "checkpoint")}
|
||||
assert "probe_device" not in names2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_check(tmp_path):
|
||||
notes_dir = tmp_path / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Put a dummy task plan file
|
||||
(notes_dir / "task_plan.md").write_text("Plan")
|
||||
|
||||
# Put a dummy verification_chat file
|
||||
import hashlib
|
||||
|
||||
subgoal_text = "Open WhatsApp"
|
||||
s_hash = hashlib.md5(subgoal_text.encode("utf-8")).hexdigest()
|
||||
(notes_dir / f"verification_chat_{s_hash}.json").write_text("[]")
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = str(tmp_path)
|
||||
mock_ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
mock_ctx.execution_setup = None
|
||||
|
||||
# Mock controller for screen data
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.take_screenshot = AsyncMock(return_value="dummy_screenshot_base64")
|
||||
|
||||
from artemis.controllers.device_controller import ScreenDataResponse
|
||||
|
||||
mock_screen_data = ScreenDataResponse(
|
||||
base64="dummy_screenshot_base64",
|
||||
elements=[],
|
||||
width=1080,
|
||||
height=2400,
|
||||
platform="android",
|
||||
async def test_checkpoint_entry_never_touches_live_screen():
|
||||
"""Evidence discipline: the checkpoint entry must not capture the current
|
||||
screen — its evidence is anchored history plus persistent probes."""
|
||||
ctx = _mock_ctx()
|
||||
report = CheckReport(
|
||||
verdicts=[CheckVerdict(item_text="x", kind="verify", status="passed", evidence="e")]
|
||||
)
|
||||
mock_controller.get_screen_data = AsyncMock(return_value=mock_screen_data)
|
||||
|
||||
# Mock LLM response with with_structured_output
|
||||
mock_llm = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.content = "Task completed successfully."
|
||||
mock_response.tool_calls = []
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
mock_structured_llm = MagicMock()
|
||||
mock_structured_llm.ainvoke = AsyncMock(
|
||||
return_value=CheckerResult(success=True, reason="Task completed successfully.")
|
||||
)
|
||||
mock_llm.with_structured_output.return_value = mock_structured_llm
|
||||
structured = MagicMock()
|
||||
structured.ainvoke = AsyncMock(return_value=report)
|
||||
llm = MagicMock()
|
||||
llm.with_structured_output.return_value = structured
|
||||
response = MagicMock()
|
||||
response.tool_calls = []
|
||||
|
||||
with (
|
||||
patch("artemis.agents.checker.checker.get_llm", return_value=llm),
|
||||
patch(
|
||||
"artemis.agents.checker.checker.UnifiedMobileController",
|
||||
return_value=mock_controller,
|
||||
"artemis.agents.checker.checker.acomplete",
|
||||
new=AsyncMock(return_value=response),
|
||||
),
|
||||
patch("artemis.agents.checker.checker.get_llm", return_value=mock_llm),
|
||||
patch(
|
||||
"artemis.utils.task_tree.build_plan_and_history",
|
||||
return_value="Task tree",
|
||||
),
|
||||
"artemis.agents.checker.checker._capture_final_screen",
|
||||
new=AsyncMock(return_value=(None, "SHOULD NOT BE CALLED")),
|
||||
) as capture_mock,
|
||||
):
|
||||
result = await run_async_check(
|
||||
mock_ctx,
|
||||
subgoal_text=subgoal_text,
|
||||
subgoal_hash=s_hash,
|
||||
raw_perception_data={"screenshot_b64": "dummy", "width": 1080, "height": 2400},
|
||||
latest_ui_hierarchy=[],
|
||||
result = await run_checkpoint_check(
|
||||
ctx,
|
||||
check_items=[_ci(text="x")],
|
||||
anchor=EvidenceAnchor(anchor_step_id="sid", trigger_ts=0.0, plan_text="- [x] G"),
|
||||
goal="the goal",
|
||||
subgoal_text="G",
|
||||
)
|
||||
|
||||
assert result["status"] == "success"
|
||||
assert result["reason"] == "Task completed successfully."
|
||||
assert mock_llm.astream.called
|
||||
assert mock_llm.with_structured_output.called
|
||||
capture_mock.assert_not_awaited()
|
||||
assert result.verdicts[0].status == "passed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_entry_captures_live_screen():
|
||||
ctx = _mock_ctx()
|
||||
report = CheckReport(verdicts=[])
|
||||
|
||||
structured = MagicMock()
|
||||
structured.ainvoke = AsyncMock(return_value=report)
|
||||
llm = MagicMock()
|
||||
llm.with_structured_output.return_value = structured
|
||||
response = MagicMock()
|
||||
response.tool_calls = []
|
||||
|
||||
with (
|
||||
patch("artemis.agents.checker.checker.get_llm", return_value=llm),
|
||||
patch(
|
||||
"artemis.agents.checker.checker.acomplete",
|
||||
new=AsyncMock(return_value=response),
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.checker.checker._capture_final_screen",
|
||||
new=AsyncMock(return_value=("b64img", "elements")),
|
||||
) as capture_mock,
|
||||
):
|
||||
await run_final_check(
|
||||
ctx,
|
||||
goal="the goal",
|
||||
plan_text="- [x] G",
|
||||
ledger=[],
|
||||
check_items=[_ci(kind="assert", when="at_end", text="no crash", parent=None)],
|
||||
)
|
||||
|
||||
capture_mock.assert_awaited_once()
|
||||
@@ -14,9 +14,7 @@
|
||||
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
from langgraph.types import Command
|
||||
from artemis.agents.diagnoser.diagnoser import Diagnoser
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.state import State
|
||||
import pytest
|
||||
@@ -148,14 +146,9 @@ async def test_diagnostic_agent_tool_calls(
|
||||
"purpose": None,
|
||||
"state": None,
|
||||
}
|
||||
mock_command = Command(
|
||||
update={
|
||||
VALIDATOR_MESSAGES_KEY: [
|
||||
ToolMessage(content="Video shows UI loading.", tool_call_id="call_2")
|
||||
]
|
||||
}
|
||||
mock_video_tool.ainvoke.return_value = ToolMessage(
|
||||
content="Video shows UI loading.", tool_call_id="call_2"
|
||||
)
|
||||
mock_video_tool.ainvoke.return_value = mock_command
|
||||
mock_get_video_tool.return_value = mock_video_tool
|
||||
|
||||
agent = Diagnoser(mock_context)
|
||||
@@ -273,12 +266,6 @@ async def test_diagnoser_background_job_lifecycle(
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
mock_get_llm.return_value = mock_llm
|
||||
|
||||
# Mock asanitize_update on mock_state
|
||||
async def mock_asanitize_update(ctx, update, agent):
|
||||
return update
|
||||
|
||||
mock_state.asanitize_update = mock_asanitize_update
|
||||
|
||||
# Instantiate the Diagnoser
|
||||
agent = Diagnoser(ctx)
|
||||
|
||||
|
||||
@@ -30,10 +30,7 @@ async def test_operator_node_fast_path():
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
mock_state.subgoal_plan = []
|
||||
mock_state.initial_goal = "Test goal"
|
||||
mock_state.operator_tactical_plan = []
|
||||
|
||||
# Mock controller and data
|
||||
mock_controller = MagicMock()
|
||||
@@ -62,18 +59,15 @@ async def test_operator_node_fast_path():
|
||||
# Patch dependencies
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx)
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# Verify state was updated with action
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
args, kwargs = mock_state.asanitize_update.call_args
|
||||
update_dict = kwargs["update"]
|
||||
update_dict = node_update
|
||||
|
||||
assert "structured_decisions" in update_dict
|
||||
decisions = json.loads(update_dict["structured_decisions"])
|
||||
assert decisions[0]["action"] == "tap"
|
||||
assert decisions[0]["coordinates"] == [54, 120]
|
||||
assert update_dict["complete_subgoals_by_ids"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -241,8 +235,6 @@ async def test_operator_node_multiple_actions():
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
mock_state.subgoal_plan = []
|
||||
mock_state.initial_goal = "Test goal"
|
||||
|
||||
mock_controller = MagicMock()
|
||||
@@ -267,11 +259,9 @@ async def test_operator_node_multiple_actions():
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx)
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
args, kwargs = mock_state.asanitize_update.call_args
|
||||
update_dict = kwargs["update"]
|
||||
update_dict = node_update
|
||||
|
||||
assert "structured_decisions" in update_dict
|
||||
decisions = json.loads(update_dict["structured_decisions"])
|
||||
@@ -296,7 +286,6 @@ async def test_operator_node_no_record_step():
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
mock_state.initial_goal = "Test goal"
|
||||
|
||||
mock_controller = MagicMock()
|
||||
@@ -314,13 +303,11 @@ async def test_operator_node_no_record_step():
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx)
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
mock_ctx.data_engine.record_step.assert_not_called()
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
args, kwargs = mock_state.asanitize_update.call_args
|
||||
update_dict = kwargs["update"]
|
||||
update_dict = node_update
|
||||
assert "structured_decisions" in update_dict
|
||||
|
||||
|
||||
@@ -336,7 +323,6 @@ async def test_operator_dynamic_prompt():
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.initial_goal = "Test goal"
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
|
||||
class MockComponent(PromptComponent):
|
||||
async def __call__(self, builder, state, ctx, **kwargs):
|
||||
@@ -367,7 +353,7 @@ async def test_operator_dynamic_prompt():
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx, prompt_components=[MockComponent()])
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -443,7 +429,6 @@ async def test_operator_history_compression():
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.initial_goal = "Test goal"
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=")
|
||||
@@ -465,10 +450,6 @@ async def test_operator_history_compression():
|
||||
"Goals: {{ initial_goal }}\nPlan:\n{{ plan_and_history"
|
||||
" }}\n# CURRENT OBSERVATION"
|
||||
),
|
||||
"troubleshooter_template": (
|
||||
"Goals: {{ initial_goal }}\nPlan:\n{{ plan_and_history"
|
||||
" }}\n# CURRENT OBSERVATION"
|
||||
),
|
||||
}
|
||||
)
|
||||
elif "task_plan.md" in path_str:
|
||||
@@ -508,7 +489,7 @@ async def test_operator_history_compression():
|
||||
patch("pathlib.Path.read_text", mock_read_text),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, last_n_detailed=5)
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -528,7 +509,7 @@ async def test_operator_loop_detection():
|
||||
node = OperatorNode(mock_ctx)
|
||||
|
||||
with pytest.raises(RuntimeError) as exc_info:
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
assert "Infinite loop detected" in str(exc_info.value)
|
||||
|
||||
@@ -546,7 +527,6 @@ async def test_operator_tracks_subagent_calls():
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = ["ask_explorer"]
|
||||
mock_state.initial_goal = "Test goal"
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=")
|
||||
@@ -589,11 +569,9 @@ async def test_operator_tracks_subagent_calls():
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool])
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
args, kwargs = mock_state.asanitize_update.call_args
|
||||
update_dict = kwargs["update"]
|
||||
update_dict = node_update
|
||||
|
||||
assert "subagent_calls" in update_dict
|
||||
assert update_dict["subagent_calls"] == [
|
||||
@@ -615,7 +593,6 @@ async def test_operator_defer_action_for_gathering():
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
mock_state.initial_goal = "Test goal"
|
||||
|
||||
mock_controller = MagicMock()
|
||||
@@ -684,15 +661,13 @@ async def test_operator_defer_action_for_gathering():
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool])
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# Subagent tool should have been executed
|
||||
mock_call_receiver.assert_called_once_with(query="check something", state=mock_state)
|
||||
|
||||
# Verify final state has the action execution from the second response
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
args, kwargs = mock_state.asanitize_update.call_args
|
||||
update_dict = kwargs["update"]
|
||||
update_dict = node_update
|
||||
|
||||
assert "structured_decisions" in update_dict
|
||||
decisions = json.loads(update_dict["structured_decisions"])
|
||||
@@ -701,100 +676,6 @@ async def test_operator_defer_action_for_gathering():
|
||||
assert decisions[0]["coordinates"] == [54, 120]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_defer_reply_to_checker_for_gathering():
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.context import ArtemisContext
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.execution_setup = None
|
||||
from pathlib import Path
|
||||
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = Path("/tmp/test_session")
|
||||
mock_ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
mock_state.initial_goal = "Test goal"
|
||||
|
||||
mock_llm = MagicMock()
|
||||
|
||||
# First response calls ask_diagnoser and reply_to_checker
|
||||
mock_response_1 = MagicMock()
|
||||
mock_response_1.tool_calls = [
|
||||
{"name": "ask_diagnoser", "args": {"query": "test"}, "id": "call_diag"},
|
||||
{
|
||||
"name": "reply_to_checker",
|
||||
"args": {"reasoning": "I solved it"},
|
||||
"id": "call_reply",
|
||||
},
|
||||
]
|
||||
|
||||
# Second response stops without action tools
|
||||
mock_response_2 = MagicMock()
|
||||
mock_response_2.tool_calls = []
|
||||
|
||||
responses = [mock_response_1, mock_response_2]
|
||||
call_count = 0
|
||||
|
||||
async def mock_ainvoke(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
if call_count < len(responses):
|
||||
res = responses[call_count]
|
||||
call_count += 1
|
||||
return res
|
||||
|
||||
mock_llm.ainvoke.side_effect = mock_ainvoke
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
from langgraph.prebuilt import InjectedState
|
||||
from typing import Annotated
|
||||
|
||||
mock_diagnoser_tool = MagicMock()
|
||||
mock_diagnoser_tool.name = "ask_diagnoser"
|
||||
mock_diagnoser_tool.args = {"state": None}
|
||||
|
||||
mock_call_receiver = Mock()
|
||||
|
||||
async def dummy_diagnoser_func(query: str, state: Annotated[MagicMock, InjectedState]):
|
||||
mock_call_receiver(query=query, state=state)
|
||||
return "Diagnostics completed"
|
||||
|
||||
mock_diagnoser_tool.coroutine = dummy_diagnoser_func
|
||||
mock_diagnoser_tool.func = None
|
||||
|
||||
with (
|
||||
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
|
||||
patch(
|
||||
"artemis.agents.operator.operator.trace_langchain_tool",
|
||||
side_effect=lambda t, ctx: t,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.operator.operator.OperatorNode._has_checker_feedback",
|
||||
return_value=True,
|
||||
),
|
||||
patch(
|
||||
"artemis.agents.operator.operator.OperatorNode._get_verification_chat_rounds",
|
||||
return_value=(1, 1),
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_diagnoser_tool])
|
||||
await node(mock_state)
|
||||
|
||||
# Subagent was called
|
||||
mock_call_receiver.assert_called_once()
|
||||
|
||||
# Verify reply to checker was NOT executed / flagged
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
args, kwargs = mock_state.asanitize_update.call_args
|
||||
update_dict = kwargs["update"]
|
||||
|
||||
assert update_dict.get("operator_replied") is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_actions_translation():
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
@@ -907,7 +788,6 @@ async def test_operator_no_defer_for_note_updating():
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
mock_state.initial_goal = "Test goal"
|
||||
|
||||
mock_controller = MagicMock()
|
||||
@@ -965,15 +845,13 @@ async def test_operator_no_defer_for_note_updating():
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_update_note_tool])
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# The update_note tool should have been executed immediately
|
||||
mock_call_receiver.assert_called_once_with(key="progress", content="step 1")
|
||||
|
||||
# The click action should NOT be deferred, so it should be in structured_decisions
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
args, kwargs = mock_state.asanitize_update.call_args
|
||||
update_dict = kwargs["update"]
|
||||
update_dict = node_update
|
||||
|
||||
assert "structured_decisions" in update_dict
|
||||
decisions = json.loads(update_dict["structured_decisions"])
|
||||
@@ -995,7 +873,6 @@ async def test_operator_defer_for_note_reading():
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
mock_state.initial_goal = "Test goal"
|
||||
|
||||
mock_controller = MagicMock()
|
||||
@@ -1058,7 +935,7 @@ async def test_operator_defer_for_note_reading():
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_read_note_tool])
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# The read_note tool should have been executed
|
||||
mock_call_receiver.assert_called_once_with(key="progress")
|
||||
@@ -1068,9 +945,7 @@ async def test_operator_defer_for_note_reading():
|
||||
# Total call_count should be 2.
|
||||
assert call_count == 2
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
args, kwargs = mock_state.asanitize_update.call_args
|
||||
update_dict = kwargs["update"]
|
||||
update_dict = node_update
|
||||
|
||||
assert "structured_decisions" in update_dict
|
||||
decisions = json.loads(update_dict["structured_decisions"])
|
||||
@@ -1092,7 +967,6 @@ async def test_operator_no_defer_for_note_appending():
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
mock_state.initial_goal = "Test goal"
|
||||
|
||||
mock_controller = MagicMock()
|
||||
@@ -1149,15 +1023,13 @@ async def test_operator_no_defer_for_note_appending():
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx, tools=[mock_append_note_tool])
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# The append_note tool should have been executed immediately
|
||||
mock_call_receiver.assert_called_once_with(key="progress", content="step 1")
|
||||
|
||||
# The click action should NOT be deferred, so it should be in structured_decisions
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
args, kwargs = mock_state.asanitize_update.call_args
|
||||
update_dict = kwargs["update"]
|
||||
update_dict = node_update
|
||||
|
||||
assert "structured_decisions" in update_dict
|
||||
decisions = json.loads(update_dict["structured_decisions"])
|
||||
@@ -1180,7 +1052,6 @@ async def test_operator_background_tasks_prompt_injection():
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.initial_goal = "Test goal"
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=")
|
||||
@@ -1231,7 +1102,7 @@ async def test_operator_background_tasks_prompt_injection():
|
||||
active_background_tasks=active_background_tasks,
|
||||
)
|
||||
captured_messages.extend(messages)
|
||||
return [], False, None, None, False
|
||||
return [], None, None, False
|
||||
|
||||
with (
|
||||
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
|
||||
@@ -1241,7 +1112,7 @@ async def test_operator_background_tasks_prompt_injection():
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx)
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# Assert that the captured messages (prompts) contain the background tasks section
|
||||
full_content = ""
|
||||
@@ -1276,7 +1147,6 @@ async def test_operator_task_plan_warning_prompt_injection():
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.initial_goal = "Test goal"
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=")
|
||||
@@ -1323,7 +1193,7 @@ async def test_operator_task_plan_warning_prompt_injection():
|
||||
task_plan="",
|
||||
)
|
||||
captured_messages_1.extend(messages)
|
||||
return [], False, None, None, False
|
||||
return [], None, None, False
|
||||
|
||||
with (
|
||||
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
|
||||
@@ -1333,7 +1203,7 @@ async def test_operator_task_plan_warning_prompt_injection():
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx)
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
full_content_1 = ""
|
||||
for m in captured_messages_1:
|
||||
@@ -1376,7 +1246,7 @@ async def test_operator_task_plan_warning_prompt_injection():
|
||||
task_plan="",
|
||||
)
|
||||
captured_messages_2.extend(messages)
|
||||
return [], False, None, None, False
|
||||
return [], None, None, False
|
||||
|
||||
with (
|
||||
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
|
||||
@@ -1386,7 +1256,7 @@ async def test_operator_task_plan_warning_prompt_injection():
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx)
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
full_content_2 = ""
|
||||
for m in captured_messages_2:
|
||||
@@ -1418,7 +1288,6 @@ async def test_operator_tool_limit_exceeded_warning():
|
||||
# Set the flag to True to simulate tool call limit exceeded in previous turn
|
||||
mock_state.operator_tool_limit_exceeded = True
|
||||
mock_state.initial_goal = "Test goal"
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0X3NjcmVlbnNob3Q=")
|
||||
@@ -1443,8 +1312,8 @@ async def test_operator_tool_limit_exceeded_warning():
|
||||
task_plan="",
|
||||
)
|
||||
captured_messages.extend(messages)
|
||||
# Return 5 elements: action_result, requested_argue, raw_thinking, native_thinking, tool_limit_exceeded
|
||||
return [], False, None, None, False
|
||||
# Return 4 elements: action_result, raw_thinking, native_thinking, tool_limit_exceeded
|
||||
return [], None, None, False
|
||||
|
||||
with (
|
||||
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
|
||||
@@ -1454,7 +1323,7 @@ async def test_operator_tool_limit_exceeded_warning():
|
||||
),
|
||||
):
|
||||
node = OperatorNode(mock_ctx)
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
full_content = ""
|
||||
for m in captured_messages:
|
||||
@@ -1469,9 +1338,7 @@ async def test_operator_tool_limit_exceeded_warning():
|
||||
"You did not execute any screen interaction actions in your last turn" in full_content
|
||||
)
|
||||
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
_, kwargs = mock_state.asanitize_update.call_args
|
||||
assert kwargs["update"].get("operator_tool_limit_exceeded") is False
|
||||
assert node_update.get("operator_tool_limit_exceeded") is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -1488,7 +1355,6 @@ async def test_operator_fallback_function_call_parsing():
|
||||
|
||||
mock_state = MagicMock()
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
mock_state.initial_goal = "Test goal"
|
||||
|
||||
mock_controller = MagicMock()
|
||||
@@ -1515,12 +1381,10 @@ async def test_operator_fallback_function_call_parsing():
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx)
|
||||
await node(mock_state)
|
||||
node_update = await node(mock_state)
|
||||
|
||||
# Verify state was updated with parsed click action
|
||||
mock_state.asanitize_update.assert_called_once()
|
||||
_, kwargs = mock_state.asanitize_update.call_args
|
||||
update_dict = kwargs["update"]
|
||||
update_dict = node_update
|
||||
|
||||
assert "structured_decisions" in update_dict
|
||||
decisions = json.loads(update_dict["structured_decisions"])
|
||||
|
||||
@@ -29,15 +29,14 @@ def test_operator_prompt_contract_matches_note_runtime_semantics():
|
||||
assert "memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`" not in prompt
|
||||
|
||||
|
||||
def test_troubleshooter_prompt_has_mutually_exclusive_recovery_branches():
|
||||
prompt = apply_operator_prompt_contract(load_operator_prompts()["troubleshooter_template"])
|
||||
|
||||
assert "invoke `reply_to_checker` directly" in prompt
|
||||
assert "This branch takes precedence over diagnosis" in prompt
|
||||
assert "perform that correction directly" in prompt
|
||||
assert "also takes precedence over generic diagnosis triggers" in prompt
|
||||
assert "A Checker rejection alone is not a mandatory diagnosis trigger" in prompt
|
||||
assert "**Validation/Checker Rejection**" not in prompt
|
||||
def test_operator_prompt_is_single_template_without_checker_dialogue():
|
||||
"""The prompt is never switched by verification results: main_template is
|
||||
the only template, and no checker-dialogue machinery is advertised."""
|
||||
prompts = load_operator_prompts()
|
||||
assert set(prompts) == {"main_template"}
|
||||
prompt = apply_operator_prompt_contract(prompts["main_template"])
|
||||
assert "reply_to_checker" not in prompt
|
||||
assert "Checker" not in prompt
|
||||
|
||||
|
||||
def test_operator_prompt_omits_environment_trust_and_explorer_directives():
|
||||
|
||||
@@ -12,10 +12,25 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
from artemis.graph.graph import check_plan_mutation_rejections
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.context import ArtemisContext, ExecutionSetup
|
||||
from artemis.graph.graph import (
|
||||
check_plan_mutation_rejections,
|
||||
check_unintended_rewrite,
|
||||
wrap_note_tool,
|
||||
)
|
||||
from artemis.graph.state import State
|
||||
|
||||
CONTINUOUS_PLAN = (
|
||||
"- [x] Open app\n"
|
||||
"- [/] [Loop:continuous] Periodically monitor for new emails "
|
||||
"(Interval: every 5 minutes)\n"
|
||||
" - [x] Polling Check #1: Done"
|
||||
)
|
||||
|
||||
|
||||
def test_check_plan_mutation_incomplete_nested():
|
||||
content_before = "- [/] Main task\n - [ ] Subtask 1"
|
||||
@@ -26,57 +41,52 @@ def test_check_plan_mutation_incomplete_nested():
|
||||
|
||||
|
||||
def test_check_plan_mutation_delete_continuous_loop():
|
||||
content_before = (
|
||||
"- [x] Open app\n"
|
||||
"- [/] [Loop] Periodically monitor for new emails "
|
||||
"(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n"
|
||||
" - [x] Polling Check #1: Done"
|
||||
)
|
||||
content_after = (
|
||||
"- [x] Open app\n"
|
||||
"- [x] Monitor for new emails\n"
|
||||
" - [x] Baseline established\n"
|
||||
" - [x] Polling Check #1: Done"
|
||||
)
|
||||
res = check_plan_mutation_rejections(content_before, content_after)
|
||||
res = check_plan_mutation_rejections(CONTINUOUS_PLAN, content_after)
|
||||
assert res is not None
|
||||
assert "cannot delete an active [Loop]" in res
|
||||
assert "cannot delete an active [Loop:continuous]" in res
|
||||
|
||||
|
||||
def test_check_plan_mutation_mark_continuous_loop_completed():
|
||||
content_before = (
|
||||
"- [x] Open app\n"
|
||||
"- [/] [Loop] Periodically monitor for new emails "
|
||||
"(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n"
|
||||
" - [x] Polling Check #1: Done"
|
||||
)
|
||||
content_after = (
|
||||
"- [x] Open app\n"
|
||||
"- [x] [Loop] Periodically monitor for new emails "
|
||||
"(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n"
|
||||
" - [x] Polling Check #1: Done"
|
||||
)
|
||||
res = check_plan_mutation_rejections(content_before, content_after)
|
||||
content_after = CONTINUOUS_PLAN.replace("- [/] [Loop:continuous]", "- [x] [Loop:continuous]")
|
||||
res = check_plan_mutation_rejections(CONTINUOUS_PLAN, content_after)
|
||||
assert res is not None
|
||||
assert "cannot be unilaterally marked as completed [x]" in res
|
||||
assert "cannot be" in res and "marked as completed [x]" in res
|
||||
|
||||
|
||||
def test_check_plan_mutation_user_stopped_allowed():
|
||||
content_after = CONTINUOUS_PLAN.replace("- [/] [Loop:continuous]", "- [x] [Loop:continuous]")
|
||||
state = MagicMock(spec=State)
|
||||
state.user_stop_requested = True
|
||||
res = check_plan_mutation_rejections(CONTINUOUS_PLAN, content_after, state=state)
|
||||
assert res is None
|
||||
|
||||
|
||||
def test_check_plan_mutation_stop_wording_is_not_a_signal():
|
||||
"""Natural-language 'stop' phrasing must NOT unlock the continuous loop —
|
||||
only the explicit release_loop signal (user_stop_requested) does."""
|
||||
content_after = CONTINUOUS_PLAN.replace("- [/] [Loop:continuous]", "- [x] [Loop:continuous]")
|
||||
state = MagicMock(spec=State)
|
||||
state.user_stop_requested = False
|
||||
state.injected_instruction = "Please stop the task now."
|
||||
res = check_plan_mutation_rejections(CONTINUOUS_PLAN, content_after, state=state)
|
||||
assert res is not None
|
||||
|
||||
|
||||
def test_check_plan_mutation_bounded_loop_completion_allowed():
|
||||
"""Plain [Loop] milestones are bounded: the model may complete them."""
|
||||
content_before = (
|
||||
"- [x] Open app\n"
|
||||
"- [/] [Loop] Periodically monitor for new emails "
|
||||
"(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n"
|
||||
" - [x] Polling Check #1: Done"
|
||||
"- [/] [Loop] Inspect candidate emails (Exit: all 5 candidates inspected)\n"
|
||||
" - [x] Candidate #5: Done"
|
||||
)
|
||||
content_after = (
|
||||
"- [x] Open app\n"
|
||||
"- [x] [Loop] Periodically monitor for new emails "
|
||||
"(Exit: Continuous monitoring until manually stopped; Interval: every 5 minutes)\n"
|
||||
" - [x] Polling Check #1: Done"
|
||||
)
|
||||
state = MagicMock(spec=State)
|
||||
state.injected_instruction = "Please stop the task now."
|
||||
res = check_plan_mutation_rejections(content_before, content_after, state=state)
|
||||
content_after = content_before.replace("- [/] [Loop]", "- [x] [Loop]")
|
||||
res = check_plan_mutation_rejections(content_before, content_after)
|
||||
assert res is None
|
||||
|
||||
|
||||
@@ -85,3 +95,209 @@ def test_check_plan_mutation_normal_bounded_task():
|
||||
content_after = "- [x] Open Settings\n- [x] Toggle WiFi"
|
||||
res = check_plan_mutation_rejections(content_before, content_after)
|
||||
assert res is None
|
||||
|
||||
|
||||
def test_check_unintended_rewrite_detects_hand_slip():
|
||||
content_before = "- [x] Open Settings app\n- [/] Toggle the WiFi switch"
|
||||
# Full rewrite: milestone 1's status unchanged but text drifted
|
||||
content_after = "- [x] Open the Settings application\n- [/] Toggle the WiFi switch"
|
||||
res = check_unintended_rewrite(content_before, content_after)
|
||||
assert res is not None
|
||||
assert "Open Settings app" in res
|
||||
assert "update_note" in res
|
||||
|
||||
|
||||
def test_check_unintended_rewrite_allows_status_flips_and_appends():
|
||||
content_before = "- [x] Open Settings\n- [/] Toggle WiFi"
|
||||
content_after = "- [x] Open Settings\n- [x] Toggle WiFi\n - [x] Confirmed enabled"
|
||||
assert check_unintended_rewrite(content_before, content_after) is None
|
||||
|
||||
|
||||
def test_check_unintended_rewrite_ignores_structural_replans():
|
||||
# Milestone count changes are declared replans, judged by planner validation
|
||||
content_before = "- [x] Open Settings\n- [/] Toggle WiFi"
|
||||
content_after = "- [x] Open Settings\n- [/] Toggle WiFi\n- [ ] Verify connectivity"
|
||||
assert check_unintended_rewrite(content_before, content_after) is None
|
||||
|
||||
|
||||
# --- §4.3 deterministic check-line guard (independent of planner validation) ---------
|
||||
|
||||
GUARDED_PLAN = (
|
||||
"- [/] Create the alarm\n"
|
||||
" - verify: alarm exists in the list\n"
|
||||
" - assert: a toast appeared\n"
|
||||
"- [ ] Next milestone\n"
|
||||
)
|
||||
|
||||
|
||||
def _guard_ctx(tmp_path, **setup_kwargs):
|
||||
# Planner validation OFF by default: the guard must hold on its own.
|
||||
setup_kwargs.setdefault("disable_planner_validation", True)
|
||||
setup_kwargs.setdefault("disable_checker", False)
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.execution_setup = ExecutionSetup(**setup_kwargs)
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = str(tmp_path)
|
||||
ctx.planner_task = None
|
||||
ctx.last_validated_plan = None
|
||||
ctx.pending_validated_plan = None
|
||||
ctx.task_plan_content_before = None
|
||||
ctx.pending_checkpoints = []
|
||||
ctx.checkpoint_tasks = {}
|
||||
return ctx
|
||||
|
||||
|
||||
def _guard_plan_path(tmp_path, content=GUARDED_PLAN):
|
||||
notes_dir = tmp_path / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = notes_dir / "task_plan.md"
|
||||
path.write_text(content, encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def _save_invoke(path):
|
||||
async def fake(tool, args, tool_call_id, state, record_trace=None):
|
||||
path.write_text(args["content"], encoding="utf-8")
|
||||
return "Success"
|
||||
|
||||
return fake
|
||||
|
||||
|
||||
def _save_tool():
|
||||
tool = MagicMock()
|
||||
tool.name = "save_note"
|
||||
tool.description = "save_note tool"
|
||||
return tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_deleting_check_line_is_restored(tmp_path):
|
||||
path = _guard_plan_path(tmp_path)
|
||||
ctx = _guard_ctx(tmp_path)
|
||||
|
||||
rewritten = "- [/] Create the alarm\n - assert: a toast appeared\n- [ ] Next milestone\n"
|
||||
with patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=_save_invoke(path),
|
||||
):
|
||||
wrapped = wrap_note_tool(ctx, _save_tool())
|
||||
await wrapped.ainvoke({"key": "task_plan", "content": rewritten, "tool_call_id": "t"})
|
||||
|
||||
content = path.read_text(encoding="utf-8")
|
||||
assert "- verify: alarm exists in the list" in content
|
||||
assert "- assert: a toast appeared" in content
|
||||
# No planner validation was needed or spawned (guard is deterministic)
|
||||
assert ctx.planner_task is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_rewording_check_line_restores_original_keeps_new(tmp_path):
|
||||
path = _guard_plan_path(tmp_path)
|
||||
ctx = _guard_ctx(tmp_path)
|
||||
|
||||
rewritten = GUARDED_PLAN.replace(
|
||||
" - verify: alarm exists in the list\n",
|
||||
" - verify: alarm roughly configured\n",
|
||||
)
|
||||
with patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=_save_invoke(path),
|
||||
):
|
||||
wrapped = wrap_note_tool(ctx, _save_tool())
|
||||
await wrapped.ainvoke({"key": "task_plan", "content": rewritten, "tool_call_id": "t"})
|
||||
|
||||
content = path.read_text(encoding="utf-8")
|
||||
# Original standard restored; the Operator's new line survives as an addition
|
||||
assert "- verify: alarm exists in the list" in content
|
||||
assert "- verify: alarm roughly configured" in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deleted_parent_turns_check_into_task_level_at_end(tmp_path):
|
||||
path = _guard_plan_path(tmp_path)
|
||||
ctx = _guard_ctx(tmp_path)
|
||||
|
||||
# Whole parent subgoal removed
|
||||
rewritten = "- [ ] Next milestone\n"
|
||||
with patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=_save_invoke(path),
|
||||
):
|
||||
wrapped = wrap_note_tool(ctx, _save_tool())
|
||||
await wrapped.ainvoke({"key": "task_plan", "content": rewritten, "tool_call_id": "t"})
|
||||
|
||||
content = path.read_text(encoding="utf-8")
|
||||
assert "- verify@end: alarm exists in the list" in content
|
||||
assert "- assert@end: a toast appeared" in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_operator_adding_check_lines_is_allowed(tmp_path):
|
||||
path = _guard_plan_path(tmp_path)
|
||||
ctx = _guard_ctx(tmp_path)
|
||||
|
||||
extended = GUARDED_PLAN + " - verify: extra criterion\n"
|
||||
with patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=_save_invoke(path),
|
||||
):
|
||||
wrapped = wrap_note_tool(ctx, _save_tool())
|
||||
result = await wrapped.ainvoke(
|
||||
{"key": "task_plan", "content": extended, "tool_call_id": "t"}
|
||||
)
|
||||
|
||||
assert result == "Success"
|
||||
assert "- verify: extra criterion" in path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guard_holds_with_planner_validation_disabled_and_checks_off(tmp_path):
|
||||
"""The guard is content-driven: it protects check lines even when every
|
||||
check switch AND planner validation are disabled."""
|
||||
path = _guard_plan_path(tmp_path)
|
||||
ctx = _guard_ctx(tmp_path, disable_checker=True, disable_planner_validation=True)
|
||||
|
||||
with patch(
|
||||
"artemis.graph.graph.invoke_tool_with_injection",
|
||||
side_effect=_save_invoke(path),
|
||||
):
|
||||
wrapped = wrap_note_tool(ctx, _save_tool())
|
||||
await wrapped.ainvoke(
|
||||
{
|
||||
"key": "task_plan",
|
||||
"content": "- [/] Create the alarm\n- [ ] Next milestone\n",
|
||||
"tool_call_id": "t",
|
||||
}
|
||||
)
|
||||
|
||||
content = path.read_text(encoding="utf-8")
|
||||
assert "- verify: alarm exists in the list" in content
|
||||
assert "- assert: a toast appeared" in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_planner_node_writes_are_not_guarded():
|
||||
"""The Planner keeps revision authority over check standards: its node is
|
||||
wired with the UNWRAPPED note tools, so the guard never applies to it."""
|
||||
from artemis.context import DeviceContext, DevicePlatform
|
||||
from artemis.graph.graph import get_graph
|
||||
|
||||
device = DeviceContext(
|
||||
host_platform="LINUX",
|
||||
mobile_platform=DevicePlatform.ANDROID,
|
||||
device_id="dummy",
|
||||
device_width=1080,
|
||||
device_height=2400,
|
||||
)
|
||||
ctx = ArtemisContext(device=device, execution_setup=ExecutionSetup())
|
||||
graph = await get_graph(ctx)
|
||||
|
||||
planner_tools = {t.name: t for t in graph.nodes["planner"].bound.afunc.tools}
|
||||
operator_tools = {t.name: t for t in graph.nodes["operator"].bound.afunc.tools}
|
||||
|
||||
# Operator's plan-writing tools go through the guard wrapper...
|
||||
assert operator_tools["save_note"].coroutine.__name__ == "wrapped_note_tool"
|
||||
assert operator_tools["update_note"].coroutine.__name__ == "wrapped_update_note"
|
||||
# ...the Planner's do not.
|
||||
assert getattr(planner_tools["save_note"].coroutine, "__name__", "") != "wrapped_note_tool"
|
||||
assert getattr(planner_tools["update_note"].coroutine, "__name__", "") != "wrapped_update_note"
|
||||
@@ -32,9 +32,6 @@ class DummyState:
|
||||
self.operator_raw_data = operator_raw_data
|
||||
self.operator_replan_reason = operator_replan_reason
|
||||
|
||||
async def asanitize_update(self, ctx, update, agent):
|
||||
return update
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.graph.graph import execution_check_node
|
||||
from artemis.graph.state import State
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_snapshot_rollback_flow(tmp_path):
|
||||
# Setup dummy directories
|
||||
base_dir = tmp_path / "trace"
|
||||
notes_dir = base_dir / "notes"
|
||||
notes_dir.mkdir(parents=True)
|
||||
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
task_plan_path.write_text("Initial Plan", encoding="utf-8")
|
||||
|
||||
# Mock Context
|
||||
mock_ctx = MagicMock(spec=ArtemisContext)
|
||||
mock_ctx.data_engine = MagicMock()
|
||||
mock_ctx.data_engine.base_dir = base_dir
|
||||
mock_ctx.execution_setup = None
|
||||
|
||||
# Mock State
|
||||
mock_state = MagicMock(spec=State)
|
||||
mock_state.subagent_calls = []
|
||||
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
|
||||
mock_state.initial_goal = "Test goal"
|
||||
mock_state.short_term_memory = ""
|
||||
mock_state.operator_raw_data = {
|
||||
"screenshot_b64": "YmFzZTY0",
|
||||
"xml_hierarchy": "<hierarchy></hierarchy>",
|
||||
"ocr_results": [],
|
||||
}
|
||||
mock_state.current_step_id = "test-step-id"
|
||||
|
||||
# Mock Controller and LLM to let Operator run minimally
|
||||
mock_controller = MagicMock()
|
||||
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0")
|
||||
mock_device_data = MagicMock()
|
||||
mock_device_data.elements = []
|
||||
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
|
||||
|
||||
mock_llm = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.tool_calls = []
|
||||
|
||||
async def mock_astream(*args, **kwargs):
|
||||
yield mock_response
|
||||
|
||||
mock_llm.astream.side_effect = mock_astream
|
||||
mock_llm.bind_tools.return_value = mock_llm
|
||||
|
||||
# 1. Run Operator to trigger snapshot
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
node = OperatorNode(mock_ctx)
|
||||
await node(mock_state)
|
||||
|
||||
# Verify snapshot was created
|
||||
snapshot_dir = notes_dir.with_name("notes_snapshot")
|
||||
assert snapshot_dir.exists()
|
||||
assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "Initial Plan"
|
||||
assert mock_ctx.task_plan_snapshot == snapshot_dir
|
||||
|
||||
# 2. Simulate Operator optimistic modification
|
||||
task_plan_path.write_text("Modified Plan", encoding="utf-8")
|
||||
|
||||
# 3. Run execution_check_node with failure to trigger rollback
|
||||
future = asyncio.Future()
|
||||
future.set_result({"status": "failed"})
|
||||
mock_ctx.checker_task = future
|
||||
|
||||
await execution_check_node(mock_state, mock_ctx)
|
||||
|
||||
# Verify rollback restored the file
|
||||
assert task_plan_path.read_text(encoding="utf-8") == "Initial Plan"
|
||||
|
||||
# 4. Run Operator again to create new snapshot (simulate next attempt)
|
||||
# First modify it again to simulate operator working on restored plan
|
||||
task_plan_path.write_text("Modified Plan 2", encoding="utf-8")
|
||||
|
||||
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
|
||||
await node(mock_state)
|
||||
|
||||
assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "Modified Plan 2"
|
||||
|
||||
# 5. Run execution_check_node with success to trigger commit (cleanup)
|
||||
future_success = asyncio.Future()
|
||||
future_success.set_result({"status": "success"})
|
||||
mock_ctx.checker_task = future_success
|
||||
|
||||
await execution_check_node(mock_state, mock_ctx)
|
||||
|
||||
# Verify snapshot was deleted
|
||||
assert not snapshot_dir.exists()
|
||||
@@ -37,18 +37,13 @@ class DummyState:
|
||||
def __init__(
|
||||
self,
|
||||
structured_decisions,
|
||||
validator_messages=[],
|
||||
current_step_id=None,
|
||||
latest_screenshot=None,
|
||||
):
|
||||
self.structured_decisions = structured_decisions
|
||||
self.validator_messages = validator_messages
|
||||
self.current_step_id = current_step_id
|
||||
self.latest_screenshot = latest_screenshot
|
||||
|
||||
async def asanitize_update(self, ctx, update, agent):
|
||||
return update
|
||||
|
||||
|
||||
class FakeActionSession:
|
||||
"""Test double for artemis.mcp.action_session.ActionSession."""
|
||||
@@ -146,6 +141,7 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
|
||||
# entirely. Its hazard class (leaked AnyIO cancel scopes) is now pinned by
|
||||
# tests/unit/mcp/test_action_session.py against the in-process ActionSession.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode triggers failure analysis on error and handles
|
||||
|
||||
Reference in new issue
Block a user