somew1nd
cbe8f4f0c6
Introduce two-gate checker settlement, explicit graph visibility and checkpoints, structured LLM reliability, centralized action specifications, and corresponding tests and design documentation.
266 lines
9.0 KiB
Python
266 lines
9.0 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from artemis.agents.checker.checker import (
|
|
CheckReport,
|
|
CheckVerdict,
|
|
_normalize_report,
|
|
assemble_checker_prompt_segments,
|
|
build_checker_tools,
|
|
build_probe_argv,
|
|
run_checkpoint_check,
|
|
run_final_check,
|
|
verdicts_allow_release,
|
|
)
|
|
from artemis.context import ArtemisContext, ExecutionSetup
|
|
from artemis.graph.checkpoints import EvidenceAnchor
|
|
from artemis.utils.plan_grammar import CheckItem
|
|
|
|
|
|
def _ci(kind="verify", when="on_complete", text="expected state", parent="p"):
|
|
return CheckItem(kind=kind, when=when, text=text, parent_key=parent)
|
|
|
|
|
|
def _mock_ctx(**setup_kwargs):
|
|
ctx = MagicMock(spec=ArtemisContext)
|
|
ctx.execution_setup = ExecutionSetup(**setup_kwargs)
|
|
ctx.data_engine = MagicMock()
|
|
ctx.data_engine.base_dir = "unused"
|
|
ctx.data_engine.get_agent_friendly_steps.return_value = []
|
|
ctx.data_engine.get_step_number.return_value = 7
|
|
return ctx
|
|
|
|
|
|
# --- probe_device: enumerated table, programmatic argv (§8 item 12) ------------------
|
|
|
|
|
|
def test_probe_argv_enumerated_kinds_allowed():
|
|
assert build_probe_argv("alarms") == ["dumpsys", "alarm"]
|
|
assert build_probe_argv("battery") == ["dumpsys", "battery"]
|
|
assert build_probe_argv("foreground") == ["dumpsys", "activity", "activities"]
|
|
assert build_probe_argv("packages") == ["pm", "list", "packages"]
|
|
assert build_probe_argv("setting", {"namespace": "system", "key": "screen_brightness"}) == [
|
|
"settings",
|
|
"get",
|
|
"system",
|
|
"screen_brightness",
|
|
]
|
|
assert build_probe_argv("content", {"uri": "content://settings/system"}) == [
|
|
"content",
|
|
"query",
|
|
"--uri",
|
|
"content://settings/system",
|
|
]
|
|
assert build_probe_argv("prop", {"key": "ro.build.version.sdk"}) == [
|
|
"getprop",
|
|
"ro.build.version.sdk",
|
|
]
|
|
|
|
|
|
def test_probe_argv_rejects_unknown_kind():
|
|
with pytest.raises(ValueError):
|
|
build_probe_argv("shell")
|
|
with pytest.raises(ValueError):
|
|
build_probe_argv("dumpsys")
|
|
|
|
|
|
def test_probe_argv_rejects_whitespace_and_metachars():
|
|
for bad in ("a key", "key;rm", "key|x", "key$", "key`x`", "key&&y", "a\nb"):
|
|
with pytest.raises(ValueError):
|
|
build_probe_argv("prop", {"key": bad})
|
|
with pytest.raises(ValueError):
|
|
build_probe_argv("setting", {"namespace": "system", "key": "a b"})
|
|
with pytest.raises(ValueError):
|
|
build_probe_argv("content", {"uri": "content://a; rm -rf /"})
|
|
|
|
|
|
def test_probe_argv_battery_set_inexpressible():
|
|
"""Mutating dumpsys subcommands cannot be expressed: no-parameter probes
|
|
reject every parameter."""
|
|
with pytest.raises(ValueError):
|
|
build_probe_argv("battery", {"extra": "set level 100"})
|
|
with pytest.raises(ValueError):
|
|
build_probe_argv("alarms", {"args": "anything"})
|
|
# And the setting namespace is a closed set
|
|
with pytest.raises(ValueError):
|
|
build_probe_argv("setting", {"namespace": "battery", "key": "level"})
|
|
|
|
|
|
# --- Release decision is node-side and never rewrites verdicts -----------------------
|
|
|
|
|
|
def test_verdicts_allow_release_semantics():
|
|
ok = CheckReport(
|
|
verdicts=[
|
|
CheckVerdict(item_text="a", kind="verify", status="passed", evidence="e"),
|
|
CheckVerdict(item_text="b", kind="verify", status="inconclusive", evidence="e"),
|
|
CheckVerdict(item_text="c", kind="assert", status="failed", evidence="e"),
|
|
]
|
|
)
|
|
# Assert failures never block release
|
|
assert verdicts_allow_release(ok)
|
|
|
|
blocked = CheckReport(
|
|
verdicts=[
|
|
CheckVerdict(item_text="a", kind="verify", status="failed", evidence="e"),
|
|
]
|
|
)
|
|
assert not verdicts_allow_release(blocked)
|
|
|
|
|
|
def test_normalize_report_downgrades_vague_failures_and_fills_missing():
|
|
items = [_ci(kind="verify", text="v1"), _ci(kind="assert", text="a1")]
|
|
report = CheckReport(
|
|
verdicts=[CheckVerdict(item_text="v1", kind="verify", status="failed", evidence=" ")]
|
|
)
|
|
normalized = _normalize_report(report, items)
|
|
by_text = {(v.kind, v.item_text): v for v in normalized.verdicts}
|
|
# Vague failed -> inconclusive (verdict value hygiene, not release logic)
|
|
assert by_text[("verify", "v1")].status == "inconclusive"
|
|
# Missing item gets an inconclusive verdict, never a silent pass
|
|
assert by_text[("assert", "a1")].status == "inconclusive"
|
|
|
|
|
|
# --- Assembly: prompt segments x entry x content x config (§8 item 18) ---------------
|
|
|
|
|
|
def test_prompt_segments_verify_only_has_no_assert_section():
|
|
prompts = {
|
|
"base_rules": "BASE",
|
|
"verify_semantics": "VERIFY-SEG",
|
|
"assert_semantics": "ASSERT-SEG",
|
|
"anchor_guide": "ANCHOR-SEG",
|
|
"final_guide": "FINAL-SEG",
|
|
"probe_guide": "PROBE-SEG",
|
|
}
|
|
text = assemble_checker_prompt_segments(
|
|
"checkpoint", [_ci(kind="verify")], probe_tool_registered=True, prompts=prompts
|
|
)
|
|
assert "VERIFY-SEG" in text
|
|
assert "ASSERT-SEG" not in text
|
|
assert "ANCHOR-SEG" in text
|
|
assert "FINAL-SEG" not in text
|
|
assert "PROBE-SEG" in text
|
|
|
|
text2 = assemble_checker_prompt_segments(
|
|
"final", [_ci(kind="assert")], probe_tool_registered=False, prompts=prompts
|
|
)
|
|
assert "ASSERT-SEG" in text2
|
|
assert "VERIFY-SEG" not in text2
|
|
assert "FINAL-SEG" in text2
|
|
assert "ANCHOR-SEG" not in text2
|
|
# Probe disabled -> zero mention of probing anywhere in the prompt
|
|
assert "PROBE-SEG" not in text2
|
|
|
|
|
|
def test_tool_table_probe_gate():
|
|
ctx = _mock_ctx(disable_device_probes=False)
|
|
names = {t.name for t in build_checker_tools(ctx, "checkpoint")}
|
|
assert "probe_device" in names
|
|
assert "read_note" in names
|
|
assert "get_step_detail" in names
|
|
assert "get_step_screenshot" in names
|
|
# Never any device action, note write, or sub-agent
|
|
assert not names & {
|
|
"save_note",
|
|
"update_note",
|
|
"append_note",
|
|
"click",
|
|
"swipe",
|
|
"ask_diagnoser",
|
|
"ask_explorer",
|
|
"run_adb_command",
|
|
}
|
|
|
|
ctx2 = _mock_ctx(disable_device_probes=True)
|
|
names2 = {t.name for t in build_checker_tools(ctx2, "checkpoint")}
|
|
assert "probe_device" not in names2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_checkpoint_entry_never_touches_live_screen():
|
|
"""Evidence discipline: the checkpoint entry must not capture the current
|
|
screen — its evidence is anchored history plus persistent probes."""
|
|
ctx = _mock_ctx()
|
|
report = CheckReport(
|
|
verdicts=[CheckVerdict(item_text="x", kind="verify", status="passed", evidence="e")]
|
|
)
|
|
|
|
structured = MagicMock()
|
|
structured.ainvoke = AsyncMock(return_value=report)
|
|
llm = MagicMock()
|
|
llm.with_structured_output.return_value = structured
|
|
response = MagicMock()
|
|
response.tool_calls = []
|
|
|
|
with (
|
|
patch("artemis.agents.checker.checker.get_llm", return_value=llm),
|
|
patch(
|
|
"artemis.agents.checker.checker.acomplete",
|
|
new=AsyncMock(return_value=response),
|
|
),
|
|
patch(
|
|
"artemis.agents.checker.checker._capture_final_screen",
|
|
new=AsyncMock(return_value=(None, "SHOULD NOT BE CALLED")),
|
|
) as capture_mock,
|
|
):
|
|
result = await run_checkpoint_check(
|
|
ctx,
|
|
check_items=[_ci(text="x")],
|
|
anchor=EvidenceAnchor(anchor_step_id="sid", trigger_ts=0.0, plan_text="- [x] G"),
|
|
goal="the goal",
|
|
subgoal_text="G",
|
|
)
|
|
|
|
capture_mock.assert_not_awaited()
|
|
assert result.verdicts[0].status == "passed"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_final_entry_captures_live_screen():
|
|
ctx = _mock_ctx()
|
|
report = CheckReport(verdicts=[])
|
|
|
|
structured = MagicMock()
|
|
structured.ainvoke = AsyncMock(return_value=report)
|
|
llm = MagicMock()
|
|
llm.with_structured_output.return_value = structured
|
|
response = MagicMock()
|
|
response.tool_calls = []
|
|
|
|
with (
|
|
patch("artemis.agents.checker.checker.get_llm", return_value=llm),
|
|
patch(
|
|
"artemis.agents.checker.checker.acomplete",
|
|
new=AsyncMock(return_value=response),
|
|
),
|
|
patch(
|
|
"artemis.agents.checker.checker._capture_final_screen",
|
|
new=AsyncMock(return_value=("b64img", "elements")),
|
|
) as capture_mock,
|
|
):
|
|
await run_final_check(
|
|
ctx,
|
|
goal="the goal",
|
|
plan_text="- [x] G",
|
|
ledger=[],
|
|
check_items=[_ci(kind="assert", when="at_end", text="no crash", parent=None)],
|
|
)
|
|
|
|
capture_mock.assert_awaited_once()
|