feat: add agentic video analysis and clarify action results

Report device actions as dispatched and preserve full failure details. Route
Flash through McpActionExecutor and use ToolFailure across helper tools and
agent loops instead of inferring failure from response text.

Require model-provided descriptions for coordinate targets. Keep observed
element metadata separate from those descriptions in safety checks, visual
summaries, history, and frontend labels.

Add Gemini Interactions API video analysis with configurable processing mode,
chunk size, and timeouts. Centralize Google provider capabilities and token
usage normalization, and update google-genai to 2.22.0.

Preserve system instructions on follow-up interactions, replan pending chunks
when falling back to static processing, and reject unanswered native video
and audio segments instead of caching placeholder results. Keep helper-tool
failure status intact through history tools and committee calls.

Validation: 1,762 Python unit tests passed (3 skipped), 40 frontend formatter
tests passed, and the production UI build completed. Ruff lint, formatting,
the quality ratchet, and git diff checks passed.
This commit is contained in:
somew1nd committed 2026-09-08 17:05:54 -07:00
1 parent 611d1a0947
commit a45daf23f8
120 files changed
+6301 -1710

No files matched your search

+127
View File
@@ -14,12 +14,15 @@
from unittest.mock import AsyncMock, MagicMock, patch
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
from langchain_core.tools import StructuredTool
import pytest
from artemis.agents.checker.checker import (
CheckReport,
CheckVerdict,
_normalize_report,
_run_check_loop,
assemble_checker_prompt_segments,
build_checker_tools,
build_probe_argv,
@@ -28,6 +31,7 @@ from artemis.agents.checker.checker import (
verdicts_allow_release,
)
from artemis.context import ArtemisContext, ExecutionSetup
from artemis.core.tool_failure import ToolFailure
from artemis.graph.checkpoints import EvidenceAnchor
from artemis.utils.plan_grammar import CheckItem
@@ -569,3 +573,126 @@ async def test_no_transcript_without_attempt_id_or_streamed_text(tmp_path):
assert _read_streams(tmp_path) == []
assert not (tmp_path / "check_streams.jsonl").exists()
# --- Probe failures are structural; history rows keep target provenance ---------------
@pytest.mark.asyncio
async def test_probe_device_invalid_kind_is_a_tool_failure():
"""A refused probe answers with a ToolFailure, so the tool loop marks the
ToolMessage as an error structurally rather than by sniffing its wording."""
from artemis.agents.checker.checker import get_probe_tool
from artemis.core.tool_failure import is_tool_failure
tool = get_probe_tool(_mock_ctx(disable_device_probes=False))
result = await tool.ainvoke({"kind": "shell", "params": None})
assert is_tool_failure(result)
assert "Error" in str(result)
@pytest.mark.asyncio
async def test_probe_device_execution_error_is_a_tool_failure():
from artemis.agents.checker.checker import get_probe_tool
from artemis.core.tool_failure import is_tool_failure
ctx = _mock_ctx(disable_device_probes=False)
ctx.get_adb_client.side_effect = RuntimeError("no adb")
tool = get_probe_tool(ctx)
result = await tool.ainvoke({"kind": "battery", "params": None})
assert is_tool_failure(result)
assert "no adb" in str(result)
def test_format_history_marks_self_described_targets_only():
"""A step without a summary renders its action through the shared renderer,
so the Checker sees which target labels are the model's own statement."""
from artemis.agents.checker.checker import _format_history
from artemis.utils.task_tree import SELF_DESCRIBED_MARKER
ctx = _mock_ctx()
ctx.data_engine.get_agent_friendly_steps.return_value = [
{
"step_number": 1,
"relative_time": "T+00:01",
"summary": "",
"action_taken": {
"action": "click",
"coordinates": [500, 600],
"target_description": "play button",
},
},
{
"step_number": 2,
"relative_time": "T+00:05",
"summary": None,
"action_taken": {
"action": "click",
"coordinates": [100, 200],
"target_text": "Login",
"target_resource_id": "com.app:id/login",
},
},
]
text = _format_history(ctx)
lines = text.splitlines()
assert lines[0].startswith("- Step 1 (T+00:01): ")
assert f"'play button' {SELF_DESCRIBED_MARKER}" in lines[0]
assert lines[1].startswith("- Step 2 (T+00:05): ")
assert "'Login'" in lines[1]
assert SELF_DESCRIBED_MARKER not in lines[1]
# Neither row is the raw JSON record.
assert "target_description" not in text
assert "target_resource_id" not in text
# --- tool results: status is structural, never sniffed from the words -----------------
# A helper tool reports failure structurally (``ToolFailure``); free-form text
# that merely starts with "Error" is an ordinary answer.
_STATUS_CASES = [
pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"),
pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"),
]
def _text_tool(name: str, result):
async def _run(key: str) -> str:
return result
return StructuredTool.from_function(coroutine=_run, name=name, description=name)
@pytest.mark.asyncio
@pytest.mark.parametrize("result, expected_status", _STATUS_CASES)
async def test_check_loop_tool_message_status_is_structural(result, expected_status):
ctx = _mock_ctx()
tool = _text_tool("read_note", result)
tool_turn = AIMessage(
content="", tool_calls=[{"name": "read_note", "args": {"key": "progress"}, "id": "c1"}]
)
final_turn = AIMessage(content="done", tool_calls=[])
llm = MagicMock()
llm.bind_tools.return_value = llm
llm.with_structured_output.return_value.ainvoke = AsyncMock(
return_value=CheckReport(verdicts=[])
)
messages = [SystemMessage(content="s"), HumanMessage(content="h")]
with (
patch("artemis.agents.checker.checker.get_llm", return_value=llm),
patch(
"artemis.agents.checker.checker.acomplete",
new=AsyncMock(side_effect=[tool_turn, final_turn]),
),
):
await _run_check_loop(ctx, messages, [tool], check_items=[])
tool_msgs = [m for m in messages if isinstance(m, ToolMessage)]
assert len(tool_msgs) == 1
assert tool_msgs[0].tool_call_id == "c1"
assert tool_msgs[0].status == expected_status
assert tool_msgs[0].content == str(result)
+39
View File
@@ -14,8 +14,10 @@
from unittest.mock import AsyncMock, Mock, patch
from langchain_core.messages import AIMessage, ToolMessage
from langchain_core.tools import StructuredTool
from artemis.agents.diagnoser.diagnoser import Diagnoser
from artemis.context import ArtemisContext
from artemis.core.tool_failure import ToolFailure
from artemis.graph.state import State
import pytest
@@ -293,3 +295,40 @@ async def test_diagnoser_background_job_lifecycle(
assert job["status"] == "completed"
assert job["result"] == "Video shows step successfully completed."
assert job["consumed"] is True
# --- tool results: status is structural, never sniffed from the words -----------------
# A helper tool reports failure structurally (``ToolFailure``); free-form text
# that merely starts with "Error" is an ordinary answer.
_STATUS_CASES = [
pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"),
pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"),
]
def _text_tool(name: str, result):
async def _run(key: str) -> str:
return result
return StructuredTool.from_function(coroutine=_run, name=name, description=name)
@pytest.mark.asyncio
@pytest.mark.parametrize("result, expected_status", _STATUS_CASES)
async def test_run_tool_message_status_is_structural(
result, expected_status, mock_context, mock_state
):
agent = Diagnoser(mock_context)
tool = _text_tool("read_note", result)
messages = await agent._run_tool(
{"name": "read_note", "args": {"key": "progress"}, "id": "c1"}, [tool], mock_state
)
assert len(messages) == 1
assert isinstance(messages[0], ToolMessage)
assert messages[0].tool_call_id == "c1"
assert messages[0].status == expected_status
assert messages[0].content == str(result)
+3 -2
View File
@@ -17,6 +17,7 @@ from adbutils import AdbClient
from langchain_core.messages import AIMessage, ToolMessage
from artemis.agents.diagnoser.diagnoser import Diagnoser
from artemis.context import ArtemisContext
from artemis.core.tool_failure import ToolFailure
from artemis.graph.state import State
import pytest
@@ -107,7 +108,7 @@ async def test_maestro_uninstall_timeout(
mock_ui_tool = AsyncMock()
mock_ui_tool.name = "get_ui_hierarchy"
mock_ui_tool.args = {}
mock_ui_tool.ainvoke.return_value = "Error: pm list packages timed out"
mock_ui_tool.ainvoke.return_value = ToolFailure("Error: pm list packages timed out")
mock_get_ui_tool.return_value = mock_ui_tool
# Run agent
@@ -255,7 +256,7 @@ async def test_ui_hierarchy_dump_stall(
mock_ui_tool = AsyncMock()
mock_ui_tool.name = "get_ui_hierarchy"
mock_ui_tool.args = {}
mock_ui_tool.ainvoke.return_value = (
mock_ui_tool.ainvoke.return_value = ToolFailure(
"Error retrieving UI hierarchy: dump_hierarchy timed out after 3 seconds"
)
mock_get_ui_tool.return_value = mock_ui_tool
@@ -37,7 +37,7 @@ STEPS = [
{
"step_number": 11,
"action_taken": [{"action": "click", "coordinates": [540, 1800], "target_text": "player"}],
"last_execution_result": {"status": "success"},
"last_execution_result": {"status": "dispatched"},
},
{
"step_number": 12,
@@ -98,6 +98,20 @@ def test_open_incident_continues_the_previous_count():
# --- rendering ----------------------------------------------------------------------
def test_render_labels_a_described_coordinate_target_by_its_description():
incident = _disappeared()
incident["action"] = {
"action": "tap",
"coordinates": [950, 288],
"normalized_coordinates": [880, 120],
"target_description": "skip ad button",
}
text = render_execution_incident(incident, STEPS)
assert 'normalized [880, 120] "skip ad button" (self-described)' in text
# An observed (index) target carries no marker.
assert '"Skip" (self-described)' not in render_execution_incident(_disappeared(), STEPS)
def test_render_disappeared_names_target_trigger_and_burst_exception():
text = render_execution_incident(_disappeared(), STEPS)
assert text.startswith(EXECUTION_INCIDENT_MARKER)
@@ -106,7 +120,7 @@ def test_render_disappeared_names_target_trigger_and_burst_exception():
# Recorded target with normalized coordinates and text.
assert 'normalized [880, 120] "Skip"' in text
# The last successful action before the incident is the trigger candidate.
assert "Your last successfully executed action was `Tapped 'player' at [540, 1800]`" in text
assert "Your last dispatched action was `Tapped 'player' at [540, 1800]`" in text
assert "(Step 11)" in text
# Facts and evidence only: the response protocol is stated once, in the
# static system prompt, so the per-turn block never restates it.
@@ -145,7 +159,7 @@ def test_render_burst_abort_and_exec_error():
def test_render_without_history_omits_the_trigger_hint():
text = render_execution_incident(_disappeared(), [])
assert "last successfully executed action" not in text
assert "last dispatched action" not in text
assert 'Its recorded target was normalized [880, 120] "Skip".' in text
+43 -2
View File
@@ -143,14 +143,17 @@ def test_flash_runner_screenshot_pruning(mock_context):
# --- Recorded action shape (coordinate space + ledger rendering) ---------------------
def _record(runner, name, args):
def _record(runner, name, args, metadata=None):
from types import SimpleNamespace
runner.ctx.data_engine = Mock()
runner.ctx.data_engine.current_step_id = None
runner.ctx.data_engine.record_step.return_value = "step-1"
exec_result = SimpleNamespace(
status="success", text_summary=f"{name} executed", metadata={}, ui_elements_text="ui"
status="success",
text_summary=f"{name} executed",
metadata=metadata or {},
ui_elements_text="ui",
)
step_id = runner._record_action_step(
name, args, exec_result, "thought", {}, b"pre", ["xml"], b"post"
@@ -205,3 +208,41 @@ def test_flash_records_render_in_the_ledger_with_their_arguments(mock_context):
assert format_actions_clean(swipe_fa) == "Swiped down"
swipe_coords = _record(runner, "swipe", {"coordinates": [556, 289, 556, 124]})
assert format_actions_clean(swipe_coords) == "Swiped from [556, 289] to [556, 124]"
def test_flash_records_the_executors_target_semantics(mock_context):
"""Flash targets are coordinates, so the model's own description (validated
and shaped by the executor into ``metadata.target_semantics``) is the only
target semantics on record: top level, the shape Pro records, rendered as the
action's label. Nothing is inferred when the executor hands back nothing."""
from artemis.utils.task_tree import format_actions_clean
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="g")
click_args = {"target": [320, 399], "target_description": "play button"}
click = _record(
runner,
"click",
click_args,
metadata={"target_semantics": {"target_description": "play button"}},
)
assert click["target_description"] == "play button"
assert "target_text" not in click
assert format_actions_clean(click) == "Tapped 'play button' (self-described) at [320, 399]"
burst = _record(
runner,
"click_sequence",
{"sequence": [[500, 280], [885, 362]], "target_descriptions": ["video body", "skip"]},
metadata={"target_semantics": {"target_descriptions": ["video body", "skip"]}},
)
assert burst["target_descriptions"] == ["video body", "skip"]
assert format_actions_clean(burst) == (
"Tapped sequence of targets: 'video body' (self-described) at [500, 280],"
" 'skip' (self-described) at [885, 362]"
)
bare = _record(runner, "click", {"target": [320, 399]}, metadata={"target_semantics": {}})
assert "target_description" not in bare
assert "target_text" not in bare
+90 -3
View File
@@ -31,7 +31,7 @@ from unittest.mock import AsyncMock, Mock, patch
import pytest
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
from artemis.agents.flash.runner import FlashRunner
from artemis.agents.flash.runner import FlashRunner, _TurnRecord
from artemis.agents.validator.tool_declarations import ToolExecutionResult
from artemis.context import ArtemisContext
from artemis.graph.state import State
@@ -43,7 +43,7 @@ from artemis.memory.transcript import (
OBSERVATION_HEADER_RE = re.compile(r"^# CURRENT OBSERVATION \[T\+\d{2,}:\d{2}\]$")
RESULT_RE = re.compile(
rf"^{re.escape(EXECUTION_RESULT_MARKER)} \(T\+\d{{2,}}:\d{{2}}\) ---\nStatus: success$"
rf"^{re.escape(EXECUTION_RESULT_MARKER)} \(T\+\d{{2,}}:\d{{2}}\) ---\nStatus: dispatched$"
)
@@ -198,6 +198,44 @@ def test_observation_tail_has_pro_shape(mock_context):
]
# ---------------------------------------------------------------------------
# Injected instruction: verbatim text for the record and the lens, wrapped
# notice for the observation tail only
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_injected_instruction_splits_verbatim_text_from_operator_notice(
mock_context, tmp_path
):
"""The user's words are what the step record, the chunk ledger and the
visual lens receive (Pro parity); the 'You MUST immediately ...' directive
is operator-facing and rides on the observation tail alone."""
import json
(tmp_path / "injected_instruction.json").write_text(
json.dumps({"instruction": "Skip the popup and log in", "release_loop": True}),
encoding="utf-8",
)
mock_context.data_engine = Mock()
mock_context.data_engine.base_dir = str(tmp_path)
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="Log in")
instruction, notice = await runner._read_injected_instruction()
assert instruction == "Skip the popup and log in"
assert notice.startswith(
"[REAL-TIME INJECTED INSTRUCTION from user]: Skip the popup and log in"
)
assert "You MUST immediately follow this instruction" in notice
assert "authorized stopping any ongoing monitoring loop" in notice
assert not (tmp_path / "injected_instruction.json").exists()
with patch("artemis.controllers.unified_controller.get_driver"):
assert await runner._read_injected_instruction() == (None, None)
# ---------------------------------------------------------------------------
# End-to-end loop over the ledger (mocked model / executor)
# ---------------------------------------------------------------------------
@@ -258,7 +296,11 @@ async def test_run_builds_prompt_from_ledger_with_session_offsets(mock_context):
AIMessage(
content="I will tap then wait.",
tool_calls=[
{"name": "click", "args": {"target": [500, 600]}, "id": "tc1"},
{
"name": "click",
"args": {"target": [500, 600], "target_description": "Settings"},
"id": "tc1",
},
{"name": "wait_for_delay", "args": {"seconds": 1}, "id": "tc2"},
],
),
@@ -358,3 +400,48 @@ async def test_run_no_tool_call_turn_is_nudged_not_terminated(mock_context):
and any(EXECUTION_RESULT_MARKER in b.get("text", "") for b in m.content)
for m in turn2_messages
)
# ---------------------------------------------------------------------------
# _TurnRecord.result: the turn's execution result in the validator-report shape
# ---------------------------------------------------------------------------
def test_turn_record_result_is_none_for_helper_only_turns():
assert _TurnRecord().result() is None
def test_turn_record_result_is_dispatched_when_every_action_was_accepted():
record = _TurnRecord(
actions=[("click", "success", "Tapped at [500, 500]"), ("swipe", "success", "Swiped up.")]
)
assert record.result() == {"status": "dispatched"}
def test_turn_record_failed_result_keeps_an_executor_message_that_names_the_action():
"""Executor messages already name the action; they are not prefixed again."""
record = _TurnRecord(
actions=[
("click", "success", "Tapped at [500, 500]"),
("swipe", "error", "Error executing swipe: direction 'sideways' is not valid"),
]
)
assert record.result() == {
"status": "failed",
"error": "Error executing swipe: direction 'sideways' is not valid",
}
def test_turn_record_failed_result_prefixes_an_error_that_does_not_name_the_action():
record = _TurnRecord(actions=[("type_text", "error", "Device is offline")])
assert record.result() == {"status": "failed", "error": "type_text: Device is offline"}
def test_turn_record_reports_the_first_failure_of_a_multi_action_turn():
record = _TurnRecord(
actions=[
("click", "error", "click rejected: coordinates out of bounds"),
("swipe", "error", "never reached"),
]
)
assert record.result()["error"] == "click rejected: coordinates out of bounds"
+1 -1
View File
@@ -316,4 +316,4 @@ def test_untracked_messages_are_never_touched(mock_context):
assert messages[0].content == "[REAL-TIME INJECTED INSTRUCTION from user]: stay put"
assert messages[2].content == "thinking"
# Image-only tool message froze with the defensive fallback text.
assert messages[1].content == [{"type": "text", "text": "Action completed."}]
assert messages[1].content == [{"type": "text", "text": "Action dispatched."}]
+176 -1
View File
@@ -20,7 +20,13 @@ import pytest
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from artemis.agents.flash.context_compressor import ScrubEdgeCompressor
from artemis.agents.flash.summarizer import VisualStepSummarizer
from artemis.agents.flash.summarizer import (
FOCUS_BLOCK_HEADER,
FOCUS_INTENT_MAX_CHARS,
VisualStepSummarizer,
build_focus_context,
render_focus_block,
)
from artemis.context import ArtemisContext
from artemis.sdk.builders import Builders
@@ -560,6 +566,175 @@ async def test_raw_lens_call_meters_llm_usage_without_touching_context_base(mock
assert meter.last_prompt_tokens == 777
@pytest.mark.asyncio
async def test_focus_block_and_provenance_marked_action_phrase_reach_the_lens(mock_context):
"""The lens input opens with the action as every history reader sees it
(self-described target marked), the verbatim arguments, the outcome, and
the operator focus block; the system prompt teaches how to use it."""
summarizer = VisualStepSummarizer(mock_context)
captured: list = []
async def mock_ainvoke(messages):
captured.append(messages)
return AIMessage(
content=(
"In Step 3, I tapped the switch marked in red on the 'Wi-Fi' row, which"
" read 'Off' before; afterwards it showed 'On' and the list stayed empty."
)
)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
summarizer.dispatch(
step_number=3,
action_name="click",
action_args={"target": [880, 410], "target_description": "Wi-Fi toggle"},
pre_img_bytes=b"pre",
post_img_bytes=b"post",
exec_outcome="Tapped at [880, 410] (normalized).",
focus=build_focus_context(
intent="The Wi-Fi row shows Off. I will tap its switch and expect it to read On.",
goal="Turn on Wi-Fi",
subgoal="Enable Wi-Fi > Tap the Wi-Fi switch",
injected_instruction="Do not join any network yet.",
),
)
await summarizer.flush()
assert summarizer.has_summary(3)
system_text = str(captured[0][0].content)
lead_text = captured[0][1].content[0]["text"]
assert "# OPERATOR FOCUS" in system_text
assert "Absence is a fact" in system_text
assert lead_text.startswith(
"Step 3 Physical Action: Tapped 'Wi-Fi toggle' (self-described) at [880, 410]\n"
)
assert "Action arguments (verbatim): click({'target': [880, 410]" in lead_text
assert "Controller Outcome: Tapped at [880, 410] (normalized)." in lead_text
assert FOCUS_BLOCK_HEADER in lead_text
assert "Task goal: Turn on Wi-Fi" in lead_text
assert "Active sub-goal: Enable Wi-Fi > Tap the Wi-Fi switch" in lead_text
assert "User instruction at this step: Do not join any network yet." in lead_text
assert "expect it to read On." in lead_text
# The focus precedes the frames: context first, evidence second.
assert captured[0][1].content[1]["text"].startswith("--- [1] BEFORE ACTION SCREEN")
@pytest.mark.asyncio
async def test_dispatch_without_focus_carries_no_focus_block(mock_context):
"""Callers without operator context (older call sites, tests) get the
plain lead block: no focus header, observed target rendered unmarked."""
summarizer = VisualStepSummarizer(mock_context)
captured: list = []
async def mock_ainvoke(messages):
captured.append(messages)
return AIMessage(
content="In Step 1, I tapped the 'Search' field marked in red; the keyboard opened."
)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
summarizer.dispatch(
step_number=1,
action_name="tap",
action_args={"coordinates": [500, 600], "target_text": "Search"},
pre_img_bytes=b"pre",
post_img_bytes=None,
exec_outcome="dispatched",
)
await summarizer.flush()
lead_text = captured[0][1].content[0]["text"]
assert lead_text.startswith("Step 1 Physical Action: Tapped 'Search' at [500, 600]\n")
assert "(self-described)" not in lead_text
assert FOCUS_BLOCK_HEADER not in lead_text
assert summarizer.get_job_payload(1)["focus"] is None
@pytest.mark.asyncio
async def test_single_frame_focus_label_forbids_describing_the_outcome(mock_context):
"""With one decision frame (every Pro step) the reasoning is labelled as the
operator's aim only, never as an expected screen change to narrate."""
summarizer = VisualStepSummarizer(mock_context)
captured: list = []
async def mock_ainvoke(messages):
captured.append(messages)
return AIMessage(
content="In Step 2, I tapped the 'Pay' button marked in red on a checkout screen totalling '$42.00'."
)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
summarizer.dispatch(
step_number=2,
action_name="tap",
action_args={"coordinates": [500, 900], "target_text": "Pay"},
pre_img_bytes=b"decision_frame",
post_img_bytes=None,
exec_outcome="dispatched",
focus=build_focus_context(intent="Tap Pay; I expect a confirmation dialog."),
)
await summarizer.flush()
lead_text = captured[0][1].content[0]["text"]
assert "the expected outcome is NOT in this frame and must not be described" in lead_text
assert "what screen change it expected" not in lead_text
# The focus text is released once the summary landed, like the frames.
assert summarizer.get_job_payload(2)["focus"] is None
def test_action_phrase_renders_flash_swipe_points_and_bursts():
"""Flash's ``start``/``end`` swipe points and a Pro burst's additional
actions render through the shared history phrase, never as None."""
phrase = VisualStepSummarizer._action_phrase(
"swipe", {"start": [500, 800], "end": [500, 200], "target_description": "video list"}
)
assert phrase == "Swiped 'video list' (self-described) from [500, 800] to [500, 200]"
assert VisualStepSummarizer._action_phrase("swipe", {"direction": "up"}) == "Swiped up"
burst = VisualStepSummarizer._action_phrase(
"tap",
{
"coordinates": [100, 200],
"target_description": "Skip ad",
"additional_actions": [
{"action": "tap", "coordinates": [300, 400], "target_text": "Play"}
],
},
)
assert burst.startswith("Fast-action burst (2 actions, unvetted): ")
assert "Tapped 'Skip ad' (self-described) at [100, 200] -> Tapped 'Play' at [300, 400]" in burst
def test_build_focus_context_drops_blank_fields_and_caps_reasoning_tail_intact():
"""Blank/None fields never appear; an over-long reasoning keeps its tail
(where the operator states the expected screen change) and announces
the cut in place."""
assert (
build_focus_context(intent=" ", goal=None, subgoal="", injected_instruction=None) is None
)
head = "H" * 500
tail = "I expect the toggle to read On."
filler = "x" * (FOCUS_INTENT_MAX_CHARS * 2)
focus = build_focus_context(intent=head + filler + tail, goal="Turn on Wi-Fi")
assert set(focus) == {"goal", "intent"}
assert focus["intent"].startswith("HHHH")
assert focus["intent"].endswith(tail)
assert "characters cut here" in focus["intent"]
assert len(focus["intent"]) < FOCUS_INTENT_MAX_CHARS + 80
block = render_focus_block(focus)
assert block.splitlines()[0] == FOCUS_BLOCK_HEADER
assert "Task goal: Turn on Wi-Fi" in block
assert "Active sub-goal" not in block
def test_flash_config_and_builder():
"""Verify Flash profile configuration model and SDK builder fluent API."""
cfg = Builders.AgentConfig.with_flash_config(
@@ -16,8 +16,10 @@ from pathlib import Path
from unittest.mock import MagicMock, patch
from langchain_core.messages import ToolMessage
from langchain_core.tools import StructuredTool
from artemis.agents.history_analyzer.history_analyzer import HistoryAnalyzer
from artemis.context import ArtemisContext
from artemis.core.tool_failure import ToolFailure
import pytest
@@ -374,3 +376,65 @@ async def test_history_analyzer_integration_with_task_tree():
assert args[3] == expected_hash
assert kwargs.get("engine") is mock_ctx.data_engine
# --- tool results: status is structural, never sniffed from the words -----------------
# A helper tool reports failure structurally (``ToolFailure``); free-form text
# that merely starts with "Error" is an ordinary answer.
_STATUS_CASES = [
pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"),
pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"),
]
def _text_tool(name: str, result):
async def _run(key: str) -> str:
return result
return StructuredTool.from_function(coroutine=_run, name=name, description=name)
@pytest.mark.asyncio
@pytest.mark.parametrize("result, expected_status", _STATUS_CASES)
async def test_history_analyzer_tool_message_status_is_structural(result, expected_status):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
mock_ctx.data_engine.get_agent_friendly_steps.return_value = [
{"step_number": 1, "relative_time": "1.2s", "summary": "Opened the settings app"}
]
tool_turn = MagicMock(content="")
tool_turn.tool_calls = [{"name": "read_note", "args": {"key": "progress"}, "id": "c1"}]
final_turn = MagicMock(content="done")
final_turn.tool_calls = []
responses = [tool_turn, final_turn]
async def mock_astream(*args, **kwargs):
yield responses.pop(0)
mock_llm = MagicMock()
mock_llm.bind_tools.return_value = mock_llm
mock_llm.astream.side_effect = mock_astream
with (
patch(
"artemis.agents.history_analyzer.history_analyzer.get_llm",
return_value=mock_llm,
),
patch.object(
HistoryAnalyzer, "_build_tools", return_value=[_text_tool("read_note", result)]
),
patch("pathlib.Path.exists", return_value=False),
):
answer = await HistoryAnalyzer(mock_ctx).run("What did the note say?")
assert answer == "done"
second_turn_messages = mock_llm.astream.call_args_list[1][0][0]
tool_msgs = [m for m in second_turn_messages if isinstance(m, ToolMessage)]
assert len(tool_msgs) == 1
assert tool_msgs[0].tool_call_id == "c1"
assert tool_msgs[0].status == expected_status
assert tool_msgs[0].content == str(result)
+78
View File
@@ -0,0 +1,78 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""LogAnalyzerNode's ReAct loop: a helper tool's failure is structural
(``ToolFailure``), never sniffed from the words of its text."""
from unittest.mock import AsyncMock, MagicMock, patch
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
from langchain_core.tools import StructuredTool
import pytest
from artemis.agents.log_analyzer.log_analyzer import LogAnalyzerNode
from artemis.context import ArtemisContext
from artemis.core.tool_failure import ToolFailure
_STATUS_CASES = [
pytest.param(ToolFailure("Error searching logs: device gone"), "error", id="tool_failure"),
pytest.param(
"Error: 12 matching log lines\nE/Foo: Error opening file", "success", id="plain_error_text"
),
]
def _text_tool(name: str, result):
async def _run(keyword: str) -> str:
return result
return StructuredTool.from_function(coroutine=_run, name=name, description=name)
@pytest.mark.asyncio
@pytest.mark.parametrize("result, expected_status", _STATUS_CASES)
async def test_agent_loop_tool_message_status_is_structural(result, expected_status):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = None
node = LogAnalyzerNode(ctx=ctx)
tool = _text_tool("search_logs", result)
base_llm = MagicMock()
base_llm.bind_tools.return_value = base_llm
tool_turn = AIMessage(
content="", tool_calls=[{"name": "search_logs", "args": {"keyword": "Error"}, "id": "c1"}]
)
final_turn = AIMessage(content="Summary.", tool_calls=[])
messages = [SystemMessage(content="s"), HumanMessage(content="h")]
with patch(
"artemis.agents.log_analyzer.log_analyzer.acomplete",
new=AsyncMock(side_effect=[tool_turn, final_turn]),
):
outcome = await node._run_agent_loop(
base_llm=base_llm,
tools_for_binding=[tool],
tools=[tool],
current_messages=messages,
state=MagicMock(),
max_iterations=5,
agent_name="Log Analyzer",
)
assert outcome == "Summary."
tool_msgs = [m for m in messages if isinstance(m, ToolMessage)]
assert len(tool_msgs) == 1
assert tool_msgs[0].tool_call_id == "c1"
assert tool_msgs[0].status == expected_status
assert tool_msgs[0].content == str(result)
+202 -37
View File
@@ -18,6 +18,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch
from artemis.agents.operator.operator import OperatorNode
from artemis.config.agent import MemoryTranscriptConfig
from artemis.context import ArtemisContext
from artemis.core.tool_failure import ToolFailure
import pytest
# These tests exercise the legacy 2-message prompt path. Since M5 the
@@ -54,7 +55,11 @@ async def test_operator_node_fast_path():
mock_response.tool_calls = [
{
"name": "click",
"args": {"target": [50, 50], "reasoning": "Test reasoning"},
"args": {
"target": [50, 50],
"target_description": "button",
"reasoning": "Test reasoning",
},
"id": "call_123",
}
]
@@ -121,7 +126,6 @@ async def test_perform_action_validation():
"target_bounds": "[100,100][300,500]",
"target_resource_id": "btn_1",
"target_class": "android.widget.Button",
"target_label_source": "index",
}
]
@@ -152,7 +156,6 @@ async def test_perform_action_validation():
"target_bounds": "[300,400][500,600]",
"target_resource_id": "input_1",
"target_class": "android.widget.EditText",
"target_label_source": "index",
}
# Test swipe direction
@@ -181,7 +184,6 @@ async def test_perform_action_validation():
"target_bounds": "[300,400][500,600]",
"target_resource_id": "input_1",
"target_class": "android.widget.EditText",
"target_label_source": "index",
}
]
@@ -235,8 +237,13 @@ async def test_perform_action_validation():
@pytest.mark.asyncio
async def test_bare_coordinate_click_enriched_by_hit_test():
"""A coordinate click hit tests the pre-action frame for element semantics."""
async def test_coordinate_click_requires_and_records_the_models_description():
"""A coordinate target carries the model's own description, never an inferred label.
Even when an indexed element sits right under the point, nothing is hit
tested on the model's behalf: the recorded semantics are exactly what the
model declared, kept apart from the observed ``target_*`` fields.
"""
from artemis.agents.operator.operator import OperatorNode
from unittest.mock import MagicMock
from artemis.graph.state import State
@@ -256,10 +263,22 @@ async def test_bare_coordinate_click_enriched_by_hit_test():
}
]
# Normalized (500, 600) on the default 1080x2400 device -> pixel (540, 1440),
# which falls inside the "Confirm" button bounds.
# Coordinates without a description are refused with a retryable error.
for args in (
{"target": [500, 600]},
{"target": [500, 600], "target_description": " "},
{"target": [500, 600], "target_description": 12},
):
actions, err = node._translate_and_validate_tool(
{"name": "click", "args": args}, mock_state
)
assert actions == []
assert err.startswith("Error:")
assert "target_description" in err
actions, err = node._translate_and_validate_tool(
{"name": "click", "args": {"target": [500, 600]}}, mock_state
{"name": "click", "args": {"target": [500, 600], "target_description": " play button "}},
mock_state,
)
assert err is None
assert actions == [
@@ -270,22 +289,59 @@ async def test_bare_coordinate_click_enriched_by_hit_test():
"normalized_coordinates": [500, 600],
"times": 1,
"delay_ms": 100,
"target_text": "Confirm",
"target_bounds": [500, 1400, 600, 1480],
"target_resource_id": "btn_confirm",
"target_class": "android.widget.Button",
"target_label_source": "hit_test",
"target_description": "play button",
}
]
# Graceful degradation: no perception data at all -> bare coordinates, "none".
mock_state.indexed_elements = []
# The same rule applies to input_text and the description is ignored for an index.
actions, err = node._translate_and_validate_tool(
{"name": "click", "args": {"target": [500, 600]}}, mock_state
{"name": "input_text", "args": {"text": "hi", "target": [500, 600]}}, mock_state
)
assert actions == [] and "target_description" in err
actions, err = node._translate_and_validate_tool(
{
"name": "input_text",
"args": {"text": "hi", "target": 1, "target_description": "ignored"},
},
mock_state,
)
assert err is None
assert actions[0]["target_text"] is None
assert actions[0]["target_label_source"] == "none"
assert actions[0]["target_text"] == "Confirm"
assert "target_description" not in actions[0]
@pytest.mark.asyncio
async def test_coordinate_swipe_requires_description_directional_does_not():
from artemis.agents.operator.operator import OperatorNode
from unittest.mock import MagicMock
from artemis.graph.state import State
node = OperatorNode(MagicMock(), transcript_config=LEGACY_TRANSCRIPT)
mock_state = MagicMock(spec=State)
mock_state.indexed_elements = []
mock_state.latest_ui_hierarchy = None
actions, err = node._translate_and_validate_tool(
{"name": "swipe", "args": {"start": [200, 600], "end": [800, 600]}}, mock_state
)
assert actions == [] and "target_description" in err
actions, err = node._translate_and_validate_tool(
{
"name": "swipe",
"args": {"start": [200, 600], "end": [800, 600], "target_description": "volume knob"},
},
mock_state,
)
assert err is None
assert actions[0]["action"] == "swipe"
assert actions[0]["target_description"] == "volume knob"
actions, err = node._translate_and_validate_tool(
{"name": "swipe", "args": {"direction": "up"}}, mock_state
)
assert err is None
assert "target_description" not in actions[0]
@pytest.mark.asyncio
@@ -314,10 +370,14 @@ async def test_operator_node_multiple_actions():
mock_response.tool_calls = [
{
"name": "input_text",
"args": {"text": "hello", "target": [50, 50]},
"args": {"text": "hello", "target": [50, 50], "target_description": "field"},
"id": "call_1",
},
{"name": "click", "args": {"target": [50, 50]}, "id": "call_2"},
{
"name": "click",
"args": {"target": [50, 50], "target_description": "button"},
"id": "call_2",
},
]
mock_llm.ainvoke = AsyncMock(return_value=mock_response)
@@ -587,7 +647,13 @@ async def test_operator_tracks_subagent_calls():
]
mock_response_2 = MagicMock()
mock_response_2.tool_calls = [{"name": "click", "args": {"target": [10, 20]}, "id": "call_2"}]
mock_response_2.tool_calls = [
{
"name": "click",
"args": {"target": [10, 20], "target_description": "button"},
"id": "call_2",
}
]
responses = [mock_response, mock_response_2]
call_count = 0
@@ -659,7 +725,11 @@ async def test_operator_defer_action_for_gathering():
"args": {"query": "check something"},
"id": "call_diagnose",
},
{"name": "click", "args": {"target": [50, 50]}, "id": "call_click"},
{
"name": "click",
"args": {"target": [50, 50], "target_description": "button"},
"id": "call_click",
},
]
# Second response only calls click
@@ -667,7 +737,7 @@ async def test_operator_defer_action_for_gathering():
mock_response_2.tool_calls = [
{
"name": "click",
"args": {"target": [50, 50]},
"args": {"target": [50, 50], "target_description": "button"},
"id": "call_click_final",
}
]
@@ -795,7 +865,6 @@ async def test_long_press_action_translation():
"target_bounds": "[100,100][300,500]",
"target_resource_id": "btn_1",
"target_class": "android.widget.Button",
"target_label_source": "index",
}
]
@@ -803,6 +872,14 @@ async def test_long_press_action_translation():
actions, err = node._translate_and_validate_tool(
{"name": "long_press", "args": {"target": [500, 600]}}, mock_state
)
assert actions == [] and "target_description" in err
actions, err = node._translate_and_validate_tool(
{
"name": "long_press",
"args": {"target": [500, 600], "target_description": "app icon"},
},
mock_state,
)
assert err is None
# Since coordinates are in 0-1000 scale, they should be converted using device width (default 1080) and height (default 2400)
# x = 500 * 1080 / 1000 = 540
@@ -814,11 +891,7 @@ async def test_long_press_action_translation():
"coordinate_space": "pixel",
"normalized_coordinates": [500, 600],
"duration": 1000,
"target_text": None,
"target_bounds": None,
"target_resource_id": None,
"target_class": None,
"target_label_source": "none",
"target_description": "app icon",
}
]
@@ -860,7 +933,11 @@ async def test_operator_no_defer_for_note_updating():
"args": {"key": "progress", "content": "step 1"},
"id": "call_save",
},
{"name": "click", "args": {"target": [50, 50]}, "id": "call_click"},
{
"name": "click",
"args": {"target": [50, 50], "target_description": "button"},
"id": "call_click",
},
]
# We only expect 1 LLM call (astream should only yield once)
@@ -943,7 +1020,11 @@ async def test_operator_defer_for_note_reading():
mock_response_1 = MagicMock()
mock_response_1.tool_calls = [
{"name": "read_note", "args": {"key": "progress"}, "id": "call_read"},
{"name": "click", "args": {"target": [50, 50]}, "id": "call_click"},
{
"name": "click",
"args": {"target": [50, 50], "target_description": "button"},
"id": "call_click",
},
]
# 2nd turn: LLM calls only click after receiving note content
@@ -951,7 +1032,7 @@ async def test_operator_defer_for_note_reading():
mock_response_2.tool_calls = [
{
"name": "click",
"args": {"target": [50, 50]},
"args": {"target": [50, 50], "target_description": "button"},
"id": "call_click_final",
}
]
@@ -1043,7 +1124,11 @@ async def test_operator_no_defer_for_note_appending():
"args": {"key": "progress", "content": "step 1"},
"id": "call_append",
},
{"name": "click", "args": {"target": [50, 50]}, "id": "call_click"},
{
"name": "click",
"args": {"target": [50, 50], "target_description": "button"},
"id": "call_click",
},
]
responses = [mock_response_1]
@@ -1291,7 +1376,7 @@ async def test_operator_fallback_function_call_parsing():
additional_kwargs={
"function_call": {
"name": "click",
"arguments": '{"target": [123, 456]}',
"arguments": '{"target": [123, 456], "target_description": "button"}',
}
},
tool_calls=[],
@@ -1339,7 +1424,12 @@ async def test_operator_swipe_translation():
actions, err = node._translate_and_validate_tool(
{
"name": "swipe",
"args": {"start": [100, 200], "end": [500, 600], "duration": 1500},
"args": {
"start": [100, 200],
"end": [500, 600],
"duration": 1500,
"target_description": "list item",
},
},
mock_state,
)
@@ -1348,12 +1438,13 @@ async def test_operator_swipe_translation():
assert actions[0]["action"] == "swipe"
assert actions[0]["coordinates"] == [108, 480, 540, 1440]
assert actions[0]["duration"] == 1500
assert actions[0]["target_description"] == "list item"
# 3. Coordinates alias
actions, err = node._translate_and_validate_tool(
{
"name": "swipe",
"args": {"coordinates": [200, 300, 400, 500]},
"args": {"coordinates": [200, 300, 400, 500], "target_description": "slider"},
},
mock_state,
)
@@ -1374,3 +1465,77 @@ async def test_operator_swipe_translation():
assert len(actions) == 1
assert actions[0]["action"] == "swipe"
assert actions[0]["coordinates"] == [648, 720, 648, 1680]
# --- helper tool results: status is structural, never sniffed from the words ----------
@pytest.mark.asyncio
@pytest.mark.parametrize(
"result, expected_status",
[
pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"),
pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"),
],
)
async def test_operator_helper_tool_message_status_is_structural(result, expected_status):
from langchain_core.messages import ToolMessage
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.execution_setup = None
mock_ctx.data_engine = None
mock_state = MagicMock()
mock_state.subagent_calls = []
mock_state.initial_goal = "Test goal"
# Turn 1: read_note next to a click. A failed helper rejects the click; a
# deferring helper defers it. Either way turn 2 carries the tool message.
turn_1 = MagicMock()
turn_1.tool_calls = [
{"name": "read_note", "args": {"key": "progress"}, "id": "call_read"},
{
"name": "click",
"args": {"target": [50, 50], "target_description": "button"},
"id": "call_click",
},
]
turn_2 = MagicMock()
turn_2.tool_calls = [
{
"name": "click",
"args": {"target": [50, 50], "target_description": "button"},
"id": "call_click_final",
}
]
mock_llm = MagicMock()
mock_llm.ainvoke = AsyncMock(side_effect=[turn_1, turn_2])
mock_llm.bind_tools.return_value = mock_llm
read_note_tool = MagicMock()
read_note_tool.name = "read_note"
read_note_tool.args = {}
async def dummy_read_note(key: str, **kwargs):
return result
read_note_tool.coroutine = dummy_read_note
read_note_tool.func = None
with (
patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm),
patch(
"artemis.agents.operator.operator.trace_langchain_tool",
side_effect=lambda t, ctx: t,
),
):
node = OperatorNode(mock_ctx, tools=[read_note_tool], transcript_config=LEGACY_TRANSCRIPT)
await node(mock_state)
assert mock_llm.ainvoke.await_count == 2
second_turn_messages = mock_llm.ainvoke.call_args_list[1].args[0]
tool_msgs = {m.tool_call_id: m for m in second_turn_messages if isinstance(m, ToolMessage)}
assert tool_msgs["call_read"].status == expected_status
assert tool_msgs["call_read"].content == str(result)
# The accompanying click is rejected only when the helper actually failed.
assert tool_msgs["call_click"].status == ("error" if expected_status == "error" else "success")
@@ -158,7 +158,12 @@ def _click_response(n_actions=1):
response = MagicMock()
response.content = ""
response.tool_calls = [
{"name": "click", "args": {"target": [50, 50]}, "id": f"call_{i}"} for i in range(n_actions)
{
"name": "click",
"args": {"target": [50, 50], "target_description": "button"},
"id": f"call_{i}",
}
for i in range(n_actions)
]
return response
@@ -53,8 +53,8 @@ from artemis.memory.transcript import (
# SHA-256 snapshots of the legacy system message with the fixed inputs below.
# Update these when an intentional template change alters the rendered prompt.
GOLDEN_EMPTY_PLAN = "2ed35be503e3872f008ae5a12616af6f90cde65f23b63adbb6da782079ae4e7c"
GOLDEN_SENTINEL_PLAN = "8bd6aa60cf3fad8d21cf5c284a01f586c57d209fcd1d2b4a517a791736f9b1de"
GOLDEN_EMPTY_PLAN = "d8cf83565c766d32893b124c974060bbd98de2d862c3b9925e546ea0c946c324"
GOLDEN_SENTINEL_PLAN = "8f871b5e5700ba5ee07ed03b80a8f82664abe68b0c170d3185bf1ee35f7821ed"
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
@@ -198,7 +198,7 @@ async def test_transcript_mode_two_turns_build_four_regions(tmp_path):
_transcript_state(
current_step_id="step-1",
structured_decisions='[{"action": "tap"}]',
last_execution_result={"status": "success"},
last_execution_result={"status": "dispatched"},
)
)
@@ -237,7 +237,7 @@ async def test_transcript_actionless_turn_commits_without_validator_message():
_transcript_state(
current_step_id="step-1",
structured_decisions="",
last_execution_result={"status": "success"},
last_execution_result={"status": "dispatched"},
)
)
+50
View File
@@ -20,6 +20,7 @@ import pytest
from artemis.agents.outputter.outputter import outputter
from artemis.config import LLM, OutputConfig # noqa: E402
from artemis.context import ArtemisContext # noqa: E402
from artemis.core.tool_failure import ToolFailure # noqa: E402
from artemis.utils.logger import get_logger # noqa: E402
logger = get_logger(__name__)
@@ -509,3 +510,52 @@ async def test_outputter_executes_save_note_tool(mock_get_llm, mock_context, moc
# Assert final result
assert result == "Verification code is 123456."
# --- tool results: status is structural, never sniffed from the words -----------------
# A helper tool reports failure structurally (``ToolFailure``); free-form text
# that merely starts with "Error" is an ordinary answer.
_STATUS_CASES = [
pytest.param(ToolFailure("Error: note 'progress' not found"), "error", id="tool_failure"),
pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"),
]
def _text_tool(name: str, result): # noqa: D103
from langchain_core.tools import StructuredTool
async def _run(key: str) -> str:
return result
return StructuredTool.from_function(coroutine=_run, name=name, description=name)
@patch("artemis.agents.outputter.outputter.get_read_note_tool_pure")
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
@pytest.mark.parametrize("result, expected_status", _STATUS_CASES)
async def test_outputter_tool_message_status_is_structural(
mock_get_llm, mock_get_read_note, result, expected_status, mock_context, mock_state
):
from langchain_core.messages import AIMessage, ToolMessage
mock_get_read_note.return_value = _text_tool("read_note", result)
_, mock_llm_with_tools, _ = setup_mock_llm(mock_get_llm)
tool_turn = AIMessage(
content="", tool_calls=[{"name": "read_note", "args": {"key": "progress"}, "id": "c1"}]
)
final_turn = AIMessage(content="The answer.", tool_calls=[])
mock_llm_with_tools.ainvoke.side_effect = [tool_turn, final_turn]
config = OutputConfig(structured_output=None, output_description=None)
answer = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
assert answer == "The answer."
second_turn_messages = mock_llm_with_tools.ainvoke.call_args_list[1].args[0]
tool_msgs = [m for m in second_turn_messages if isinstance(m, ToolMessage)]
assert len(tool_msgs) == 1
assert tool_msgs[0].tool_call_id == "c1"
assert tool_msgs[0].status == expected_status
assert tool_msgs[0].content == str(result)
@@ -64,6 +64,10 @@ def test_coordinates_only_target_is_labelled_as_surface():
first_text = messages[1].content[0]["text"]
assert "Kind: coordinates only" in first_text
# No hit test runs any more: the block states that nothing was recorded, it
# does not claim the point was looked up in the UI hierarchy.
assert "no target metadata recorded" in first_text
assert "UI hierarchy" not in first_text
assert "Label:" not in first_text
assert "Resource ID:" not in first_text
assert not any("Original Thinking" in t for t in _texts(messages))
@@ -77,13 +81,12 @@ def test_label_alone_counts_as_named_control():
assert "Resource ID" not in text
def test_unlabelled_hit_test_element_is_still_a_control():
def test_unlabelled_indexed_element_is_still_a_control():
item = {
"action": "tap",
"coordinates": [1001, 718],
"target_class": "android.widget.ImageView",
"target_bounds": [922, 655, 1080, 781],
"target_label_source": "hit_test",
}
text = pp._describe_target(item)
assert "Kind: specific UI control" in text
@@ -92,9 +95,33 @@ def test_unlabelled_hit_test_element_is_still_a_control():
assert "[922, 655, 1080, 781]" in text
def test_described_coordinate_target_is_its_own_kind():
"""The model's description is presented as its belief, not as observed metadata."""
item = {"action": "tap", "coordinates": [540, 400], "target_description": "play button"}
text = pp._describe_target(item)
assert "Kind: described target" in text
assert "Operator's description: play button" in text
assert "Kind: specific UI control" not in text
assert "Label:" not in text
def test_observed_metadata_outranks_description():
item = {
"action": "tap",
"coordinates": [10, 10],
"target_text": "Submit",
"target_description": "ignored",
}
text = pp._describe_target(item)
assert "Kind: specific UI control" in text
assert "Label: Submit" in text
assert "described target" not in text
def test_prompt_rules_are_keyed_on_target_kind():
prompt = Path(pp.__file__).with_name("pixel_safety_net.md").read_text(encoding="utf-8")
assert "`Kind: specific UI control`" in prompt
assert "`Kind: described target`" in prompt
assert "`Kind: coordinates only`" in prompt
assert "hidden" in prompt
assert "Identify what appears under the red dot" in prompt
+58 -3
View File
@@ -37,12 +37,16 @@ class DummyState:
operator_native_thinking=None,
last_execution_result=None,
current_step_id=None,
initial_goal=None,
injected_instruction=None,
):
self.structured_decisions = structured_decisions
self.operator_raw_thinking = operator_raw_thinking
self.operator_native_thinking = operator_native_thinking
self.last_execution_result = last_execution_result
self.current_step_id = current_step_id
self.initial_goal = initial_goal
self.injected_instruction = injected_instruction
@pytest.fixture
@@ -58,6 +62,7 @@ def mock_context(tmp_path):
post_path.write_bytes(b"POST_IMAGE")
engine = Mock()
engine.base_dir = None # no task_plan on disk: no sub-goal in the focus
engine.get_step_number.return_value = 7
engine.get_step_record.return_value = SimpleNamespace(
action_taken=[
@@ -80,7 +85,7 @@ def mock_context(tmp_path):
async def test_summarizer_dispatches_visual_lens(mock_context):
"""The node dispatches to the shared service instead of calling an LLM."""
state = DummyState(
last_execution_result={"status": "success"},
last_execution_result={"status": "dispatched"},
current_step_id="12345678-1234-5678-1234-567812345678",
)
@@ -98,8 +103,9 @@ async def test_summarizer_dispatches_visual_lens(mock_context):
},
pre_img_bytes=b"PRE_IMAGE",
post_img_bytes=b"POST_IMAGE",
exec_outcome="success",
exec_outcome="dispatched",
data_engine_step_id="12345678-1234-5678-1234-567812345678",
focus=None,
)
# No direct capsule write from the node: the lens owns the versioned
# summary write when its background job completes.
@@ -138,7 +144,7 @@ async def test_summarizer_falls_back_to_structured_decisions(mock_context):
' "coordinate_space": "pixel"},'
' {"action": "tap", "coordinates": [540, 1200], "coordinate_space": "pixel"}]'
),
last_execution_result={"status": "success"},
last_execution_result={"status": "dispatched"},
current_step_id="12345678-1234-5678-1234-567812345678",
)
@@ -153,6 +159,55 @@ async def test_summarizer_falls_back_to_structured_decisions(mock_context):
]
@pytest.mark.asyncio
async def test_summarizer_passes_operator_focus(mock_context, tmp_path):
"""The operator's reasoning, the goal, the injected instruction and the live
sub-goal leaf ride along as the lens's focus context; the record's
reasoning is the fallback when the state carries none."""
notes = tmp_path / "notes"
notes.mkdir()
(notes / "task_plan.md").write_text(
"- [/] Enable Wi-Fi\n - [/] Tap the Wi-Fi switch so it reads On\n- [ ] Join network\n",
encoding="utf-8",
)
mock_context.data_engine.base_dir = str(tmp_path)
mock_context.data_engine.get_step_record.return_value = SimpleNamespace(
action_taken=[
{
"action": "tap",
"coordinates": [950, 984],
"coordinate_space": "pixel",
"target_description": "Wi-Fi switch",
}
],
operator_raw_thinking="Recorded reasoning: the switch reads Off; tapping it should read On.",
)
state = DummyState(
last_execution_result={"status": "dispatched"},
current_step_id="12345678-1234-5678-1234-567812345678",
initial_goal="Turn on Wi-Fi",
injected_instruction="Do not join any network yet.",
)
await SummarizerNode(mock_context)(state)
kwargs = mock_context.step_memory.dispatch.call_args.kwargs
assert kwargs["action_args"]["target_description"] == "Wi-Fi switch"
assert kwargs["focus"] == {
"goal": "Turn on Wi-Fi",
"subgoal": "Enable Wi-Fi > Tap the Wi-Fi switch so it reads On",
"injected_instruction": "Do not join any network yet.",
"intent": "Recorded reasoning: the switch reads Off; tapping it should read On.",
}
# Live state reasoning outranks the record's copy.
mock_context.step_memory.dispatch.reset_mock()
state.operator_raw_thinking = "Live reasoning wins."
await SummarizerNode(mock_context)(state)
focus = mock_context.step_memory.dispatch.call_args.kwargs["focus"]
assert focus["intent"] == "Live reasoning wins."
@pytest.mark.asyncio
async def test_summarizer_noop_without_step_id(mock_context):
state = DummyState(current_step_id=None)
+3 -3
View File
@@ -257,7 +257,7 @@ async def test_validator_burst_success_executes_every_member(
result = await node(state)
report = result["last_execution_result"]
assert report["status"] == "success"
assert report["status"] == "dispatched"
assert report["burst"] is True
assert report["incident"] is None
assert result["open_incident"] is None
@@ -287,7 +287,7 @@ async def test_validator_success_closes_open_incident(mock_mcp, mock_context, te
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
result = await node(state)
assert result["last_execution_result"]["status"] == "success"
assert result["last_execution_result"]["status"] == "dispatched"
assert result["open_incident"] is None
# The closed record is handed over once so the Operator settles the intent.
assert result["last_closed_incident"]["kind"] == "safety_net"
@@ -805,7 +805,7 @@ async def test_validator_pre_execution_validation_ocr_direct_to_pixel(
assert "last_execution_result" in result
report = result["last_execution_result"]
assert report["status"] == "success"
assert report["status"] == "dispatched"
@pytest.mark.asyncio
File diff suppressed because it is too large. Load diff
+160
View File
@@ -14,10 +14,21 @@
import os
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.video_analyzer import chunk_conversation
from artemis.agents.video_analyzer.chunk_conversation import (
_MISSING_SUMMARY_ERROR,
_NO_TOOL_CALL_ERROR,
SubAgentAnswerExhausted,
_ChunkMedia,
_drive_sub_agent_loop,
_run_native_chunk_conversation,
)
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
from artemis.context import ArtemisContext
from google.genai import types
import pytest
@@ -29,6 +40,7 @@ async def test_video_analyzer_run():
mock_ctx.llm_config = MagicMock()
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static"))
# Mock Gemini Client
mock_client = MagicMock()
@@ -80,6 +92,7 @@ async def test_video_analyzer_preserves_thought_signature():
mock_ctx.llm_config = MagicMock()
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static"))
# Mock MobileDeviceController
mock_controller = MagicMock()
@@ -256,6 +269,7 @@ async def test_video_analyzer_sub_agent_confidence_validation():
mock_ctx.llm_config = MagicMock()
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.7-flash"
mock_ctx.agent_config = SimpleNamespace(video_analyzer=SimpleNamespace(processing="static"))
# Mock MobileDeviceController
mock_controller = MagicMock()
@@ -454,3 +468,149 @@ async def test_video_analyzer_sub_agent_confidence_validation():
model_turns = [c for c in sub_turn_2_contents if c.role == "model"]
assert len(model_turns) == 1
assert model_turns[0].parts[0].thought_signature == sub_dummy_signature
# ------------------------------------------------- static sub-agent loop
def _static_media(tmp_path) -> _ChunkMedia:
media = _ChunkMedia()
media.path = tmp_path / "segment.mp4"
media.path.write_bytes(b"video")
media.compressed_path = media.path
media.actual_start = 0.0
media.prompt_with_context = "Watch the toggle."
return media
def _valid_static_event() -> dict:
return {
"start_time": 1.0,
"end_time": 2.0,
"transcription": "Battery Saver toggled on",
"confidence_score": 0.9,
"verification_timestamp_secs": 1.5,
}
def _scripted_static_turns(*turns):
"""Fakes ``_stream_sub_agent_turn`` with ``(text, function_calls)`` per turn."""
scripted = list(turns)
async def fake_turn(analyzer, sub_agent_contents, trace_id):
text, function_calls = scripted.pop(0)
return text, function_calls, []
return fake_turn
def _static_analyzer() -> MagicMock:
analyzer = MagicMock()
analyzer.model_name = "gemini-3.7-flash"
analyzer.sub_system_prompt = "system"
analyzer.ctx.data_engine = None
return analyzer
@pytest.mark.asyncio
async def test_static_submit_answer_without_summary_is_sent_back_then_accepted(tmp_path):
first = types.FunctionCall(
name="submit_answer", args={"timeline_events": [_valid_static_event()]}
)
second = types.FunctionCall(
name="submit_answer",
args={"summary": "s2", "analysis": "a2", "timeline_events": []},
)
contents: list = []
with (
patch.object(
chunk_conversation,
"_stream_sub_agent_turn",
new=_scripted_static_turns(("", [first]), ("", [second])),
),
patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist,
):
answer = await _drive_sub_agent_loop(
_static_analyzer(), contents, None, _static_media(tmp_path), 0.0, 5.0, "verify"
)
assert answer == ("s2", "a2")
# Only the accepted answer reaches persistence.
persist.assert_awaited_once()
assert persist.await_args.args[1] == []
rejection = contents[-1]
assert rejection.role == "user"
assert rejection.parts[0].function_response.response["error"] == _MISSING_SUMMARY_ERROR
@pytest.mark.asyncio
async def test_static_loop_raises_when_no_turn_calls_submit_answer(tmp_path):
with (
patch.object(
chunk_conversation,
"_stream_sub_agent_turn",
new=_scripted_static_turns(("text only", []), ("still text", [])),
),
patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist,
pytest.raises(SubAgentAnswerExhausted) as raised,
):
await _drive_sub_agent_loop(
_static_analyzer(), [], None, _static_media(tmp_path), 0.0, 5.0, "verify"
)
assert raised.value.reason == _NO_TOOL_CALL_ERROR
persist.assert_not_awaited()
@pytest.mark.asyncio
async def test_static_last_turn_timeline_failure_raises_and_persists_nothing(tmp_path):
bad = dict(_valid_static_event(), confidence_score="HIGH")
out_of_range = dict(_valid_static_event(), verification_timestamp_secs=42.0)
first = types.FunctionCall(
name="submit_answer", args={"summary": "s1", "timeline_events": [bad]}
)
second = types.FunctionCall(
name="submit_answer", args={"summary": "s2", "timeline_events": [out_of_range]}
)
with (
patch.object(
chunk_conversation,
"_stream_sub_agent_turn",
new=_scripted_static_turns(("", [first]), ("", [second])),
),
patch.object(chunk_conversation, "_persist_timeline_events", new=AsyncMock()) as persist,
pytest.raises(SubAgentAnswerExhausted) as raised,
):
await _drive_sub_agent_loop(
_static_analyzer(), [], None, _static_media(tmp_path), 0.0, 5.0, "verify"
)
assert "verification_timestamp_secs" in raised.value.reason
assert "42.0" in str(raised.value)
persist.assert_not_awaited()
@pytest.mark.asyncio
async def test_exhausted_static_chunk_is_not_committed_to_the_blackboard(tmp_path):
analyzer = _static_analyzer()
uploaded = SimpleNamespace(name="files/abc", uri="https://files/abc", mime_type="video/mp4")
analyzer.upload_and_poll_file = AsyncMock(return_value=uploaded)
analyzer.client.aio.files.delete = AsyncMock()
analyzer.cloud_files_to_cleanup = set()
with (
patch.object(
chunk_conversation,
"_stream_sub_agent_turn",
new=_scripted_static_turns(("text only", []), ("still text", [])),
),
pytest.raises(SubAgentAnswerExhausted),
):
await _run_native_chunk_conversation(
analyzer, _static_media(tmp_path), 0.0, 5.0, 0.0, 5.0, "verify", "lease-1"
)
analyzer.blackboard.complete_segment.assert_not_called()
analyzer.client.aio.files.delete.assert_awaited_once_with(name="files/abc")
@@ -235,3 +235,215 @@ async def test_run_universal_multi_turn_reasoning(mock_context):
assert status == "success"
assert "launched successfully at 3.5s" in outcome
assert agent.exec_extract_segment_metadata.call_count == 1
# --- No-usable-answer handling -------------------------------------------
#
# The universal engine is single-shot: when the model neither calls
# submit_answer with a summary nor replies with text, it must raise
# SubAgentAnswerExhausted so the caller fails the blackboard segment instead
# of committing placeholder text as a succeeded entry.
_BLANK_SUBMIT = AIMessage(
content="",
tool_calls=[
{
"name": "submit_answer",
"args": {"summary": " ", "analysis": "", "timeline_events": []},
"id": "blank_call",
}
],
)
_EMPTY_RESPONSE = AIMessage(content="", tool_calls=[])
_TEXT_ONLY = AIMessage(content="Prose answer: the toast appeared at 2.0s.", tool_calls=[])
def _universal_agent(mock_context) -> VideoAnalyzer:
agent = VideoAnalyzer(mock_context)
agent.use_native_gemini = False
return agent
async def _run_chunk_with_response(agent, tmp_path, response):
"""Drives the universal chunk path with a stubbed model reply.
Returns ``(result_or_exception, persist_mock)`` so tests can assert both
the outcome and whether the blackboard persistence step was reached.
"""
raw_video = tmp_path / "raw.mp4"
raw_video.write_bytes(b"raw_bytes")
persist = AsyncMock()
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.extract_keyframes_from_video",
return_value=[(0.0, b"frame0")],
),
patch(
"artemis.agents.video_analyzer.video_analyzer.extract_frames_at_timestamps",
return_value=[],
),
patch(
"artemis.agents.video_analyzer.video_analyzer.extract_audio_from_video",
new_callable=AsyncMock,
side_effect=RuntimeError("no audio track"),
),
patch(
"artemis.agents.video_analyzer.universal_engine._persist_universal_events",
persist,
),
patch.object(
agent,
"_invoke_universal_model",
new_callable=AsyncMock,
return_value=response,
),
):
try:
result = await agent._exec_single_chunk_universal(
compressed_path=raw_video,
raw_path=raw_video,
start_time=0.0,
end_time=3.0,
actual_start=0.0,
prompt_with_context="Find the toast.",
specific_query="toast",
)
except Exception as exc: # noqa: BLE001 - the exception type is the assertion
return exc, persist
return result, persist
@pytest.mark.asyncio
@pytest.mark.parametrize("response", [_EMPTY_RESPONSE, _BLANK_SUBMIT])
async def test_exec_single_chunk_universal_raises_without_usable_answer(
mock_context, tmp_path, response
):
"""No tool call + no text, or blank-summary tool call + no text -> raise, no persistence."""
from artemis.agents.video_analyzer.reliability import SubAgentAnswerExhausted
agent = _universal_agent(mock_context)
outcome, persist = await _run_chunk_with_response(agent, tmp_path, response)
assert isinstance(outcome, SubAgentAnswerExhausted)
assert outcome.turns == 1
assert "submit_answer" in outcome.reason
persist.assert_not_awaited()
assert agent.blackboard_entries == []
@pytest.mark.asyncio
async def test_exec_single_chunk_universal_text_only_is_degraded_answer(mock_context, tmp_path):
"""Prose instead of the tool is a real (degraded) answer, not a placeholder."""
agent = _universal_agent(mock_context)
outcome, persist = await _run_chunk_with_response(agent, tmp_path, _TEXT_ONLY)
assert isinstance(outcome, str)
assert "Summary: Prose answer: the toast appeared at 2.0s." in outcome
assert "Analysis: Prose answer: the toast appeared at 2.0s." in outcome
persist.assert_awaited_once()
# A degraded text answer carries no timeline events.
assert persist.await_args.args[1] == []
@pytest.mark.asyncio
async def test_exec_single_chunk_universal_blank_summary_falls_through_to_text(
mock_context, tmp_path
):
"""submit_answer with a blank summary is ignored; the text content becomes the answer."""
response = AIMessage(
content="Fallback prose from the model.",
tool_calls=[
{
"name": "submit_answer",
"args": {
"summary": "",
"analysis": "Orphaned analysis.",
"timeline_events": [{"start_time": 0.0, "end_time": 1.0}],
},
"id": "blank_call",
}
],
)
agent = _universal_agent(mock_context)
outcome, persist = await _run_chunk_with_response(agent, tmp_path, response)
assert isinstance(outcome, str)
assert "Summary: Fallback prose from the model." in outcome
assert "Orphaned analysis" not in outcome
# The blank tool call is dropped wholesale, events included.
assert persist.await_args.args[1] == []
async def _run_audio_with_response(agent, tmp_path, response):
"""Drives the universal audio path with a stubbed model reply."""
fake_audio = tmp_path / "audio.aac"
fake_audio.write_bytes(b"audio_bytes")
with (
patch.object(
agent,
"_invoke_universal_model",
new_callable=AsyncMock,
return_value=response,
),
patch.object(
agent, "_record_blackboard_entry", wraps=agent._record_blackboard_entry
) as record,
):
try:
result = await agent._exec_analyze_audio_universal(
audio_path=fake_audio,
start_time=1.0,
end_time=4.0,
actual_start=1.0,
prompt_with_context="Listen for the toast.",
specific_query="toast sound",
)
except Exception as exc: # noqa: BLE001 - the exception type is the assertion
return exc, record
return result, record
@pytest.mark.asyncio
@pytest.mark.parametrize("response", [_EMPTY_RESPONSE, _BLANK_SUBMIT])
async def test_exec_analyze_audio_universal_raises_without_usable_answer(
mock_context, tmp_path, response
):
"""Audio path: no usable answer -> raise before anything reaches the blackboard."""
from artemis.agents.video_analyzer.reliability import SubAgentAnswerExhausted
agent = _universal_agent(mock_context)
outcome, record = await _run_audio_with_response(agent, tmp_path, response)
assert isinstance(outcome, SubAgentAnswerExhausted)
assert outcome.turns == 1
record.assert_not_called()
assert agent.blackboard_entries == []
@pytest.mark.asyncio
async def test_exec_analyze_audio_universal_text_only_is_degraded_answer(mock_context, tmp_path):
"""Audio path: prose instead of the tool is still committed as a degraded answer."""
agent = _universal_agent(mock_context)
outcome, record = await _run_audio_with_response(agent, tmp_path, _TEXT_ONLY)
assert isinstance(outcome, str)
assert "Audio Summary: Prose answer: the toast appeared at 2.0s." in outcome
record.assert_called_once()
assert len(agent.blackboard_entries) == 1
assert agent.blackboard_entries[0]["summary"] == "Prose answer: the toast appeared at 2.0s."
def test_sub_agent_answer_exhausted_classifies_as_unknown():
"""The raise must land in the retryable / split / fallback bucket of the callers."""
from artemis.agents.video_analyzer.reliability import (
SubAgentAnswerExhausted,
VideoFailureCategory,
classify_video_failure,
)
failure = classify_video_failure(SubAgentAnswerExhausted(1, "x"))
assert failure.category is VideoFailureCategory.UNKNOWN
assert failure.retryable is True
assert failure.should_split is True
assert failure.should_fallback is True
@@ -0,0 +1,157 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Native Gemini audio agent loop: only accepted answers reach the blackboard."""
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
from artemis.agents.video_analyzer.audio_native import (
_BAD_CONFIDENCE_ERROR,
_AudioMedia,
_drive_audio_agent_loop,
_run_native_audio_conversation,
)
from artemis.agents.video_analyzer.chunk_conversation import (
_MISSING_SUMMARY_ERROR,
_NO_TOOL_CALL_ERROR,
SUB_AGENT_MAX_TURNS,
)
from artemis.agents.video_analyzer.reliability import (
SubAgentAnswerExhausted,
VideoFailureCategory,
classify_video_failure,
)
from artemis.agents.video_analyzer.universal_tools import build_submit_answer_declaration
from google.genai import types
import pytest
def _audio_response(text: str, function_calls: list) -> SimpleNamespace:
"""Fakes a non-streamed ``generate_content`` response for one audio turn."""
parts = [types.Part(function_call=fc) for fc in function_calls]
if text:
parts.append(types.Part.from_text(text=text))
return SimpleNamespace(
text=text,
function_calls=function_calls,
candidates=[SimpleNamespace(content=SimpleNamespace(parts=parts))],
)
def _audio_analyzer(*turns) -> MagicMock:
"""Analyzer stub whose Gemini client answers the scripted ``(text, calls)`` turns."""
analyzer = MagicMock()
analyzer.model_name = "gemini-3.7-flash"
analyzer.audio_system_prompt = "system"
analyzer.submit_answer_tool = build_submit_answer_declaration()
analyzer.model_call_timeout_seconds = 5.0
analyzer.ctx.data_engine = None
analyzer.client.aio.models.generate_content = AsyncMock(
side_effect=[_audio_response(text, calls) for text, calls in turns]
)
return analyzer
def _audio_media(tmp_path) -> _AudioMedia:
media = _AudioMedia()
media.audio_path = tmp_path / "segment.wav"
media.audio_path.write_bytes(b"audio")
media.actual_start = 0.0
media.prompt_with_context = "Listen for the chime."
return media
@pytest.mark.asyncio
async def test_audio_submit_answer_without_summary_is_sent_back_then_accepted():
first = types.FunctionCall(
name="submit_answer", args={"summary": " ", "confidence_score": 0.9}
)
second = types.FunctionCall(
name="submit_answer",
args={"summary": "chime\nheard", "analysis": "a2", "confidence_score": 0.7},
)
analyzer = _audio_analyzer(("", [first]), ("", [second]))
contents: list = []
answer = await _drive_audio_agent_loop(analyzer, contents)
assert answer == ("chime heard", "a2", 0.7)
assert analyzer.client.aio.models.generate_content.await_count == 2
# The blank summary went back to the model as a function_response error.
rejections = [
c
for c in contents
if c.role == "user" and c.parts and c.parts[0].function_response is not None
]
assert len(rejections) == 1
assert rejections[0].parts[0].function_response.response["error"] == _MISSING_SUMMARY_ERROR
@pytest.mark.asyncio
async def test_audio_loop_raises_when_no_turn_calls_submit_answer():
analyzer = _audio_analyzer(("text only", []), ("still text", []))
with pytest.raises(SubAgentAnswerExhausted) as raised:
await _drive_audio_agent_loop(analyzer, [])
assert raised.value.turns == SUB_AGENT_MAX_TURNS
assert raised.value.reason == _NO_TOOL_CALL_ERROR
assert "No summary provided" not in str(raised.value)
@pytest.mark.asyncio
async def test_audio_loop_raises_when_confidence_score_is_never_valid():
first = types.FunctionCall(
name="submit_answer", args={"summary": "s1", "confidence_score": "HIGH"}
)
second = types.FunctionCall(
name="submit_answer", args={"summary": "s2", "confidence_score": 1.5}
)
analyzer = _audio_analyzer(("", [first]), ("", [second]))
contents: list = []
with pytest.raises(SubAgentAnswerExhausted) as raised:
await _drive_audio_agent_loop(analyzer, contents)
assert raised.value.reason == _BAD_CONFIDENCE_ERROR
assert contents[-1].role == "user"
assert contents[-1].parts[0].function_response.response["error"] == _BAD_CONFIDENCE_ERROR
@pytest.mark.asyncio
async def test_exhausted_audio_segment_is_not_committed_and_cloud_file_is_deleted(tmp_path):
analyzer = _audio_analyzer(("text only", []), ("still text", []))
uploaded = SimpleNamespace(name="files/abc", uri="https://files/abc", mime_type="audio/wav")
analyzer.upload_and_poll_file = AsyncMock(return_value=uploaded)
analyzer.client.aio.files.delete = AsyncMock()
analyzer.cloud_files_to_cleanup = {"files/abc"}
analyzer._record_blackboard_entry = MagicMock()
analyzer.blackboard = MagicMock()
with pytest.raises(SubAgentAnswerExhausted):
await _run_native_audio_conversation(
analyzer, _audio_media(tmp_path), 0.0, 5.0, 0.0, 5.0, "chime", "lease-1"
)
analyzer.blackboard.complete_segment.assert_not_called()
analyzer._record_blackboard_entry.assert_not_called()
analyzer.client.aio.files.delete.assert_awaited_once_with(name="files/abc")
assert analyzer.cloud_files_to_cleanup == set()
def test_sub_agent_exhaustion_is_classified_unknown():
failure = classify_video_failure(SubAgentAnswerExhausted(2, "x"))
assert failure.category is VideoFailureCategory.UNKNOWN
+162
View File
@@ -3,6 +3,7 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
@@ -10,6 +11,7 @@ from langchain_core.messages import AIMessage, HumanMessage
import pytest
from artemis.agents.video_analyzer.reliability import (
AgenticVideoDegraded,
VideoCircuitBreaker,
VideoFailureCategory,
classify_video_failure,
@@ -290,3 +292,163 @@ async def test_invoke_with_retry_caps_unknown_failures_by_policy():
await va._invoke_with_retry(op, "test-op", max_attempts=5)
# UNKNOWN failures follow the shared policy, not the caller's outer cap.
assert calls == retry_policy_for(FailureCategory.UNKNOWN).max_attempts
# ------------------------------------------------ mid-run agentic degrade
class _AgenticRejected(Exception):
status_code = 400
@pytest.mark.asyncio
async def test_pending_chunk_sized_for_agentic_ceiling_is_not_run_statically(tmp_path):
"""A chunk planned at 600s whose run degraded before it started is handed back."""
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(_context())
analyzer.use_native_gemini = True
# A sibling chunk already flipped the run to static (60s ceiling).
analyzer.video_processing = "static"
analyzer.chunk_size_seconds = analyzer.static_chunk_size_seconds
controller = SimpleNamespace(extract_segment_metadata=AsyncMock())
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.get_controller",
return_value=controller,
),
patch(
"artemis.agents.video_analyzer.chunk_native._run_native_chunk_conversation",
new=AsyncMock(),
) as static_path,
patch.object(analyzer, "_exec_single_chunk_universal", new=AsyncMock()) as universal,
pytest.raises(AgenticVideoDegraded) as raised,
):
await analyzer._exec_single_chunk(0.0, 150.0, "verify toggle")
# Refused before any extraction/upload; no static or fallback analysis.
controller.extract_segment_metadata.assert_not_awaited()
static_path.assert_not_awaited()
universal.assert_not_awaited()
assert classify_video_failure(raised.value).should_split
# The lease is released so the re-planned pieces can be claimed.
assert analyzer.blackboard.missing_intervals(0.0, 150.0, "verify toggle") == [(0.0, 150.0)]
@pytest.mark.asyncio
async def test_sibling_chunk_degraded_mid_run_is_replanned_into_static_pieces(tmp_path):
"""Two 600s chunks: the first is rejected while the second is still in media prep."""
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(_context())
analyzer.use_native_gemini = True
analyzer.video_processing = "agentic"
analyzer.chunk_size_seconds = analyzer.agentic_chunk_size_seconds
assert analyzer.chunk_size_seconds == 600.0
raw_video = tmp_path / "segment.mp4"
raw_video.write_bytes(b"video")
first_rejected = asyncio.Event()
async def extract(start: float, end: float):
if start >= 600.0 and analyzer.video_processing == "agentic":
# The second 600s chunk is mid-prep when the first one degrades.
await first_rejected.wait()
return SimpleNamespace(
success=True,
video_path=raw_video,
actual_start_relative_time=start,
duration_seconds=end - start,
warning=None,
)
controller = SimpleNamespace(extract_segment_metadata=AsyncMock(side_effect=extract))
agentic_calls: list[tuple[float, float]] = []
async def agentic(analyzer_, media, cs, ce, *rest):
agentic_calls.append((cs, ce))
first_rejected.set()
raise _AgenticRejected("400 INVALID_ARGUMENT: processing 'agentic' is not supported")
async def static_answer(analyzer_, media, cs, ce, *rest):
return f"[from {cs:.1f}s to {ce:.1f}s] Summary: static Analysis: ok"
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.get_controller",
return_value=controller,
),
patch(
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
new=AsyncMock(return_value=raw_video),
),
patch(
"artemis.agents.video_analyzer.chunk_native.run_agentic_chunk_conversation",
new=AsyncMock(side_effect=agentic),
),
patch(
"artemis.agents.video_analyzer.chunk_native._run_native_chunk_conversation",
new=AsyncMock(side_effect=static_answer),
) as static_path,
patch.object(analyzer, "_exec_single_chunk_universal", new=AsyncMock()) as universal,
patch("artemis.agents.video_analyzer.video_analyzer._record_llm_event"),
):
result = await analyzer.exec_spawn_sub_agent(0.0, 1200.0, "verify toggle")
# Only the first chunk ever reached the agentic API; the second never ran
# its 600s clip statically and neither went to the universal fallback.
assert agentic_calls == [(0.0, 600.0)]
universal.assert_not_awaited()
segments = sorted((call.args[2], call.args[3]) for call in static_path.await_args_list)
assert len(segments) == 20
assert segments[0] == (0.0, 60.0)
assert segments[-1] == (1140.0, 1200.0)
assert all(ce - cs <= 60.0 for cs, ce in segments)
assert "PARTIAL" not in result
assert result.count("Summary: static") == 20
assert analyzer.video_processing == "static"
@pytest.mark.asyncio
async def test_short_degraded_chunk_is_retried_statically_once():
"""A 5s chunk (below min_chunk*2) is re-run as-is instead of failing."""
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(_context())
calls: list[tuple[float, float]] = []
async def analyze(start: float, end: float, query: str) -> str:
calls.append((start, end))
if len(calls) == 1:
analyzer.video_processing = "static"
analyzer.chunk_size_seconds = analyzer.static_chunk_size_seconds
raise AgenticVideoDegraded("agentic rejected")
return f"[from {start:.1f}s to {end:.1f}s] Summary: static {query}"
with patch.object(analyzer, "_exec_single_chunk", side_effect=analyze) as child:
result = await analyzer.exec_spawn_sub_agent(0.0, 5.0, "find toggle")
assert child.await_count == 2
assert calls == [(0.0, 5.0), (0.0, 5.0)]
assert "PARTIAL" not in result
assert "failed" not in result.lower()
assert result.count("Summary: static find toggle") == 1
@pytest.mark.asyncio
async def test_static_retry_of_degraded_chunk_is_bounded():
"""A second degrade on the same interval is terminal: no loop, no bisect."""
with patch("artemis.agents.video_analyzer.video_analyzer.settings.GOOGLE_API_KEY", None):
analyzer = VideoAnalyzer(_context())
calls: list[tuple[float, float]] = []
async def analyze(start: float, end: float, query: str) -> str:
calls.append((start, end))
raise AgenticVideoDegraded("agentic rejected again")
with patch.object(analyzer, "_exec_single_chunk", side_effect=analyze) as child:
# 30s is above min_chunk*2, so a bisect would otherwise be legal.
result = await analyzer.exec_spawn_sub_agent(0.0, 30.0, "find toggle")
assert child.await_count == 2
assert calls == [(0.0, 30.0), (0.0, 30.0)]
assert result.startswith("All sub-agent chunks failed")
assert "0.0s-30.0s [bad_request]" in result
@@ -0,0 +1,39 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The video-processing knob is typed once, in the Google provider module."""
from typing import get_args
from pydantic import ValidationError
import pytest
from artemis.config.agent import VideoAnalyzerConfig
from artemis.llm.google import VideoProcessing
def test_processing_field_reuses_the_provider_literal():
assert VideoAnalyzerConfig.model_fields["processing"].annotation is VideoProcessing
assert set(get_args(VideoProcessing)) == {"auto", "agentic", "static"}
def test_processing_defaults_to_auto_and_accepts_every_mode():
assert VideoAnalyzerConfig().processing == "auto"
for mode in get_args(VideoProcessing):
assert VideoAnalyzerConfig(processing=mode).processing == mode
def test_processing_rejects_unknown_modes():
with pytest.raises(ValidationError, match="processing"):
VideoAnalyzerConfig(processing="bogus")
+1 -1
View File
@@ -615,7 +615,7 @@ def test_pro_pixel_record_is_normalized_exactly_once(tmp_path):
action_taken=[action],
last_execution_result={
"status": "success",
"execution": [dict(action, attempts=["Success"])],
"execution": [dict(action, attempts=["Dispatched"])],
},
extra_metadata={"width": 1080, "height": 2400},
)
@@ -130,59 +130,41 @@ async def test_ask_explorer_exception_is_contained():
assert result.text_summary == "Error executing ask_explorer: boom"
# ---------------------------------------------------------------------------
# Legacy MobileActionExecutor parity (same ask_explorer semantics)
# ---------------------------------------------------------------------------
def _make_legacy_executor(**kwargs):
from artemis.agents.validator.tool_declarations import MobileActionExecutor
ctx = Mock()
ctx.data_engine = None
ctx.device.device_width = 1080
ctx.device.device_height = 2400
actuator = Mock()
actuator.controller = Mock()
return MobileActionExecutor(ctx, actuator=actuator, **kwargs)
@pytest.mark.asyncio
async def test_legacy_executor_ask_explorer_status_follows_the_outcome():
executor = _make_legacy_executor()
async def test_default_agent_name_reaches_the_tier_resolver_and_status_follows_the_outcome():
"""Without an explicit agent name the executor asks the Explorer as ``validator``;
a clean not-found is an answer, an Explorer run failure is an error."""
executor = _make_executor()
assert executor.agent_name == "validator"
not_found = ExplorerOutcome(message="Not visible.")
with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=not_found)):
# The pipeline functions are imported at module load in the legacy executor.
with patch(
"artemis.agents.validator.tool_declarations.locate",
new=AsyncMock(return_value=not_found),
) as locate:
result = await executor.execute(
"ask_explorer", {"task_description": "gear icon"}, "tc", _state()
)
with patch(
"artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=not_found)
) as locate:
result = await executor.execute(
"ask_explorer", {"task_description": "gear icon"}, "tc", _state()
)
assert locate.await_args.args[2] == "gear icon"
assert locate.await_args.kwargs["agent_name"] == "validator"
assert result.status == "success" # a clean not-found is an answer, not a failure
assert result.status == "success"
failed = ExplorerOutcome.failure("Explorer failed: boom")
with patch(
"artemis.agents.validator.tool_declarations.locate", new=AsyncMock(return_value=failed)
):
with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=failed)):
result = await executor.execute("ask_explorer", {"query": "gear icon"}, "tc", _state())
assert result.status == "error"
@pytest.mark.asyncio
async def test_legacy_executor_agent_name_reaches_the_tier_resolver():
executor = _make_legacy_executor(agent_name="flash")
async def test_flash_agent_name_reaches_the_tier_resolver():
"""The Flash profile resolves the Explorer tier from ``agent_name="flash"``."""
executor = _make_executor(agent_name="flash")
with patch(
"artemis.agents.validator.tool_declarations.locate",
"artemis.tools.explorer_tool.locate",
new=AsyncMock(return_value=ExplorerOutcome(message="no")),
) as locate:
await executor.execute("ask_explorer", {"query": "x"}, "tc", _state())
assert locate.await_args.kwargs["agent_name"] == "flash"
assert locate.await_args.args[0] is executor.ctx
# ---------------------------------------------------------------------------
@@ -293,19 +275,21 @@ async def test_history_tool_exception_is_contained():
tool = _history_tool_double(side_effect=RuntimeError("db gone"))
with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool):
result = await executor.execute("replay_steps", {"query": "x"}, "tc", None)
assert result.status == "success" # contained: the answer explains the failure
# Contained (no exception escapes) and reported structurally: the answer
# explains the failure and the status says it is one.
assert result.status == "error"
assert result.text_summary == "replay_steps failed: db gone"
@pytest.mark.asyncio
async def test_history_tools_dispatch_to_the_real_shared_instances():
"""No per-tool branch: every history tool name resolves through the package."""
executor = _make_executor() # ctx.data_engine is None -> tools degrade to text
executor = _make_executor() # ctx.data_engine is None -> tools report a failure
for name in ("search_history", "replay_steps", "get_step_screenshot"):
result = await executor.execute(
name, {"query": "x", "start_step": 1, "step_number": 1}, "tc", None
)
assert result.status == "success"
assert result.status == "error"
assert (
"no active execution history" in result.text_summary
or "no execution history" in result.text_summary
+181 -109
View File
@@ -12,11 +12,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for record-time target semantics enrichment in McpActionExecutor."""
"""Coordinate targets must be described by the model, never inferred.
The Flash/Validator dialect addresses every target by coordinates, so the
executor refuses a click, long press, focused input, coordinate swipe or click
sequence that does not say what it is aiming at. The description is recorded
by the runner and never reaches the wire.
"""
from unittest.mock import Mock
from artemis.mcp.action_executor import McpActionExecutor
import pytest
from artemis.mcp.action_executor import McpActionExecutor, _ArgError
def _make_executor(width=1080, height=2400):
@@ -28,131 +36,195 @@ def _make_executor(width=1080, height=2400):
return McpActionExecutor(ctx, actuator=actuator)
def _state_with_elements(elements):
def _state():
state = Mock()
state.indexed_elements = elements
state.indexed_elements = []
state.indexed_points = [[540, 1440], [745, 1440]]
state.latest_ui_hierarchy = None
return state
def test_click_target_semantics_from_pre_action_frame():
@pytest.mark.parametrize(
"name, args, label",
[
("click", {"target": [500, 600]}, "click"),
("long_press", {"target": [500, 600]}, "long press"),
("input_text", {"text": "hi", "target": [500, 600]}, "input text"),
("swipe", {"start": [200, 600], "end": [800, 600]}, "swipe"),
],
)
def test_coordinate_targets_without_description_are_refused(name, args, label):
executor = _make_executor()
# Normalized (500, 600) on a 1080x2400 device -> pixel (540, 1440).
state = _state_with_elements(
[
{
"text": "Confirm",
"bounds": [500, 1400, 600, 1480],
"class": "android.widget.Button",
"resource_id": "btn_confirm",
"is_ocr": False,
}
]
with pytest.raises(_ArgError) as excinfo:
executor._translate(name, args, _state())
message = str(excinfo.value)
assert message.startswith(f"Error during {label}")
assert "target_description" in message
@pytest.mark.parametrize("blank", ["", " ", None, 7])
def test_blank_or_non_string_description_is_refused(blank):
executor = _make_executor()
with pytest.raises(_ArgError):
executor._translate("click", {"target": [500, 600], "target_description": blank}, _state())
def test_description_is_accepted_and_kept_off_the_wire():
executor = _make_executor()
wire_name, wire_args, _, _ = executor._translate(
"click", {"target": [500, 600], "target_description": "play button"}, _state()
)
semantics = executor._target_semantics("click", {"target": [500, 600]}, state)
assert semantics == {
"target_label_source": "hit_test",
"target_text": "Confirm",
"target_class": "android.widget.Button",
"target_resource_id": "btn_confirm",
}
assert wire_name == "click"
assert wire_args == {"target": [500, 600], "times": 1, "delay_ms": 100}
assert "target_description" not in wire_args
def test_semantics_degrade_to_none_without_element_data():
executor = _make_executor()
# Empty perception data
semantics = executor._target_semantics(
"click", {"target": [500, 600]}, _state_with_elements([])
_, swipe_args, _, _ = executor._translate(
"swipe",
{"start": [200, 600], "end": [800, 600], "target_description": "brightness knob"},
_state(),
)
assert semantics == {"target_label_source": "none"}
# No state at all
semantics = executor._target_semantics("click", {"target": [500, 600]}, None)
assert semantics == {"target_label_source": "none"}
assert swipe_args == {"start": [200, 600], "end": [800, 600], "duration_ms": 400}
def test_non_targeted_actions_and_missing_targets_skip_enrichment():
def test_recorded_target_is_the_cleaned_description_in_pro_shape():
"""The recorded semantics come from the same check that requires them."""
executor = _make_executor()
state = _state_with_elements([{"text": "X", "bounds": [0, 0, 10, 10]}])
assert executor._target_semantics("press_key", {"key": "BACK"}, state) is None
# input_text without a coordinate target (focused field typing)
assert executor._target_semantics("input_text", {"text": "hi", "target": None}, state) is None
# Multi-point actions without resolvable points skip enrichment too.
assert executor._target_semantics("click_sequence", {"sequence": []}, state) is None
assert executor._target_semantics("swipe", {"start": None, "end": None}, state) is None
def test_click_sequence_hit_tests_every_point_first_point_is_main_label():
"""M5: multi-point actions get per-point best-effort semantics; the first
point's fields are hoisted as the action's main label."""
executor = _make_executor()
state = _state_with_elements(
[
{
"text": "Digit 1",
"bounds": [500, 1400, 600, 1480],
"class": "android.widget.Button",
"is_ocr": False,
},
{
"text": "Digit 2",
"bounds": [700, 1400, 800, 1480],
"class": "android.widget.Button",
"is_ocr": False,
},
]
_, _, _, recorded = executor._translate(
"click", {"target": [1, 2], "target_description": " ok "}, _state()
)
# Normalized points -> pixels: (500,600)->(540,1440) hits Digit 1;
# (690,600)->(745,1440) hits Digit 2; (10,10)->(10,24) hits nothing.
semantics = executor._target_semantics(
assert recorded == {"target_description": "ok"}
_, _, _, recorded = executor._translate(
"long_press", {"target": [1, 2], "target_description": "thumbnail"}, _state()
)
assert recorded == {"target_description": "thumbnail"}
_, _, _, recorded = executor._translate(
"click_sequence", {"sequence": [[1, 2]], "target_descriptions": ["a "]}, _state()
)
assert recorded == {"target_descriptions": ["a"]}
# No coordinate target: nothing is recorded, nothing inferred.
_, _, _, recorded = executor._translate("press_key", {"key": "BACK"}, _state())
assert recorded == {}
def test_coordinate_swipe_records_its_description():
executor = _make_executor()
_, wire_args, _, recorded = executor._translate(
"swipe",
{"start": [200, 600], "end": [800, 600], "target_description": " brightness knob "},
_state(),
)
assert recorded == {"target_description": "brightness knob"}
assert "target_description" not in wire_args
def test_targeted_input_records_its_description():
executor = _make_executor()
_, wire_args, _, recorded = executor._translate(
"input_text",
{"text": "hi", "target": [500, 600], "target_description": "search input"},
_state(),
)
assert wire_args["target"] == [500, 600]
assert recorded == {"target_description": "search input"}
assert "target_description" not in wire_args
def test_focused_typing_without_a_target_needs_no_description():
executor = _make_executor()
_, wire_args, _, recorded = executor._translate(
"input_text", {"text": "hi", "target": None}, _state()
)
assert wire_args["target"] is None
assert recorded == {}
def test_focused_typing_ignores_a_stray_description():
"""Typing into the focused field has no coordinate target: a description the
model tacked on anyway is not recorded, so no self-described target enters
the history for an action that aimed at nothing."""
executor = _make_executor()
_, wire_args, _, recorded = executor._translate(
"input_text",
{"text": "hi", "target": None, "target_description": "search input"},
_state(),
)
assert wire_args["target"] is None
assert recorded == {}
def test_directional_swipe_needs_no_description():
executor = _make_executor()
wire_name, wire_args, _, recorded = executor._translate("swipe", {"direction": "up"}, _state())
assert wire_name == "swipe"
assert "start" in wire_args and "end" in wire_args
assert recorded == {}
def test_directional_swipe_ignores_a_stray_description():
executor = _make_executor()
_, wire_args, _, recorded = executor._translate(
"swipe", {"direction": "up", "target_description": "the feed"}, _state()
)
assert "target_description" not in wire_args
assert recorded == {}
def test_click_sequence_requires_one_description_per_entry():
executor = _make_executor()
sequence = [[500, 600], [690, 600], [10, 10]]
with pytest.raises(_ArgError) as excinfo:
executor._translate("click_sequence", {"sequence": sequence}, _state())
assert "target_descriptions" in str(excinfo.value)
assert "3 expected" in str(excinfo.value)
# Wrong length is refused too.
with pytest.raises(_ArgError):
executor._translate(
"click_sequence",
{"sequence": sequence, "target_descriptions": ["video body", "skip"]},
_state(),
)
_, wire_args, _, recorded = executor._translate(
"click_sequence",
{"sequence": [[500, 600], [690, 600], [10, 10]], "delay_ms": 50},
state,
{"sequence": sequence, "target_descriptions": ["video body", "digit 2", "corner"]},
_state(),
)
assert semantics["target_text"] == "Digit 1"
assert semantics["target_label_source"] == "hit_test"
per_point = semantics["points_semantics"]
assert len(per_point) == 3
assert per_point[0]["target_text"] == "Digit 1"
assert per_point[1]["target_text"] == "Digit 2"
assert per_point[2] == {"target_label_source": "none"}
assert wire_args["sequence"] == [[500, 600], [690, 600], [10, 10]]
assert "target_descriptions" not in wire_args
assert recorded == {"target_descriptions": ["video body", "digit 2", "corner"]}
def test_swipe_hit_tests_start_and_end_start_is_main_label():
@pytest.mark.parametrize("index_entry", [2, "2", 2.0, [2]])
def test_click_sequence_refuses_element_indices(index_entry):
"""click_sequence takes coordinate pairs only. An index would be resolved
against the element list and then recorded under the model's own
description, reading back as a self-described coordinate target."""
executor = _make_executor()
state = _state_with_elements(
[
{
"text": "Brightness slider",
"bounds": [100, 1400, 900, 1480],
"class": "android.widget.SeekBar",
"is_ocr": False,
}
]
)
semantics = executor._target_semantics(
"swipe", {"start": [200, 600], "end": [800, 600], "duration_ms": 400}, state
)
assert semantics["target_text"] == "Brightness slider"
assert semantics["start_semantics"]["target_text"] == "Brightness slider"
assert semantics["end_semantics"]["target_text"] == "Brightness slider"
assert semantics["target_label_source"] == "hit_test"
sequence = [[500, 600], index_entry, [10, 10]]
with pytest.raises(_ArgError) as excinfo:
executor._translate(
"click_sequence",
{"sequence": sequence, "target_descriptions": ["video body", "digit 2", "corner"]},
_state(),
)
message = str(excinfo.value)
assert message.startswith("Error during click sequence")
assert "entry 2" in message
assert "coordinate" in message
assert "ask_explorer" in message
def test_long_press_uses_ocr_fallback():
def test_click_sequence_accepts_a_serialized_pair_list():
executor = _make_executor()
state = _state_with_elements(
[
{
"text": "OCR Word",
"bounds": [520, 1420, 560, 1460],
"class": None,
"resource_id": None,
"is_ocr": True,
}
]
_, wire_args, _, _ = executor._translate(
"click_sequence",
{"sequence": "[[500, 600], [10, 10]]", "target_descriptions": ["video body", "corner"]},
_state(),
)
semantics = executor._target_semantics("long_press", {"target": [500, 600]}, state)
assert semantics["target_label_source"] == "ocr"
assert semantics["target_text"] == "OCR Word"
assert wire_args["sequence"] == [[500, 600], [10, 10]]
+2 -1
View File
@@ -55,7 +55,8 @@ SCOPED_PROMPT_FILES = [
# note keys, and the availability-slot template helper rendered by
# apply_operator_prompt_contract. Anything new landing here should be a conscious
# decision.
NON_TOOL_IDENTIFIERS = frozenset({"analysis", "task_plan", "tool_enum"})
#: Backticked identifiers in prompts that are parameters or note keys, not tools.
NON_TOOL_IDENTIFIERS = frozenset({"analysis", "task_plan", "tool_enum", "target_description"})
_TOOL_REF = re.compile(r"(?:`([a-z][a-z0-9_]{2,})`|\b([a-z][a-z0-9_]{2,})\()")
+1 -1
View File
@@ -40,7 +40,7 @@ async def test_click_end_to_end():
try:
res = await session.call("click", {"target": [500, 500]})
assert res.ok
assert res.message == "Clicked at [500, 500] (normalized) successfully."
assert res.message == "Tapped at [500, 500] (normalized)."
assert actuator.action_history[-1]["action"] == "tap"
assert actuator.action_history[-1]["x"] == 500
assert actuator.action_history[-1]["y"] == 1000
+7 -9
View File
@@ -38,7 +38,7 @@ async def test_click_converts_normalized_to_pixels(actuator):
res = await actuator.click(500, 500)
assert res.ok and res.code is ActionCode.OK
# Historical wording is part of the contract: transcripts and traces assert on it.
assert res.message == "Clicked at [500, 500] (normalized) successfully."
assert res.message == "Tapped at [500, 500] (normalized)."
assert res.normalized_coordinates == [500, 500]
assert actuator.action_history[-1] == {
"action": "tap",
@@ -53,9 +53,7 @@ async def test_click_converts_normalized_to_pixels(actuator):
async def test_click_sequence_taps_each_point(actuator):
res = await actuator.click_sequence([(100, 100), (900, 900)], delay_ms=1)
assert res.ok
assert res.message == (
"Sequence clicked successfully: Tapped at [100, 200]; Tapped at [900, 1800]"
)
assert res.message == ("Tapped in sequence at [100, 100]; [900, 900] (normalized).")
taps = [h for h in actuator.action_history if h["action"] == "tap"]
assert [(t["x"], t["y"]) for t in taps] == [(100, 200), (900, 1800)]
@@ -64,7 +62,7 @@ async def test_click_sequence_taps_each_point(actuator):
async def test_long_press_message_and_duration(actuator):
res = await actuator.long_press(250, 750, duration_ms=1500)
assert res.ok
assert res.message == ("Long pressed at [250, 750] (normalized) for 1500ms successfully.")
assert res.message == "Long-pressed at [250, 750] (normalized) for 1500ms."
assert res.duration_ms == 1500
@@ -72,7 +70,7 @@ async def test_long_press_message_and_duration(actuator):
async def test_swipe_reports_normalized_endpoints(actuator):
res = await actuator.swipe((500, 800), (500, 200), 400)
assert res.ok
assert res.message == "Swipe completed successfully. Swiped from [500, 800] to [500, 200]."
assert res.message == "Swiped from [500, 800] to [500, 200] (normalized)."
assert res.normalized_coordinates == [500, 800, 500, 200]
@@ -82,7 +80,7 @@ async def test_press_key_forwards_unknown_keycode_to_driver(actuator):
# matching the historical adb_server behavior of accepting any Android key event.
res = await actuator.press_key("KEYCODE_DPAD_DOWN")
assert res.ok
assert res.message == "Executed key press 'KEYCODE_DPAD_DOWN'."
assert res.message == "Pressed key 'KEYCODE_DPAD_DOWN'."
assert actuator.action_history[-1]["action"] == "press_key"
@@ -98,7 +96,7 @@ async def test_input_text_append_moves_cursor_and_preserves_text(actuator, monke
monkeypatch.setattr("asyncio.sleep", _instant_sleep)
res = await actuator.input_text("hello", (500, 300), clear_exist=False)
assert res.ok
assert res.message == "Executed typing 'hello'."
assert res.message == "Typed 'hello' at [500, 300] (normalized)."
actions = [h["action"] for h in actuator.action_history]
# Append semantics: focus tap, cursor-to-end (KEYCODE_MOVE_END), then type without
# clearing — matching the historical adb_server focus_and_input_text contract.
@@ -128,7 +126,7 @@ async def _instant_sleep(_delay, *args, **kwargs):
async def test_wait_for_delay(actuator):
res = await actuator.wait_for_delay(1)
assert res.ok
assert res.message == "Waited for 1ms successfully."
assert res.message == "Waited 1ms."
def test_partial_capabilities_are_reported():
+4 -4
View File
@@ -43,7 +43,7 @@ def _step(number: int, subgoal_hash: str, summary: str | None = None) -> dict:
"summary": summary or f"Step {number} summary text.",
"action_taken": {"action": "click", "target_text": f"button {number}"},
"operator_raw_thinking": f"thinking {number}",
"last_execution_result": {"status": "success"},
"last_execution_result": {"status": "dispatched"},
"extra_metadata": {"subgoal_hash": subgoal_hash},
}
@@ -182,9 +182,9 @@ RENDERED_CHUNK = (
" - Steps 1–3: opened the app and logged in\n"
"\n"
"③ Step action ledger\n"
"- Step 1 (T+00:10): click['button 1'] -> executed\n"
"- Step 2 (T+00:20): click['button 2'] -> executed\n"
"- Step 3 (T+00:30): click['button 3'] -> executed"
"- Step 1 (T+00:10): click['button 1'] -> dispatched\n"
"- Step 2 (T+00:20): click['button 2'] -> dispatched\n"
"- Step 3 (T+00:30): click['button 3'] -> dispatched"
)
+21 -21
View File
@@ -63,7 +63,7 @@ def _step(i: int, subgoal: str = "hash-a", **overrides) -> dict:
"timestamp": SESSION_START + 10.0 * i,
"summary": f"Visual transition of step {i}.",
"action_taken": {"action": "tap", "target_text": f"btn{i}", "coordinates": [i, i]},
"last_execution_result": {"status": "success"},
"last_execution_result": {"status": "dispatched"},
"operator_raw_thinking": f"thinking {i}",
"interleaved_events": [],
"extra_metadata": {"subgoal_hash": subgoal},
@@ -201,7 +201,7 @@ def _run_turns(ledger, chunker, start: int, upto: int, hash_for=None):
for i in range(start, upto + 1):
ledger.commit_staged(
step_key=f"s{i - 1}" if i > 1 else None,
validator_result={"status": "success"} if i > 1 else None,
validator_result={"status": "dispatched"} if i > 1 else None,
)
rendered = ledger.render([_observation(i)])
ledger.stage_turn(_turn(i))
@@ -226,7 +226,7 @@ def test_action_ledger_lines_carry_step_number_offset_action_and_result():
steps = [_step(1), _step(2, last_execution_result={"status": "failed", "error": "boom"})]
ledger_text = build_action_ledger(steps, SESSION_START)
lines = ledger_text.splitlines()
assert lines[0] == "- Step 1 (T+00:10): Tapped 'btn1' at [1, 1] -> executed"
assert lines[0] == "- Step 1 (T+00:10): Tapped 'btn1' at [1, 1] -> dispatched"
assert lines[1].startswith("- Step 2 (T+00:20): Tapped 'btn2' at [2, 2] -> Error: boom")
@@ -289,7 +289,7 @@ def test_action_ledger_minimal_width_keeps_step_and_time_addressability():
minimal = build_action_ledger(steps, SESSION_START, minimal=True)
for i in range(1, 4):
assert f"- Step {i} (T+00:{10 * i:02d}): Tapped 'btn{i}'" in minimal
assert "-> executed" not in minimal
assert "-> dispatched" not in minimal
# ---------------------------------------------------------------------------
@@ -396,7 +396,7 @@ def test_milestone_switch_closes_previous_segment_whole_and_gates_swap():
# Capsule ready → the next render swaps: frozen ①②③ block, originals out.
capsule.resolve("chunk:1-4", _capsule(1, 4))
ledger.commit_staged(step_key="s8", validator_result={"status": "success"})
ledger.commit_staged(step_key="s8", validator_result={"status": "dispatched"})
rendered_after = ledger.render([_observation(9)])
assert len(chunker.chunks) == 1
assert chunker.awaiting_chunks == ()
@@ -517,7 +517,7 @@ def test_frozen_region_stays_byte_stable_between_swap_events():
# Swap chunk 1 (Steps 1–4) once its capsule is ready.
capsule.resolve("chunk:1-4", _capsule(1, 4))
ledger.commit_staged(step_key="s8", validator_result={"status": "success"})
ledger.commit_staged(step_key="s8", validator_result={"status": "dispatched"})
ledger.render([_observation(9)])
assert len(chunker.chunks) == 1
frozen_before = _frozen_text(ledger)
@@ -537,7 +537,7 @@ def test_frozen_region_stays_byte_stable_between_swap_events():
# Its capsule readies: the next render swaps and re-renders the region.
capsule.resolve(pending.capsule_key, _capsule(5, 8))
ledger.commit_staged(step_key="s12", validator_result={"status": "success"})
ledger.commit_staged(step_key="s12", validator_result={"status": "dispatched"})
ledger.render([_observation(13)])
assert len(chunker.chunks) == 2
frozen_after = _frozen_text(ledger)
@@ -572,7 +572,7 @@ def test_chunk_block_renders_three_bands_in_order():
assert "Verified: fact-1" in block
# ② lines carry step references; ③ is the untouched mechanical ledger.
assert "- Steps 1–4:" in block
assert "- Step 1 (T+00:10): Tapped 'btn1' at [1, 1] -> executed" in block
assert "- Step 1 (T+00:10): Tapped 'btn1' at [1, 1] -> dispatched" in block
assert " ago" not in block
@@ -843,7 +843,7 @@ def test_hard_threshold_renders_l3_snapshot_with_minimal_step_index():
for chunk in chunker.chunks:
for n in range(chunk.start_step_number, chunk.end_step_number + 1):
assert f"- Step {n} (T+" in frozen
assert "-> executed" not in frozen.split("--- Step index")[1]
assert "-> dispatched" not in frozen.split("--- Step index")[1]
def test_step_addressability_survives_every_compression_level():
@@ -1033,7 +1033,7 @@ async def test_flush_harvests_late_capsule_and_persists_ready_version():
assert ledger.frozen_blocks == ()
# A later render (e.g. after a process resume) swaps normally.
ledger.commit_staged(step_key="s8", validator_result={"status": "success"})
ledger.commit_staged(step_key="s8", validator_result={"status": "dispatched"})
ledger.render([_observation(9)])
assert chunker.chunks and chunker.chunks[0].status == "ready"
@@ -1069,7 +1069,7 @@ def test_failed_capsule_redispatches_and_retains_original():
# Bounded retries exhausted → explicit failed state, chunk stays pending.
capsule._failed.add(key)
meter["value"] = 75_000 # pressure render = a re-dispatch occasion
ledger.commit_staged(step_key="s8", validator_result={"status": "success"})
ledger.commit_staged(step_key="s8", validator_result={"status": "dispatched"})
ledger.render([_observation(9)])
assert capsule.submitted.count(key) == 2 # re-dispatched
@@ -1088,7 +1088,7 @@ def test_hard_threshold_force_swaps_pending_chunks_into_l3():
assert ledger.frozen_blocks == ()
meter["value"] = 95_000 # ≥ 0.9 * 100k → hard
ledger.commit_staged(step_key="s8", validator_result={"status": "success"})
ledger.commit_staged(step_key="s8", validator_result={"status": "dispatched"})
ledger.render([_observation(9)])
assert chunker.awaiting_chunks == ()
@@ -1128,7 +1128,7 @@ def test_compression_announces_one_tool_line_from_dispatch_to_swap():
assert "context_tokens" not in args
capsule.resolve("chunk:1-4", _capsule(1, 4))
ledger.commit_staged(step_key="s8", validator_result={"status": "success"})
ledger.commit_staged(step_key="s8", validator_result={"status": "dispatched"})
ledger.render([_observation(9)])
done = engine.traces[-1]
@@ -1155,7 +1155,7 @@ def test_forced_swap_announces_snapshot_without_a_ratio():
assert any(t["status"] == "running" for t in engine.traces)
meter["value"] = 95_000
ledger.commit_staged(step_key="s8", validator_result={"status": "success"})
ledger.commit_staged(step_key="s8", validator_result={"status": "dispatched"})
ledger.render([_observation(9)]) # pressure closes the open tail too, then force-swaps all
running_ids = {t["trace_id"] for t in engine.traces if t["status"] == "running"}
@@ -1313,7 +1313,7 @@ def test_capsule_payload_carries_each_turn_transcript_as_the_operator_saw_it():
for i in range(1, 7):
ledger.commit_staged(
step_key=f"s{i - 1}" if i > 1 else None,
validator_result={"status": "success"} if i > 1 else None,
validator_result={"status": "dispatched"} if i > 1 else None,
)
ledger.render([_observation(i)])
ledger.stage_turn(_rich_turn(i) if i == 2 else _turn(i))
@@ -1329,7 +1329,7 @@ def test_capsule_payload_carries_each_turn_transcript_as_the_operator_saw_it():
assert '[tool call] ask_explorer({"question": "price?"})' in transcript
assert "[tool result ask_explorer]\nPrice shown: ¥39" in transcript
assert '[tool call] click({"target": 2})' in transcript
assert "--- Action Execution Result" in transcript and "Status: success" in transcript
assert "--- Action Execution Result" in transcript and "Status: dispatched" in transcript
assert transcript.index("Price shown") < transcript.index("click(")
# What the scrub edge already removed from the operator's view.
assert "Buy button" not in transcript and "IMG_2" not in transcript
@@ -1364,7 +1364,7 @@ def test_chunk_without_ledger_falls_back_to_per_step_projection():
def test_turn_transcript_is_capped_in_place_inside_the_capsule_request():
lens = StepCapsuleLens(model_name="test", llm=object())
cap = StepCapsuleLens.MAX_TURN_TRANSCRIPT_CHARS
step = {"step_number": 4, "offset": "T+00:40", "action": "tap", "outcome": "executed"}
step = {"step_number": 4, "offset": "T+00:40", "action": "tap", "outcome": "dispatched"}
payload = {
"start_step": 4,
"end_step": 4,
@@ -1399,7 +1399,7 @@ def test_visual_transition_line_is_skipped_when_the_transcript_already_has_it():
"step_number": 4,
"offset": "T+00:40",
"action": "tap",
"outcome": "executed",
"outcome": "dispatched",
"visual_summary": "The cart badge changed from 0 to 1.",
}
resolved = f"[observation]\n{HISTORY_SUMMARY_PREFIX}The cart badge changed from 0 to 1."
@@ -1479,15 +1479,15 @@ async def test_ready_capsule_releases_its_turn_transcripts():
def test_multi_action_turn_renders_one_block_with_every_recorded_step():
lens = StepCapsuleLens(model_name="test", llm=object())
steps = [
{"step_number": n, "offset": f"T+00:{n}0", "action": f"tap btn{n}", "outcome": "executed"}
{"step_number": n, "offset": f"T+00:{n}0", "action": f"tap btn{n}", "outcome": "dispatched"}
for n in (7, 8)
]
payload = {"start_step": 7, "end_step": 8, "steps": steps}
payload["turns"] = [{"steps": steps, "transcript": "[operator]\nfired two taps"}]
text = lens.build_messages(payload)[1].content
assert "## Steps 7–8 (T+00:70 → T+00:80)" in text
assert "- Step 7 (T+00:70): tap btn7 -> executed" in text
assert "- Step 8 (T+00:80): tap btn8 -> executed" in text
assert "- Step 7 (T+00:70): tap btn7 -> dispatched" in text
assert "- Step 8 (T+00:80): tap btn8 -> dispatched" in text
assert "fired two taps" in text
+6 -6
View File
@@ -131,7 +131,7 @@ def test_validator_result_message_carries_session_offset():
ledger,
2,
prev_key="step-1",
prev_result={"status": "success", "execution": [{"action": "tap", "attempts": ["ok"]}]},
prev_result={"status": "dispatched", "execution": [{"action": "tap", "attempts": ["ok"]}]},
)
result_messages = [
@@ -142,7 +142,7 @@ def test_validator_result_message_carries_session_offset():
assert len(result_messages) == 1
text = result_messages[0].content[0]["text"]
assert OFFSET_RE.search(text), text
assert "Status: success" in text
assert "Status: dispatched" in text
assert "ago" not in text
@@ -229,7 +229,7 @@ def test_tool_call_response_pairs_are_never_split():
ledger,
i,
prev_key=f"step-{i - 1}" if i > 1 else None,
prev_result={"status": "success"} if i > 1 else None,
prev_result={"status": "dispatched"} if i > 1 else None,
tool_call_count=2,
)
@@ -279,7 +279,7 @@ def test_no_ago_wording_in_ledger_output():
ledger,
i,
prev_key=f"step-{i - 1}" if i > 1 else None,
prev_result={"status": "success"} if i > 1 else None,
prev_result={"status": "dispatched"} if i > 1 else None,
)
blob = " ".join(str(m.content) for m in ledger.active_messages)
assert " ago" not in blob
@@ -406,7 +406,7 @@ def test_ledger_turn_transcript_reflects_the_scrubbed_active_region():
]
ledger.stage_turn(turn(1))
ledger.commit_staged(step_key="s1", validator_result={"status": "success"})
ledger.commit_staged(step_key="s1", validator_result={"status": "dispatched"})
ledger.stage_turn(turn(2))
ledger.commit_staged(step_key="s2", validator_result={"status": "failed", "error": "boom"})
ledger.render([HumanMessage(content=[{"type": "text", "text": "tail"}])])
@@ -415,6 +415,6 @@ def test_ledger_turn_transcript_reflects_the_scrubbed_active_region():
text = ledger.turn_transcript(first)
assert "thought 1" in text and "[tool call] click({})" in text
assert "Button 1" not in text # depth-1 strip already applied
assert "--- Action Execution Result (T+00:00) ---\nStatus: success" in text
assert "--- Action Execution Result (T+00:00) ---\nStatus: dispatched" in text
assert "Status: failed" in ledger.turn_transcript(second)
assert "thought 2" not in text # a turn transcript never leaks into another turn
+146
View File
@@ -0,0 +1,146 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Provider identity and Gemini capability table."""
from artemis.llm.google import (
gemini_version,
is_agentic_video_auto_eligible,
is_gemini_model,
is_google_chat_model,
is_google_family_provider,
is_google_provider,
resolve_video_processing,
strip_provider_prefix,
supports_agentic_video,
supports_thinking_level,
)
from artemis.llm.router import ModelProvider
import pytest
@pytest.mark.parametrize(
"value, google, family",
[
("google", True, True),
("Gemini", True, True),
(ModelProvider.GOOGLE, True, True),
(ModelProvider.GEMINI, True, True),
("vertexai", False, True),
("vertex-ai", False, True),
(ModelProvider.VERTEX_AI, False, True),
("openai", False, False),
("anthropic", False, False),
(None, False, False),
("", False, False),
],
)
def test_provider_identity_tiers(value, google, family):
assert is_google_provider(value) is google
assert is_google_family_provider(value) is family
def test_google_chat_model_is_detected_by_class_name():
class ChatGoogleGenerativeAI: # noqa: N801 - mirrors the real class name
pass
class ChatVertexAI:
pass
class ChatOpenAI:
pass
assert is_google_chat_model(ChatGoogleGenerativeAI()) is True
assert is_google_chat_model(ChatVertexAI()) is True
assert is_google_chat_model(ChatOpenAI()) is False
assert is_google_chat_model(None) is False
def test_model_name_helpers():
assert strip_provider_prefix("google/gemini-3.8-flash") == "gemini-3.8-flash"
assert strip_provider_prefix("gemini-3.8-flash") == "gemini-3.8-flash"
assert strip_provider_prefix(None) == ""
assert is_gemini_model("openrouter/google/gemini-3.7-flash") is True
assert is_gemini_model("claude-sonnet-5") is False
assert gemini_version("gemini-3.8-flash") == (3, 8)
assert gemini_version("gemini-3.5-flash-lite-preview") == (3, 5)
# Variants still report their family version (thinking-level gate).
assert gemini_version("gemini-3.8-flash-image") == (3, 8)
assert gemini_version("gemini-2.5-flash-tts") == (2, 5)
assert gemini_version("gemini-robotics-er-2-preview") is None
@pytest.mark.parametrize(
"model, expected",
[
("gemini-3.8-flash", True),
("gemini-3.5-flash-lite", True),
("gemini-2.5-flash", False),
("gemini-2.0-flash", False),
("gemini-1.5-pro", False),
("gemini-robotics-er-2-preview", True),
("gpt-5", True),
],
)
def test_supports_thinking_level(model, expected):
assert supports_thinking_level(model) is expected
@pytest.mark.parametrize(
"model, supported, auto",
[
("gemini-3.8-flash", True, True),
("gemini-3.7-flash", True, True),
("gemini-3.6-flash", True, True),
("google/gemini-3.9-flash", True, True),
("gemini-3.5-flash-lite", True, False),
("gemini-3.5-flash", False, False),
("gemini-3.8-pro", False, False),
("gemini-2.5-flash", False, False),
("gemini-robotics-er-2-preview", False, False),
("claude-sonnet-5", False, False),
# Release suffixes stay in the base family.
("models/gemini-3.7-flash-preview-09-2026", True, True),
("gemini-3.8-flash-latest", True, True),
("gemini-3.8-flash-001", True, True),
("gemini-3.8-flash-exp", True, True),
("gemini-3.5-flash-lite-preview", True, False),
# Variant models carry their own capabilities and never inherit.
("gemini-3.8-flash-image", False, False),
("gemini-3.8-flash-tts", False, False),
("gemini-3.8-flash-live", False, False),
("gemini-3.8-flash-native-audio", False, False),
("gemini-3.8-flash-image-preview", False, False),
("gemini-3.8-flash-8b", False, False),
],
)
def test_agentic_video_capability(model, supported, auto):
assert supports_agentic_video(model) is supported
assert is_agentic_video_auto_eligible(model) is auto
def test_resolve_video_processing_honours_knob_and_capability():
assert resolve_video_processing("auto", "gemini-3.8-flash") == "agentic"
assert resolve_video_processing("auto", "gemini-3.5-flash-lite") == "static"
assert resolve_video_processing("auto", "gemini-2.5-flash") == "static"
assert resolve_video_processing("agentic", "gemini-3.5-flash-lite") == "agentic"
assert resolve_video_processing("agentic", "gemini-2.5-flash") == "static"
assert resolve_video_processing("static", "gemini-3.8-flash") == "static"
# Explicit "agentic" is honoured on Flash-Lite but never on a variant.
assert resolve_video_processing("agentic", "gemini-3.8-flash-image") == "static"
assert resolve_video_processing("auto", "gemini-3.8-flash-tts") == "static"
# Unknown or non-string knobs behave like "auto".
assert resolve_video_processing(None, "gemini-3.8-flash") == "agentic"
assert resolve_video_processing(object(), "gemini-3.8-flash") == "agentic"
assert resolve_video_processing("bogus", "gemini-3.5-flash-lite") == "static"
+106
View File
@@ -0,0 +1,106 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Token-usage normalization across the LangChain, generate_content and Interactions shapes."""
from types import SimpleNamespace
from artemis.llm.google import normalize_usage, usage_from_message
from langchain_core.messages import AIMessage
def test_langchain_usage_metadata_shape():
usage = normalize_usage(
{
"input_tokens": 1200,
"output_tokens": 80,
"total_tokens": 1280,
"input_token_details": {"cache_read": 900},
"output_token_details": {"reasoning": 30},
}
)
assert usage == {
"prompt_tokens": 1200,
"completion_tokens": 80,
"total_tokens": 1280,
"cached_tokens": 900,
"thought_tokens": 30,
"tool_use_tokens": 0,
}
def test_generate_content_usage_metadata_object_shape():
raw = SimpleNamespace(
prompt_token_count=500,
candidates_token_count=25,
total_token_count=560,
cached_content_token_count=400,
thoughts_token_count=35,
tool_use_prompt_token_count=None,
)
assert normalize_usage(raw) == {
"prompt_tokens": 500,
"completion_tokens": 25,
"total_tokens": 560,
"cached_tokens": 400,
"thought_tokens": 35,
"tool_use_tokens": 0,
}
def test_interactions_usage_shape_counts_tool_use_and_thoughts():
raw = SimpleNamespace(
total_input_tokens=191,
total_output_tokens=655,
total_tokens=10571,
total_cached_tokens=0,
total_thought_tokens=12,
total_tool_use_tokens=9725,
)
assert normalize_usage(raw) == {
"prompt_tokens": 191,
"completion_tokens": 655,
"total_tokens": 10571,
"cached_tokens": 0,
"thought_tokens": 12,
"tool_use_tokens": 9725,
}
def test_total_is_derived_when_missing_and_empty_payloads_return_none():
assert normalize_usage({"prompt_tokens": 3, "completion_tokens": 4})["total_tokens"] == 7
assert normalize_usage(None) is None
assert normalize_usage({}) is None
assert normalize_usage({"prompt_tokens": "not-a-number"}) is None
assert normalize_usage(SimpleNamespace()) is None
def test_usage_from_message_prefers_langchain_then_response_metadata():
msg = AIMessage(content="ok")
msg.usage_metadata = {"input_tokens": 10, "output_tokens": 2, "total_tokens": 12}
assert usage_from_message(msg)["prompt_tokens"] == 10
fallback = AIMessage(content="ok")
fallback.response_metadata = {"token_usage": {"prompt_tokens": 7, "completion_tokens": 1}}
assert usage_from_message(fallback) == {
"prompt_tokens": 7,
"completion_tokens": 1,
"total_tokens": 8,
"cached_tokens": 0,
"thought_tokens": 0,
"tool_use_tokens": 0,
}
assert usage_from_message(AIMessage(content="none")) is None
assert usage_from_message(object()) is None
+7
View File
@@ -20,6 +20,7 @@ from unittest.mock import MagicMock
import pytest
from artemis.core.tool_failure import is_tool_failure
from artemis.tools.history import ReplayStepsTool, replay_steps, replay_steps_text
@@ -82,6 +83,12 @@ def test_replay_single_step_reversed_bounds_and_missing_step():
assert replay_steps_text(_reader([]), 9, 12) == "Error: no recorded steps in range 9–12."
assert "must be integers" in replay_steps_text(_reader([]), "x")
# Every refusal is a structural failure, not just "Error"-looking text.
assert is_tool_failure(replay_steps_text(_reader([]), 9))
assert is_tool_failure(replay_steps_text(_reader([]), 9, 12))
assert is_tool_failure(replay_steps_text(_reader([]), "x"))
assert not is_tool_failure(replay_steps_text(_reader([_friendly_step(2)]), 2))
def test_replay_caps_the_range_per_call():
reader = _reader([_friendly_step(n) for n in range(1, 30)])
+39
View File
@@ -31,6 +31,7 @@ import pytest
from PIL import Image
from artemis.context import ArtemisContext
from artemis.core.tool_failure import is_tool_failure
from artemis.data_engine.engine import DataEngine
from artemis.tools.history import (
SearchHistoryTool,
@@ -310,6 +311,17 @@ async def test_execute_requires_query_or_range(engine):
tool = SearchHistoryTool()
out = await tool.execute(ctx=SimpleNamespace(data_engine=engine), query="", step_range=None)
assert "needs a query and/or a step_range" in out
# A usage error is a request the tool did not serve: reported structurally.
assert is_tool_failure(out)
@pytest.mark.asyncio
async def test_execute_reports_history_load_error_structurally():
reader = MagicMock()
reader.get_agent_friendly_steps.side_effect = RuntimeError("db locked")
out = await search_history.execute(ctx=SimpleNamespace(data_engine=reader), query="login")
assert "search_history failed to load history: db locked" in out
assert is_tool_failure(out)
@pytest.mark.asyncio
@@ -332,3 +344,30 @@ def test_availability_requires_engine_and_config():
assert search_history.is_available(SimpleNamespace(data_engine=None)) is False
assert search_history.is_available(SimpleNamespace(data_engine=object())) is True
assert search_history_available(SimpleNamespace(data_engine=object())) is True
def test_action_matches_keep_target_provenance(engine):
"""The action haystack is the shared ledger rendering, so an excerpt shows a
self-described coordinate target with its marker and an observed index
target without one."""
from artemis.utils.task_tree import SELF_DESCRIBED_MARKER
engine.record_step(
summary="Started playback.",
action_taken={
"action": "click",
"coordinates": [500, 600],
"target_description": "play button",
},
last_execution_result={"status": "dispatched"},
)
_flush(engine)
described = search_history_text(engine, query="play button", recall_config=_cfg())
assert "[Step 4 " in described
assert f"'play button' {SELF_DESCRIBED_MARKER}" in described
observed = search_history_text(engine, query="Login entry", recall_config=_cfg())
assert "[Step 1 " in observed
assert "'Login entry'" in observed
assert SELF_DESCRIBED_MARKER not in observed
@@ -0,0 +1,86 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Every way a history tool fails to do what was asked is reported
structurally (a ``ToolFailure``), never as a plain string the caller would
have to sniff: no history, a usage error, or an exception from the reader."""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from artemis.core.tool_failure import is_tool_failure
from artemis.tools.history import (
GetStepScreenshotTool,
ReplayStepsTool,
SearchHistoryTool,
)
def _ctx():
return SimpleNamespace(data_engine=MagicMock())
@pytest.mark.asyncio
async def test_search_history_reader_exception_is_a_tool_failure():
with patch(
"artemis.tools.history.search_history_text", side_effect=RuntimeError("index corrupt")
):
out = await SearchHistoryTool().execute(ctx=_ctx(), query="login")
assert is_tool_failure(out)
assert out == "search_history failed: index corrupt"
@pytest.mark.asyncio
async def test_search_history_usage_error_is_a_tool_failure():
"""Asking for neither a query nor a range is a request the tool did not
serve: the guidance is the failure text the model reads."""
out = await SearchHistoryTool().execute(ctx=_ctx(), query="", step_range=None)
assert is_tool_failure(out)
assert "needs a query and/or a step_range" in out
@pytest.mark.asyncio
async def test_replay_steps_reader_exception_is_a_tool_failure():
with patch("artemis.tools.history.replay_steps_text", side_effect=OSError("db gone")):
out = await ReplayStepsTool().execute(ctx=_ctx(), start_step=3)
assert is_tool_failure(out)
assert out == "replay_steps failed: db gone"
@pytest.mark.asyncio
async def test_get_step_screenshot_exception_is_a_tool_failure():
with patch(
"artemis.tools.history.load_step_screenshot",
side_effect=FileNotFoundError("no screenshot for step 7"),
):
out = await GetStepScreenshotTool().execute(ctx=_ctx(), step_number=7, which="post")
assert is_tool_failure(out)
assert out == "get_step_screenshot failed: no screenshot for step 7"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("tool", "kwargs"),
[
(SearchHistoryTool(), {"query": "x"}),
(ReplayStepsTool(), {"start_step": 1}),
(GetStepScreenshotTool(), {"step_number": 1}),
],
ids=lambda v: getattr(v, "name", None),
)
async def test_missing_history_is_a_tool_failure(tool, kwargs):
assert is_tool_failure(await tool.execute(ctx=None, **kwargs))
assert is_tool_failure(await tool.execute(ctx=SimpleNamespace(data_engine=None), **kwargs))
+141
View File
@@ -18,7 +18,10 @@ import tempfile
import unittest
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from artemis.context import ArtemisContext
from artemis.core.tool_failure import ToolFailure
from artemis.tools.committee_tool import get_ask_committee_tool
@@ -210,3 +213,141 @@ class TestCommitteeTool(unittest.IsolatedAsyncioTestCase):
if __name__ == "__main__":
unittest.main()
# --- member tool results: status is structural, never sniffed from the words ----------
def _text_tool(name: str, result):
from langchain_core.tools import StructuredTool
async def _run(start_step: int, end_step: int | None = None) -> str:
return result
return StructuredTool.from_function(coroutine=_run, name=name, description=name)
async def _run_committee_with_history_tool(tmp_path, history_tool):
"""Drives one debate round whose History Analyzer calls ``history_tool``
once; returns the messages of that member's second turn."""
from langchain_core.messages import AIMessage
(tmp_path / "notes").mkdir()
screenshot = tmp_path / "shot.jpg"
screenshot.write_bytes(b"fake image bytes")
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
ctx.data_engine.get_agent_friendly_steps.return_value = [
{"pre_image_name": "test_image", "step_number": 1}
]
ctx.execution_setup = MagicMock()
ctx.execution_setup.committee_debate_rounds = 1
state = MagicMock()
state.initial_goal = "Test Goal"
state.messages = []
state.latest_ui_hierarchy = None
state.latest_screenshot = str(screenshot)
state.focused_app_info = None
state.device_date = None
state.structured_decisions = None
state.complete_subgoals_by_ids = []
def _text_turn(text):
return AIMessage(content=text, tool_calls=[])
llm_pl = MagicMock()
llm_pl.bind_tools.return_value = llm_pl
llm_pl.ainvoke = AsyncMock(side_effect=[_text_turn("Op: tap."), _text_turn("Final.")])
llm_diag = MagicMock()
llm_diag.bind_tools.return_value = llm_diag
llm_diag.ainvoke = AsyncMock(return_value=_text_turn("Diag: fine."))
llm_hist = MagicMock()
llm_hist.bind_tools.return_value = llm_hist
llm_hist.ainvoke = AsyncMock(
side_effect=[
AIMessage(
content="",
tool_calls=[
{"name": "replay_steps", "args": {"start_step": 1, "end_step": 1}, "id": "c1"}
],
),
_text_turn("Hist: step 1 seen."),
]
)
members = {
"planner_avatar": llm_pl,
"diagnoser_expert": llm_diag,
"history_analyzer_expert": llm_hist,
}
with (
patch(
"artemis.tools.committee_tool.get_llm",
side_effect=lambda ctx, name, temperature=None: members[name],
),
patch("artemis.tools.committee_tool.get_history_tools", return_value=[history_tool]),
patch("artemis.tools.committee_tool.trace_langchain_tool", side_effect=lambda t, ctx: t),
):
tool = get_ask_committee_tool(ctx)
from artemis.data_engine.trace import CURRENT_TRACE_ID
token = CURRENT_TRACE_ID.set("test_trace_id")
try:
outcome = await tool.ainvoke({"avatar_directive": "Advocate", "state": state})
finally:
CURRENT_TRACE_ID.reset(token)
assert "Final." in outcome
assert llm_hist.ainvoke.await_count == 2
return llm_hist.ainvoke.call_args_list[1].args[0]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"result, expected_status",
[
pytest.param(ToolFailure("Error: step 1 not found."), "error", id="tool_failure"),
pytest.param("Error 404 was typed into the search box", "success", id="plain_error_text"),
],
)
async def test_committee_member_tool_message_status_is_structural(
tmp_path, result, expected_status
):
from langchain_core.messages import ToolMessage
messages = await _run_committee_with_history_tool(tmp_path, _text_tool("replay_steps", result))
tool_msgs = [m for m in messages if isinstance(m, ToolMessage)]
assert len(tool_msgs) == 1
assert tool_msgs[0].tool_call_id == "c1"
assert tool_msgs[0].status == expected_status
assert tool_msgs[0].content == str(result)
@pytest.mark.asyncio
@pytest.mark.parametrize("tool_status", ["error", "success"])
async def test_committee_member_keeps_tool_message_status(tmp_path, tool_status):
"""A tool answering with a ``ToolMessage`` (video analyzer, note tools)
carries failure only in ``status``: its content is plain str. The member
loop must read the status off the raw message, not off the unwrapped
content, or every such failure is re-labelled a success."""
from langchain_core.messages import ToolMessage
returned = ToolMessage(tool_call_id="c1", content="boom", status=tool_status)
messages = await _run_committee_with_history_tool(
tmp_path, _text_tool("replay_steps", returned)
)
tool_msgs = [m for m in messages if isinstance(m, ToolMessage)]
assert len(tool_msgs) == 1
assert tool_msgs[0].tool_call_id == "c1"
assert tool_msgs[0].status == tool_status
assert tool_msgs[0].content == "boom"
+7 -5
View File
@@ -75,7 +75,7 @@ async def test_launch_app_direct_execution_with_driver(mock_driver):
"""Verify direct execution with BaseDeviceDriver."""
result = await launch_app.execute(driver=mock_driver, app_name="com.android.settings")
mock_driver.launch_app.assert_called_once_with("com.android.settings")
assert "App 'com.android.settings' launched successfully." in result
assert result == "Launched app 'com.android.settings'."
@pytest.mark.asyncio
@@ -96,7 +96,7 @@ async def test_launch_app_direct_execution_with_ctx(mock_ctx):
result = await launch_app.execute(ctx=mock_ctx, app_name="My App")
mock_find.assert_called_once_with(ctx=mock_ctx, app_name="My App")
mock_launch.assert_called_once_with(ctx=mock_ctx, app_package="com.example.myapp")
assert "App 'My App' launched successfully." in result
assert result == "Launched app 'My App' (com.example.myapp); foreground confirmed."
@pytest.mark.asyncio
@@ -154,14 +154,16 @@ async def test_launch_app_with_state_command(mock_ctx, mock_state):
assert isinstance(cmd, ToolMessage)
assert cmd.tool_call_id == "call_la_1"
assert cmd.status == "success"
assert "App 'State App' launched successfully." in cmd.content
assert (
cmd.content == "Launched app 'State App' (com.example.stateapp); foreground confirmed."
)
@pytest.mark.asyncio
async def test_launch_app_callable_execution(mock_driver):
"""Verify invoking launch_app directly as a callable."""
result = await launch_app(driver=mock_driver, app_name="com.android.chrome")
assert "App 'com.android.chrome' launched successfully." in result
assert result == "Launched app 'com.android.chrome'."
@pytest.mark.asyncio
@@ -190,4 +192,4 @@ async def test_get_launch_app_tool_langchain_ainvoke(mock_ctx):
),
):
result = await la_tool.ainvoke({"app_name": "Invoke App"})
assert "App 'Invoke App' launched successfully." in result
assert result == "Launched app 'Invoke App' (com.example.ainvoke); foreground confirmed."
+119
View File
@@ -0,0 +1,119 @@
"""Helper-tool failures are structural (``ToolFailure`` / ``ToolMessage.status``),
never inferred from the words of the text."""
from unittest.mock import Mock
from langchain_core.messages import ToolMessage
from langchain_core.tools import StructuredTool
import pytest
from artemis.core.tool_failure import ToolFailure, is_tool_failure
from artemis.data_engine import engine as engine_mod
from artemis.tools.tool_wrapper import invoke_tool_with_injection
from artemis.utils.notes import format_list_notes_failure, format_read_note_failure
def test_tool_failure_is_a_plain_string_to_every_text_consumer():
failure = ToolFailure("Failed to save note plan.md: disk full")
assert isinstance(failure, str)
assert failure == "Failed to save note plan.md: disk full"
assert f"{failure}" == "Failed to save note plan.md: disk full"
assert failure.startswith("Failed")
def test_is_tool_failure_reads_the_structure_not_the_words():
assert is_tool_failure(ToolFailure("Error: note not found"))
assert is_tool_failure(ToolFailure("plain wording still counts"))
# Free-form text may legitimately start with "Error" or "Failed".
assert not is_tool_failure("Error 404 was typed into the search box")
assert not is_tool_failure("Failed login dialog is visible")
assert not is_tool_failure("Successfully saved note")
assert not is_tool_failure(None)
def test_is_tool_failure_honours_the_tool_message_status():
assert is_tool_failure(ToolMessage(tool_call_id="t", content="anything", status="error"))
assert not is_tool_failure(
ToolMessage(tool_call_id="t", content="Error-looking text", status="success")
)
def test_is_tool_failure_finds_a_failure_text_block():
assert is_tool_failure([{"type": "text", "text": ToolFailure("boom")}])
assert is_tool_failure([ToolFailure("boom"), {"type": "image_url", "image_url": {}}])
assert not is_tool_failure([{"type": "text", "text": "Error: fine"}])
def test_note_failure_formatters_report_structurally():
assert is_tool_failure(format_read_note_failure("plan", "file not found"))
assert is_tool_failure(format_read_note_failure("plan", "permission denied"))
assert is_tool_failure(format_list_notes_failure("boom"))
def test_tool_message_content_coerces_a_tool_failure_to_plain_str():
"""Pydantic drops the ``str`` subclass: the failure must be read from the raw
tool return *before* the message is built, and carried as ``status``."""
raw = ToolFailure("boom")
assert is_tool_failure(raw)
msg = ToolMessage(tool_call_id="t", content=raw)
assert type(msg.content) is str
assert is_tool_failure(msg.content) is False
assert is_tool_failure(msg) is False # default status is "success"
explicit = ToolMessage(
tool_call_id="t", content=raw, status="error" if is_tool_failure(raw) else "success"
)
assert is_tool_failure(explicit)
# --- invoke_tool_with_injection: the trace span follows the structure ---------------
def _text_tool(name: str, result):
async def _run(query: str) -> str:
return result
return StructuredTool.from_function(coroutine=_run, name=name, description=name)
def _stub_engine(monkeypatch):
engine = Mock()
engine.current_step_id = "step-1"
monkeypatch.setattr(engine_mod, "_CURRENT_DATA_ENGINE", engine, raising=False)
return engine
def _final_trace_call(engine):
calls = [c for c in engine.record_trace.call_args_list if c.kwargs.get("status") != "running"]
assert len(calls) == 1
return calls[0].kwargs
@pytest.mark.asyncio
async def test_invoke_tool_with_injection_traces_a_tool_failure_as_failed(monkeypatch):
engine = _stub_engine(monkeypatch)
tool = _text_tool("read_note", ToolFailure("Failed to read note plan.md: not found"))
result = await invoke_tool_with_injection(tool, {"query": "plan.md"}, "tc-1")
assert is_tool_failure(result)
span = _final_trace_call(engine)
assert span["status"] == "failed"
assert span["name"] == "read_note"
assert span["payload"]["error"] == "Failed to read note plan.md: not found"
assert "result" not in span["payload"]
@pytest.mark.asyncio
async def test_invoke_tool_with_injection_traces_plain_error_text_as_success(monkeypatch):
engine = _stub_engine(monkeypatch)
tool = _text_tool("search_history", "Error: 3 matches for the typed search term")
result = await invoke_tool_with_injection(tool, {"query": "Error:"}, "tc-2")
assert not is_tool_failure(result)
span = _final_trace_call(engine)
assert span["status"] == "success"
assert span["payload"]["result"] == "Error: 3 matches for the typed search term"
assert "error" not in span["payload"]
+3 -3
View File
@@ -65,7 +65,7 @@ async def test_wait_direct_execution(mock_ctx):
result = await wait.execute(ctx=mock_ctx, seconds=5)
mock_sleep.assert_called_once_with(5)
assert result == "Successfully waited for 5 seconds."
assert result == "Waited 5 seconds."
@pytest.mark.asyncio
@@ -77,7 +77,7 @@ async def test_wait_callable_execution(mock_ctx):
result = await wait(ctx=mock_ctx, seconds=3)
mock_sleep.assert_called_once_with(3)
assert result == "Successfully waited for 3 seconds."
assert result == "Waited 3 seconds."
@pytest.mark.asyncio
@@ -90,7 +90,7 @@ async def test_wait_tool(mock_ctx):
duration = end_time - start_time
assert duration >= 1.9 # Allow slight timing variations
assert "Successfully waited for 2 seconds" in result
assert "Waited 2 seconds" in result
@pytest.mark.asyncio
+50
View File
@@ -0,0 +1,50 @@
"""The ``attempts`` marker contract between the Validator and the renderers.
The Validator writes ``"Dispatched"`` as the terminal attempt of an action the
device accepted (kept only when a retry preceded it); every failed attempt keeps
its error text. The renderers must read a retried-then-dispatched action as a
clean dispatch, never as a failure.
"""
from artemis.utils.task_tree import (
_burst_member_status,
failed_execution_error,
format_result_clean,
)
def _report(attempts: list[str], status: str = "dispatched") -> dict:
return {
"status": status,
"execution": [
{"action": "tap", "normalized_coordinates": [500, 500], "attempts": attempts}
],
}
def test_retried_then_dispatched_action_is_not_a_failure():
report = _report(["Error: tap rejected", "Dispatched"])
assert failed_execution_error(report) is None
assert format_result_clean(report) is None
def test_terminal_failure_keeps_every_attempt():
report = _report(["Error: tap rejected", "Error: tap rejected again"], status="failed")
assert failed_execution_error(report) == "Error: tap rejected | Error: tap rejected again"
assert format_result_clean(report) == "Error: tap rejected | Error: tap rejected again"
def test_burst_member_labels_follow_the_dispatch_vocabulary():
report = {
"status": "failed",
"execution": [
{"action": "tap"},
{"action": "tap", "attempts": ["Error: boom"]},
{"action": "tap", "attempts": ["Skipped (burst aborted)"]},
],
}
assert _burst_member_status(report, 0) == " (dispatched)"
assert _burst_member_status(report, 1) == " (FAILED: Error: boom)"
assert _burst_member_status(report, 2) == " (skipped)"
assert _burst_member_status(report, 3) == " (not dispatched)"
assert _burst_member_status({"execution": [{"attempts": ["Dispatched"]}]}, 0) == " (dispatched)"
+9 -26
View File
@@ -12,9 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for record-time action semantics enrichment (element hit testing)."""
"""Tests for element hit testing (used by the Explorer's candidate de-dup)."""
from artemis.utils.element_hit_test import find_element_at_point, hit_test_semantics
from artemis.utils.element_hit_test import find_element_at_point
def _xml_el(text, bounds, resource_id=None, cls="android.widget.TextView"):
@@ -37,14 +37,6 @@ def test_smallest_covering_element_wins():
assert source == "hit_test"
assert el["text"] == "Button"
semantics = hit_test_semantics(elements, 250, 230)
assert semantics == {
"target_label_source": "hit_test",
"target_text": "Button",
"target_class": "android.widget.TextView",
"target_resource_id": "btn_ok",
}
def test_xml_element_preferred_over_smaller_ocr_element():
elements = [
@@ -73,11 +65,9 @@ def test_ocr_fallback_when_no_xml_element_covers_point():
"is_ocr": True,
},
]
semantics = hit_test_semantics(elements, 150, 150)
assert semantics["target_label_source"] == "ocr"
assert semantics["target_text"] == "OCR Label"
assert "target_class" not in semantics
assert "target_resource_id" not in semantics
el, source = find_element_at_point(elements, 150, 150)
assert source == "ocr"
assert el["text"] == "OCR Label"
def test_string_bounds_are_parsed():
@@ -88,13 +78,12 @@ def test_string_bounds_are_parsed():
def test_graceful_degradation_to_none():
# No elements at all
assert hit_test_semantics(None, 100, 100) == {"target_label_source": "none"}
assert hit_test_semantics([], 100, 100) == {"target_label_source": "none"}
assert find_element_at_point(None, 100, 100) == (None, "none")
assert find_element_at_point([], 100, 100) == (None, "none")
# Point outside every element
elements = [_xml_el("Item", [0, 0, 50, 50])]
assert hit_test_semantics(elements, 500, 500) == {"target_label_source": "none"}
assert find_element_at_point(elements, 500, 500) == (None, "none")
# Malformed element data must not raise
malformed = [
@@ -105,10 +94,4 @@ def test_graceful_degradation_to_none():
"not-a-dict",
None,
]
assert hit_test_semantics(malformed, 10, 10) == {"target_label_source": "none"}
def test_element_without_optional_fields_only_reports_source():
elements = [{"text": "", "bounds": [0, 0, 100, 100], "class": None, "resource_id": None}]
semantics = hit_test_semantics(elements, 50, 50)
assert semantics == {"target_label_source": "hit_test"}
assert find_element_at_point(malformed, 10, 10) == (None, "none")
+86 -1
View File
@@ -532,7 +532,7 @@ def test_build_plan_and_history_fast_action_burst():
"* [Planned Fast-Action Burst]: 3 actions fired back to back without the"
" safety net" in output
)
assert " 1. Tapped element at [500, 900] (executed)" in output
assert " 1. Tapped element at [500, 900] (dispatched)" in output
assert " 2. Tapped 'Skip' at [880, 120] (FAILED: Error: tap rejected)" in output
assert " 3. Pressed key 'BACK' (skipped)" in output
assert (
@@ -751,3 +751,88 @@ def test_format_action_clean_reads_flash_arguments_and_maps_manage_app():
)
== "Tapped 'Save' at [1, 2]"
)
def test_format_action_clean_marks_self_described_coordinate_targets():
"""A coordinate target's label is the model's own description, and every
later reader sees it marked as such; an index target's observed element
text carries no marker."""
from artemis.utils.task_tree import SELF_DESCRIBED_MARKER, format_action_clean
assert SELF_DESCRIBED_MARKER == "(self-described)"
assert (
format_action_clean(
{"action": "tap", "coordinates": [540, 1440], "target_description": "play button"}
)
== "Tapped 'play button' (self-described) at [540, 1440]"
)
# Observed text outranks the description when both are present, unmarked.
assert (
format_action_clean(
{
"action": "tap",
"coordinates": [1, 2],
"target_text": "Play",
"target_description": "play button",
}
)
== "Tapped 'Play' at [1, 2]"
)
assert (
format_action_clean(
{
"action": "long_press_on",
"coordinates": [1, 2],
"duration": 1500,
"target_description": "app icon",
}
)
== "Long pressed 'app icon' (self-described) at [1, 2] for 1500ms"
)
assert (
format_action_clean(
{
"action": "focus_and_input_text",
"coordinates": [3, 4],
"text": "hello",
"target_description": "search box",
}
)
== "Inputted 'hello' into 'search box' (self-described) at [3, 4]"
)
assert (
format_action_clean(
{
"action": "swipe",
"coordinates": [100, 500, 900, 500],
"duration": 800,
"target_description": "brightness knob",
}
)
== "Swiped 'brightness knob' (self-described) from [100, 500] to [900, 500] over 800ms"
)
# Flash keeps the description under ``args``; it is read from there too.
assert (
format_action_clean(
{
"action": "click",
"coordinates": [320, 399],
"args": {"target": [320, 399], "target_description": "skip ad"},
}
)
== "Tapped 'skip ad' (self-described) at [320, 399]"
)
assert format_action_clean(
{
"action": "click_sequence",
"coordinates": [[500, 280], [885, 362]],
"target_descriptions": ["video body", "skip button"],
}
) == (
"Tapped sequence of targets: 'video body' (self-described) at [500, 280],"
" 'skip button' (self-described) at [885, 362]"
)
# Unnamed targets stay unnamed: no marker without a description.
assert format_action_clean({"action": "tap", "coordinates": [1, 2]}) == (
"Tapped element at [1, 2]"
)