feat: add agentic video analysis and clarify action results

Report device actions as dispatched and preserve full failure details. Route
Flash through McpActionExecutor and use ToolFailure across helper tools and
agent loops instead of inferring failure from response text.

Require model-provided descriptions for coordinate targets. Keep observed
element metadata separate from those descriptions in safety checks, visual
summaries, history, and frontend labels.

Add Gemini Interactions API video analysis with configurable processing mode,
chunk size, and timeouts. Centralize Google provider capabilities and token
usage normalization, and update google-genai to 2.22.0.

Preserve system instructions on follow-up interactions, replan pending chunks
when falling back to static processing, and reject unanswered native video
and audio segments instead of caching placeholder results. Keep helper-tool
failure status intact through history tools and committee calls.

Validation: 1,762 Python unit tests passed (3 skipped), 40 frontend formatter
tests passed, and the production UI build completed. Ruff lint, formatting,
the quality ratchet, and git diff checks passed.

tests/unit/mcp/test_action_executor_helper_tools.py

@@ -130,59 +130,41 @@ async def test_ask_explorer_exception_is_contained():
     assert result.text_summary == "Error executing ask_explorer: boom"
 
 
-# ---------------------------------------------------------------------------
-# Legacy MobileActionExecutor parity (same ask_explorer semantics)
-# ---------------------------------------------------------------------------
-
-
-def _make_legacy_executor(**kwargs):
-    from artemis.agents.validator.tool_declarations import MobileActionExecutor
-
-    ctx = Mock()
-    ctx.data_engine = None
-    ctx.device.device_width = 1080
-    ctx.device.device_height = 2400
-    actuator = Mock()
-    actuator.controller = Mock()
-    return MobileActionExecutor(ctx, actuator=actuator, **kwargs)
-
-
 @pytest.mark.asyncio
-async def test_legacy_executor_ask_explorer_status_follows_the_outcome():
-    executor = _make_legacy_executor()
+async def test_default_agent_name_reaches_the_tier_resolver_and_status_follows_the_outcome():
+    """Without an explicit agent name the executor asks the Explorer as ``validator``;
+    a clean not-found is an answer, an Explorer run failure is an error."""
+    executor = _make_executor()
     assert executor.agent_name == "validator"
 
     not_found = ExplorerOutcome(message="Not visible.")
-    with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=not_found)):
-        # The pipeline functions are imported at module load in the legacy executor.
-        with patch(
-            "artemis.agents.validator.tool_declarations.locate",
-            new=AsyncMock(return_value=not_found),
-        ) as locate:
-            result = await executor.execute(
-                "ask_explorer", {"task_description": "gear icon"}, "tc", _state()
-            )
+    with patch(
+        "artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=not_found)
+    ) as locate:
+        result = await executor.execute(
+            "ask_explorer", {"task_description": "gear icon"}, "tc", _state()
+        )
     assert locate.await_args.args[2] == "gear icon"
     assert locate.await_args.kwargs["agent_name"] == "validator"
-    assert result.status == "success"  # a clean not-found is an answer, not a failure
+    assert result.status == "success"
 
     failed = ExplorerOutcome.failure("Explorer failed: boom")
-    with patch(
-        "artemis.agents.validator.tool_declarations.locate", new=AsyncMock(return_value=failed)
-    ):
+    with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=failed)):
         result = await executor.execute("ask_explorer", {"query": "gear icon"}, "tc", _state())
     assert result.status == "error"
 
 
 @pytest.mark.asyncio
-async def test_legacy_executor_agent_name_reaches_the_tier_resolver():
-    executor = _make_legacy_executor(agent_name="flash")
+async def test_flash_agent_name_reaches_the_tier_resolver():
+    """The Flash profile resolves the Explorer tier from ``agent_name="flash"``."""
+    executor = _make_executor(agent_name="flash")
     with patch(
-        "artemis.agents.validator.tool_declarations.locate",
+        "artemis.tools.explorer_tool.locate",
         new=AsyncMock(return_value=ExplorerOutcome(message="no")),
     ) as locate:
         await executor.execute("ask_explorer", {"query": "x"}, "tc", _state())
     assert locate.await_args.kwargs["agent_name"] == "flash"
+    assert locate.await_args.args[0] is executor.ctx
 
 
 # ---------------------------------------------------------------------------
@@ -293,19 +275,21 @@ async def test_history_tool_exception_is_contained():
     tool = _history_tool_double(side_effect=RuntimeError("db gone"))
     with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool):
         result = await executor.execute("replay_steps", {"query": "x"}, "tc", None)
-    assert result.status == "success"  # contained: the answer explains the failure
+    # Contained (no exception escapes) and reported structurally: the answer
+    # explains the failure and the status says it is one.
+    assert result.status == "error"
     assert result.text_summary == "replay_steps failed: db gone"
 
 
 @pytest.mark.asyncio
 async def test_history_tools_dispatch_to_the_real_shared_instances():
     """No per-tool branch: every history tool name resolves through the package."""
-    executor = _make_executor()  # ctx.data_engine is None -> tools degrade to text
+    executor = _make_executor()  # ctx.data_engine is None -> tools report a failure
     for name in ("search_history", "replay_steps", "get_step_screenshot"):
         result = await executor.execute(
             name, {"query": "x", "start_step": 1, "step_number": 1}, "tc", None
         )
-        assert result.status == "success"
+        assert result.status == "error"
         assert (
             "no active execution history" in result.text_summary
             or "no execution history" in result.text_summary

tests/unit/mcp/test_action_executor_semantics.py

@@ -12,11 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Tests for record-time target semantics enrichment in McpActionExecutor."""
+"""Coordinate targets must be described by the model, never inferred.
+
+The Flash/Validator dialect addresses every target by coordinates, so the
+executor refuses a click, long press, focused input, coordinate swipe or click
+sequence that does not say what it is aiming at. The description is recorded
+by the runner and never reaches the wire.
+"""
 
 from unittest.mock import Mock
 
-from artemis.mcp.action_executor import McpActionExecutor
+import pytest
+
+from artemis.mcp.action_executor import McpActionExecutor, _ArgError
 
 
 def _make_executor(width=1080, height=2400):
@@ -28,131 +36,195 @@ def _make_executor(width=1080, height=2400):
     return McpActionExecutor(ctx, actuator=actuator)
 
 
-def _state_with_elements(elements):
+def _state():
     state = Mock()
-    state.indexed_elements = elements
+    state.indexed_elements = []
+    state.indexed_points = [[540, 1440], [745, 1440]]
+    state.latest_ui_hierarchy = None
     return state
 
 
-def test_click_target_semantics_from_pre_action_frame():
+@pytest.mark.parametrize(
+    "name, args, label",
+    [
+        ("click", {"target": [500, 600]}, "click"),
+        ("long_press", {"target": [500, 600]}, "long press"),
+        ("input_text", {"text": "hi", "target": [500, 600]}, "input text"),
+        ("swipe", {"start": [200, 600], "end": [800, 600]}, "swipe"),
+    ],
+)
+def test_coordinate_targets_without_description_are_refused(name, args, label):
     executor = _make_executor()
-    # Normalized (500, 600) on a 1080x2400 device -> pixel (540, 1440).
-    state = _state_with_elements(
-        [
-            {
-                "text": "Confirm",
-                "bounds": [500, 1400, 600, 1480],
-                "class": "android.widget.Button",
-                "resource_id": "btn_confirm",
-                "is_ocr": False,
-            }
-        ]
+    with pytest.raises(_ArgError) as excinfo:
+        executor._translate(name, args, _state())
+    message = str(excinfo.value)
+    assert message.startswith(f"Error during {label}")
+    assert "target_description" in message
+
+
+@pytest.mark.parametrize("blank", ["", "   ", None, 7])
+def test_blank_or_non_string_description_is_refused(blank):
+    executor = _make_executor()
+    with pytest.raises(_ArgError):
+        executor._translate("click", {"target": [500, 600], "target_description": blank}, _state())
+
+
+def test_description_is_accepted_and_kept_off_the_wire():
+    executor = _make_executor()
+    wire_name, wire_args, _, _ = executor._translate(
+        "click", {"target": [500, 600], "target_description": "play button"}, _state()
     )
-    semantics = executor._target_semantics("click", {"target": [500, 600]}, state)
-    assert semantics == {
-        "target_label_source": "hit_test",
-        "target_text": "Confirm",
-        "target_class": "android.widget.Button",
-        "target_resource_id": "btn_confirm",
-    }
+    assert wire_name == "click"
+    assert wire_args == {"target": [500, 600], "times": 1, "delay_ms": 100}
+    assert "target_description" not in wire_args
 
-
-def test_semantics_degrade_to_none_without_element_data():
-    executor = _make_executor()
-
-    # Empty perception data
-    semantics = executor._target_semantics(
-        "click", {"target": [500, 600]}, _state_with_elements([])
+    _, swipe_args, _, _ = executor._translate(
+        "swipe",
+        {"start": [200, 600], "end": [800, 600], "target_description": "brightness knob"},
+        _state(),
     )
-    assert semantics == {"target_label_source": "none"}
-
-    # No state at all
-    semantics = executor._target_semantics("click", {"target": [500, 600]}, None)
-    assert semantics == {"target_label_source": "none"}
+    assert swipe_args == {"start": [200, 600], "end": [800, 600], "duration_ms": 400}
 
 
-def test_non_targeted_actions_and_missing_targets_skip_enrichment():
+def test_recorded_target_is_the_cleaned_description_in_pro_shape():
+    """The recorded semantics come from the same check that requires them."""
     executor = _make_executor()
-    state = _state_with_elements([{"text": "X", "bounds": [0, 0, 10, 10]}])
-
-    assert executor._target_semantics("press_key", {"key": "BACK"}, state) is None
-    # input_text without a coordinate target (focused field typing)
-    assert executor._target_semantics("input_text", {"text": "hi", "target": None}, state) is None
-    # Multi-point actions without resolvable points skip enrichment too.
-    assert executor._target_semantics("click_sequence", {"sequence": []}, state) is None
-    assert executor._target_semantics("swipe", {"start": None, "end": None}, state) is None
-
-
-def test_click_sequence_hit_tests_every_point_first_point_is_main_label():
-    """M5: multi-point actions get per-point best-effort semantics; the first
-    point's fields are hoisted as the action's main label."""
-    executor = _make_executor()
-    state = _state_with_elements(
-        [
-            {
-                "text": "Digit 1",
-                "bounds": [500, 1400, 600, 1480],
-                "class": "android.widget.Button",
-                "is_ocr": False,
-            },
-            {
-                "text": "Digit 2",
-                "bounds": [700, 1400, 800, 1480],
-                "class": "android.widget.Button",
-                "is_ocr": False,
-            },
-        ]
+    _, _, _, recorded = executor._translate(
+        "click", {"target": [1, 2], "target_description": " ok "}, _state()
     )
-    # Normalized points -> pixels: (500,600)->(540,1440) hits Digit 1;
-    # (690,600)->(745,1440) hits Digit 2; (10,10)->(10,24) hits nothing.
-    semantics = executor._target_semantics(
+    assert recorded == {"target_description": "ok"}
+
+    _, _, _, recorded = executor._translate(
+        "long_press", {"target": [1, 2], "target_description": "thumbnail"}, _state()
+    )
+    assert recorded == {"target_description": "thumbnail"}
+
+    _, _, _, recorded = executor._translate(
+        "click_sequence", {"sequence": [[1, 2]], "target_descriptions": ["a "]}, _state()
+    )
+    assert recorded == {"target_descriptions": ["a"]}
+
+    # No coordinate target: nothing is recorded, nothing inferred.
+    _, _, _, recorded = executor._translate("press_key", {"key": "BACK"}, _state())
+    assert recorded == {}
+
+
+def test_coordinate_swipe_records_its_description():
+    executor = _make_executor()
+    _, wire_args, _, recorded = executor._translate(
+        "swipe",
+        {"start": [200, 600], "end": [800, 600], "target_description": " brightness knob "},
+        _state(),
+    )
+    assert recorded == {"target_description": "brightness knob"}
+    assert "target_description" not in wire_args
+
+
+def test_targeted_input_records_its_description():
+    executor = _make_executor()
+    _, wire_args, _, recorded = executor._translate(
+        "input_text",
+        {"text": "hi", "target": [500, 600], "target_description": "search input"},
+        _state(),
+    )
+    assert wire_args["target"] == [500, 600]
+    assert recorded == {"target_description": "search input"}
+    assert "target_description" not in wire_args
+
+
+def test_focused_typing_without_a_target_needs_no_description():
+    executor = _make_executor()
+    _, wire_args, _, recorded = executor._translate(
+        "input_text", {"text": "hi", "target": None}, _state()
+    )
+    assert wire_args["target"] is None
+    assert recorded == {}
+
+
+def test_focused_typing_ignores_a_stray_description():
+    """Typing into the focused field has no coordinate target: a description the
+    model tacked on anyway is not recorded, so no self-described target enters
+    the history for an action that aimed at nothing."""
+    executor = _make_executor()
+    _, wire_args, _, recorded = executor._translate(
+        "input_text",
+        {"text": "hi", "target": None, "target_description": "search input"},
+        _state(),
+    )
+    assert wire_args["target"] is None
+    assert recorded == {}
+
+
+def test_directional_swipe_needs_no_description():
+    executor = _make_executor()
+    wire_name, wire_args, _, recorded = executor._translate("swipe", {"direction": "up"}, _state())
+    assert wire_name == "swipe"
+    assert "start" in wire_args and "end" in wire_args
+    assert recorded == {}
+
+
+def test_directional_swipe_ignores_a_stray_description():
+    executor = _make_executor()
+    _, wire_args, _, recorded = executor._translate(
+        "swipe", {"direction": "up", "target_description": "the feed"}, _state()
+    )
+    assert "target_description" not in wire_args
+    assert recorded == {}
+
+
+def test_click_sequence_requires_one_description_per_entry():
+    executor = _make_executor()
+    sequence = [[500, 600], [690, 600], [10, 10]]
+
+    with pytest.raises(_ArgError) as excinfo:
+        executor._translate("click_sequence", {"sequence": sequence}, _state())
+    assert "target_descriptions" in str(excinfo.value)
+    assert "3 expected" in str(excinfo.value)
+
+    # Wrong length is refused too.
+    with pytest.raises(_ArgError):
+        executor._translate(
+            "click_sequence",
+            {"sequence": sequence, "target_descriptions": ["video body", "skip"]},
+            _state(),
+        )
+
+    _, wire_args, _, recorded = executor._translate(
         "click_sequence",
-        {"sequence": [[500, 600], [690, 600], [10, 10]], "delay_ms": 50},
-        state,
+        {"sequence": sequence, "target_descriptions": ["video body", "digit 2", "corner"]},
+        _state(),
     )
-    assert semantics["target_text"] == "Digit 1"
-    assert semantics["target_label_source"] == "hit_test"
-    per_point = semantics["points_semantics"]
-    assert len(per_point) == 3
-    assert per_point[0]["target_text"] == "Digit 1"
-    assert per_point[1]["target_text"] == "Digit 2"
-    assert per_point[2] == {"target_label_source": "none"}
+    assert wire_args["sequence"] == [[500, 600], [690, 600], [10, 10]]
+    assert "target_descriptions" not in wire_args
+    assert recorded == {"target_descriptions": ["video body", "digit 2", "corner"]}
 
 
-def test_swipe_hit_tests_start_and_end_start_is_main_label():
+@pytest.mark.parametrize("index_entry", [2, "2", 2.0, [2]])
+def test_click_sequence_refuses_element_indices(index_entry):
+    """click_sequence takes coordinate pairs only. An index would be resolved
+    against the element list and then recorded under the model's own
+    description, reading back as a self-described coordinate target."""
     executor = _make_executor()
-    state = _state_with_elements(
-        [
-            {
-                "text": "Brightness slider",
-                "bounds": [100, 1400, 900, 1480],
-                "class": "android.widget.SeekBar",
-                "is_ocr": False,
-            }
-        ]
-    )
-    semantics = executor._target_semantics(
-        "swipe", {"start": [200, 600], "end": [800, 600], "duration_ms": 400}, state
-    )
-    assert semantics["target_text"] == "Brightness slider"
-    assert semantics["start_semantics"]["target_text"] == "Brightness slider"
-    assert semantics["end_semantics"]["target_text"] == "Brightness slider"
-    assert semantics["target_label_source"] == "hit_test"
+    sequence = [[500, 600], index_entry, [10, 10]]
+
+    with pytest.raises(_ArgError) as excinfo:
+        executor._translate(
+            "click_sequence",
+            {"sequence": sequence, "target_descriptions": ["video body", "digit 2", "corner"]},
+            _state(),
+        )
+    message = str(excinfo.value)
+    assert message.startswith("Error during click sequence")
+    assert "entry 2" in message
+    assert "coordinate" in message
+    assert "ask_explorer" in message
 
 
-def test_long_press_uses_ocr_fallback():
+def test_click_sequence_accepts_a_serialized_pair_list():
     executor = _make_executor()
-    state = _state_with_elements(
-        [
-            {
-                "text": "OCR Word",
-                "bounds": [520, 1420, 560, 1460],
-                "class": None,
-                "resource_id": None,
-                "is_ocr": True,
-            }
-        ]
+    _, wire_args, _, _ = executor._translate(
+        "click_sequence",
+        {"sequence": "[[500, 600], [10, 10]]", "target_descriptions": ["video body", "corner"]},
+        _state(),
     )
-    semantics = executor._target_semantics("long_press", {"target": [500, 600]}, state)
-    assert semantics["target_label_source"] == "ocr"
-    assert semantics["target_text"] == "OCR Word"
+    assert wire_args["sequence"] == [[500, 600], [10, 10]]

tests/unit/mcp/test_action_manifest.py

@@ -55,7 +55,8 @@ SCOPED_PROMPT_FILES = [
 # note keys, and the availability-slot template helper rendered by
 # apply_operator_prompt_contract. Anything new landing here should be a conscious
 # decision.
-NON_TOOL_IDENTIFIERS = frozenset({"analysis", "task_plan", "tool_enum"})
+#: Backticked identifiers in prompts that are parameters or note keys, not tools.
+NON_TOOL_IDENTIFIERS = frozenset({"analysis", "task_plan", "tool_enum", "target_description"})
 
 _TOOL_REF = re.compile(r"(?:`([a-z][a-z0-9_]{2,})`|\b([a-z][a-z0-9_]{2,})\()")
 

tests/unit/mcp/test_action_server.py

@@ -40,7 +40,7 @@ async def test_click_end_to_end():
     try:
         res = await session.call("click", {"target": [500, 500]})
         assert res.ok
-        assert res.message == "Clicked at [500, 500] (normalized) successfully."
+        assert res.message == "Tapped at [500, 500] (normalized)."
         assert actuator.action_history[-1]["action"] == "tap"
         assert actuator.action_history[-1]["x"] == 500
         assert actuator.action_history[-1]["y"] == 1000

tests/unit/mcp/test_actuators.py

@@ -38,7 +38,7 @@ async def test_click_converts_normalized_to_pixels(actuator):
     res = await actuator.click(500, 500)
     assert res.ok and res.code is ActionCode.OK
     # Historical wording is part of the contract: transcripts and traces assert on it.
-    assert res.message == "Clicked at [500, 500] (normalized) successfully."
+    assert res.message == "Tapped at [500, 500] (normalized)."
     assert res.normalized_coordinates == [500, 500]
     assert actuator.action_history[-1] == {
         "action": "tap",
@@ -53,9 +53,7 @@ async def test_click_converts_normalized_to_pixels(actuator):
 async def test_click_sequence_taps_each_point(actuator):
     res = await actuator.click_sequence([(100, 100), (900, 900)], delay_ms=1)
     assert res.ok
-    assert res.message == (
-        "Sequence clicked successfully: Tapped at [100, 200]; Tapped at [900, 1800]"
-    )
+    assert res.message == ("Tapped in sequence at [100, 100]; [900, 900] (normalized).")
     taps = [h for h in actuator.action_history if h["action"] == "tap"]
     assert [(t["x"], t["y"]) for t in taps] == [(100, 200), (900, 1800)]
 
@@ -64,7 +62,7 @@ async def test_click_sequence_taps_each_point(actuator):
 async def test_long_press_message_and_duration(actuator):
     res = await actuator.long_press(250, 750, duration_ms=1500)
     assert res.ok
-    assert res.message == ("Long pressed at [250, 750] (normalized) for 1500ms successfully.")
+    assert res.message == "Long-pressed at [250, 750] (normalized) for 1500ms."
     assert res.duration_ms == 1500
 
 
@@ -72,7 +70,7 @@ async def test_long_press_message_and_duration(actuator):
 async def test_swipe_reports_normalized_endpoints(actuator):
     res = await actuator.swipe((500, 800), (500, 200), 400)
     assert res.ok
-    assert res.message == "Swipe completed successfully. Swiped from [500, 800] to [500, 200]."
+    assert res.message == "Swiped from [500, 800] to [500, 200] (normalized)."
     assert res.normalized_coordinates == [500, 800, 500, 200]
 
 
@@ -82,7 +80,7 @@ async def test_press_key_forwards_unknown_keycode_to_driver(actuator):
     # matching the historical adb_server behavior of accepting any Android key event.
     res = await actuator.press_key("KEYCODE_DPAD_DOWN")
     assert res.ok
-    assert res.message == "Executed key press 'KEYCODE_DPAD_DOWN'."
+    assert res.message == "Pressed key 'KEYCODE_DPAD_DOWN'."
     assert actuator.action_history[-1]["action"] == "press_key"
 
 
@@ -98,7 +96,7 @@ async def test_input_text_append_moves_cursor_and_preserves_text(actuator, monke
     monkeypatch.setattr("asyncio.sleep", _instant_sleep)
     res = await actuator.input_text("hello", (500, 300), clear_exist=False)
     assert res.ok
-    assert res.message == "Executed typing 'hello'."
+    assert res.message == "Typed 'hello' at [500, 300] (normalized)."
     actions = [h["action"] for h in actuator.action_history]
     # Append semantics: focus tap, cursor-to-end (KEYCODE_MOVE_END), then type without
     # clearing — matching the historical adb_server focus_and_input_text contract.
@@ -128,7 +126,7 @@ async def _instant_sleep(_delay, *args, **kwargs):
 async def test_wait_for_delay(actuator):
     res = await actuator.wait_for_delay(1)
     assert res.ok
-    assert res.message == "Waited for 1ms successfully."
+    assert res.message == "Waited 1ms."
 
 
 def test_partial_capabilities_are_reported():