feat: add agentic video analysis and clarify action results
Report device actions as dispatched and preserve full failure details. Route Flash through McpActionExecutor and use ToolFailure across helper tools and agent loops instead of inferring failure from response text. Require model-provided descriptions for coordinate targets. Keep observed element metadata separate from those descriptions in safety checks, visual summaries, history, and frontend labels. Add Gemini Interactions API video analysis with configurable processing mode, chunk size, and timeouts. Centralize Google provider capabilities and token usage normalization, and update google-genai to 2.22.0. Preserve system instructions on follow-up interactions, replan pending chunks when falling back to static processing, and reject unanswered native video and audio segments instead of caching placeholder results. Keep helper-tool failure status intact through history tools and committee calls. Validation: 1,762 Python unit tests passed (3 skipped), 40 frontend formatter tests passed, and the production UI build completed. Ruff lint, formatting, the quality ratchet, and git diff checks passed.
This commit is contained in:
somew1nd committed
1 parent
611d1a0947
commit
a45daf23f8
120 files changed
+6301
-1710
No files matched your search
@@ -12,11 +12,19 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Tests for record-time target semantics enrichment in McpActionExecutor."""
|
||||
"""Coordinate targets must be described by the model, never inferred.
|
||||
|
||||
The Flash/Validator dialect addresses every target by coordinates, so the
|
||||
executor refuses a click, long press, focused input, coordinate swipe or click
|
||||
sequence that does not say what it is aiming at. The description is recorded
|
||||
by the runner and never reaches the wire.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
from artemis.mcp.action_executor import McpActionExecutor
|
||||
import pytest
|
||||
|
||||
from artemis.mcp.action_executor import McpActionExecutor, _ArgError
|
||||
|
||||
|
||||
def _make_executor(width=1080, height=2400):
|
||||
@@ -28,131 +36,195 @@ def _make_executor(width=1080, height=2400):
|
||||
return McpActionExecutor(ctx, actuator=actuator)
|
||||
|
||||
|
||||
def _state_with_elements(elements):
|
||||
def _state():
|
||||
state = Mock()
|
||||
state.indexed_elements = elements
|
||||
state.indexed_elements = []
|
||||
state.indexed_points = [[540, 1440], [745, 1440]]
|
||||
state.latest_ui_hierarchy = None
|
||||
return state
|
||||
|
||||
|
||||
def test_click_target_semantics_from_pre_action_frame():
|
||||
@pytest.mark.parametrize(
|
||||
"name, args, label",
|
||||
[
|
||||
("click", {"target": [500, 600]}, "click"),
|
||||
("long_press", {"target": [500, 600]}, "long press"),
|
||||
("input_text", {"text": "hi", "target": [500, 600]}, "input text"),
|
||||
("swipe", {"start": [200, 600], "end": [800, 600]}, "swipe"),
|
||||
],
|
||||
)
|
||||
def test_coordinate_targets_without_description_are_refused(name, args, label):
|
||||
executor = _make_executor()
|
||||
# Normalized (500, 600) on a 1080x2400 device -> pixel (540, 1440).
|
||||
state = _state_with_elements(
|
||||
[
|
||||
{
|
||||
"text": "Confirm",
|
||||
"bounds": [500, 1400, 600, 1480],
|
||||
"class": "android.widget.Button",
|
||||
"resource_id": "btn_confirm",
|
||||
"is_ocr": False,
|
||||
}
|
||||
]
|
||||
with pytest.raises(_ArgError) as excinfo:
|
||||
executor._translate(name, args, _state())
|
||||
message = str(excinfo.value)
|
||||
assert message.startswith(f"Error during {label}")
|
||||
assert "target_description" in message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("blank", ["", " ", None, 7])
|
||||
def test_blank_or_non_string_description_is_refused(blank):
|
||||
executor = _make_executor()
|
||||
with pytest.raises(_ArgError):
|
||||
executor._translate("click", {"target": [500, 600], "target_description": blank}, _state())
|
||||
|
||||
|
||||
def test_description_is_accepted_and_kept_off_the_wire():
|
||||
executor = _make_executor()
|
||||
wire_name, wire_args, _, _ = executor._translate(
|
||||
"click", {"target": [500, 600], "target_description": "play button"}, _state()
|
||||
)
|
||||
semantics = executor._target_semantics("click", {"target": [500, 600]}, state)
|
||||
assert semantics == {
|
||||
"target_label_source": "hit_test",
|
||||
"target_text": "Confirm",
|
||||
"target_class": "android.widget.Button",
|
||||
"target_resource_id": "btn_confirm",
|
||||
}
|
||||
assert wire_name == "click"
|
||||
assert wire_args == {"target": [500, 600], "times": 1, "delay_ms": 100}
|
||||
assert "target_description" not in wire_args
|
||||
|
||||
|
||||
def test_semantics_degrade_to_none_without_element_data():
|
||||
executor = _make_executor()
|
||||
|
||||
# Empty perception data
|
||||
semantics = executor._target_semantics(
|
||||
"click", {"target": [500, 600]}, _state_with_elements([])
|
||||
_, swipe_args, _, _ = executor._translate(
|
||||
"swipe",
|
||||
{"start": [200, 600], "end": [800, 600], "target_description": "brightness knob"},
|
||||
_state(),
|
||||
)
|
||||
assert semantics == {"target_label_source": "none"}
|
||||
|
||||
# No state at all
|
||||
semantics = executor._target_semantics("click", {"target": [500, 600]}, None)
|
||||
assert semantics == {"target_label_source": "none"}
|
||||
assert swipe_args == {"start": [200, 600], "end": [800, 600], "duration_ms": 400}
|
||||
|
||||
|
||||
def test_non_targeted_actions_and_missing_targets_skip_enrichment():
|
||||
def test_recorded_target_is_the_cleaned_description_in_pro_shape():
|
||||
"""The recorded semantics come from the same check that requires them."""
|
||||
executor = _make_executor()
|
||||
state = _state_with_elements([{"text": "X", "bounds": [0, 0, 10, 10]}])
|
||||
|
||||
assert executor._target_semantics("press_key", {"key": "BACK"}, state) is None
|
||||
# input_text without a coordinate target (focused field typing)
|
||||
assert executor._target_semantics("input_text", {"text": "hi", "target": None}, state) is None
|
||||
# Multi-point actions without resolvable points skip enrichment too.
|
||||
assert executor._target_semantics("click_sequence", {"sequence": []}, state) is None
|
||||
assert executor._target_semantics("swipe", {"start": None, "end": None}, state) is None
|
||||
|
||||
|
||||
def test_click_sequence_hit_tests_every_point_first_point_is_main_label():
|
||||
"""M5: multi-point actions get per-point best-effort semantics; the first
|
||||
point's fields are hoisted as the action's main label."""
|
||||
executor = _make_executor()
|
||||
state = _state_with_elements(
|
||||
[
|
||||
{
|
||||
"text": "Digit 1",
|
||||
"bounds": [500, 1400, 600, 1480],
|
||||
"class": "android.widget.Button",
|
||||
"is_ocr": False,
|
||||
},
|
||||
{
|
||||
"text": "Digit 2",
|
||||
"bounds": [700, 1400, 800, 1480],
|
||||
"class": "android.widget.Button",
|
||||
"is_ocr": False,
|
||||
},
|
||||
]
|
||||
_, _, _, recorded = executor._translate(
|
||||
"click", {"target": [1, 2], "target_description": " ok "}, _state()
|
||||
)
|
||||
# Normalized points -> pixels: (500,600)->(540,1440) hits Digit 1;
|
||||
# (690,600)->(745,1440) hits Digit 2; (10,10)->(10,24) hits nothing.
|
||||
semantics = executor._target_semantics(
|
||||
assert recorded == {"target_description": "ok"}
|
||||
|
||||
_, _, _, recorded = executor._translate(
|
||||
"long_press", {"target": [1, 2], "target_description": "thumbnail"}, _state()
|
||||
)
|
||||
assert recorded == {"target_description": "thumbnail"}
|
||||
|
||||
_, _, _, recorded = executor._translate(
|
||||
"click_sequence", {"sequence": [[1, 2]], "target_descriptions": ["a "]}, _state()
|
||||
)
|
||||
assert recorded == {"target_descriptions": ["a"]}
|
||||
|
||||
# No coordinate target: nothing is recorded, nothing inferred.
|
||||
_, _, _, recorded = executor._translate("press_key", {"key": "BACK"}, _state())
|
||||
assert recorded == {}
|
||||
|
||||
|
||||
def test_coordinate_swipe_records_its_description():
|
||||
executor = _make_executor()
|
||||
_, wire_args, _, recorded = executor._translate(
|
||||
"swipe",
|
||||
{"start": [200, 600], "end": [800, 600], "target_description": " brightness knob "},
|
||||
_state(),
|
||||
)
|
||||
assert recorded == {"target_description": "brightness knob"}
|
||||
assert "target_description" not in wire_args
|
||||
|
||||
|
||||
def test_targeted_input_records_its_description():
|
||||
executor = _make_executor()
|
||||
_, wire_args, _, recorded = executor._translate(
|
||||
"input_text",
|
||||
{"text": "hi", "target": [500, 600], "target_description": "search input"},
|
||||
_state(),
|
||||
)
|
||||
assert wire_args["target"] == [500, 600]
|
||||
assert recorded == {"target_description": "search input"}
|
||||
assert "target_description" not in wire_args
|
||||
|
||||
|
||||
def test_focused_typing_without_a_target_needs_no_description():
|
||||
executor = _make_executor()
|
||||
_, wire_args, _, recorded = executor._translate(
|
||||
"input_text", {"text": "hi", "target": None}, _state()
|
||||
)
|
||||
assert wire_args["target"] is None
|
||||
assert recorded == {}
|
||||
|
||||
|
||||
def test_focused_typing_ignores_a_stray_description():
|
||||
"""Typing into the focused field has no coordinate target: a description the
|
||||
model tacked on anyway is not recorded, so no self-described target enters
|
||||
the history for an action that aimed at nothing."""
|
||||
executor = _make_executor()
|
||||
_, wire_args, _, recorded = executor._translate(
|
||||
"input_text",
|
||||
{"text": "hi", "target": None, "target_description": "search input"},
|
||||
_state(),
|
||||
)
|
||||
assert wire_args["target"] is None
|
||||
assert recorded == {}
|
||||
|
||||
|
||||
def test_directional_swipe_needs_no_description():
|
||||
executor = _make_executor()
|
||||
wire_name, wire_args, _, recorded = executor._translate("swipe", {"direction": "up"}, _state())
|
||||
assert wire_name == "swipe"
|
||||
assert "start" in wire_args and "end" in wire_args
|
||||
assert recorded == {}
|
||||
|
||||
|
||||
def test_directional_swipe_ignores_a_stray_description():
|
||||
executor = _make_executor()
|
||||
_, wire_args, _, recorded = executor._translate(
|
||||
"swipe", {"direction": "up", "target_description": "the feed"}, _state()
|
||||
)
|
||||
assert "target_description" not in wire_args
|
||||
assert recorded == {}
|
||||
|
||||
|
||||
def test_click_sequence_requires_one_description_per_entry():
|
||||
executor = _make_executor()
|
||||
sequence = [[500, 600], [690, 600], [10, 10]]
|
||||
|
||||
with pytest.raises(_ArgError) as excinfo:
|
||||
executor._translate("click_sequence", {"sequence": sequence}, _state())
|
||||
assert "target_descriptions" in str(excinfo.value)
|
||||
assert "3 expected" in str(excinfo.value)
|
||||
|
||||
# Wrong length is refused too.
|
||||
with pytest.raises(_ArgError):
|
||||
executor._translate(
|
||||
"click_sequence",
|
||||
{"sequence": sequence, "target_descriptions": ["video body", "skip"]},
|
||||
_state(),
|
||||
)
|
||||
|
||||
_, wire_args, _, recorded = executor._translate(
|
||||
"click_sequence",
|
||||
{"sequence": [[500, 600], [690, 600], [10, 10]], "delay_ms": 50},
|
||||
state,
|
||||
{"sequence": sequence, "target_descriptions": ["video body", "digit 2", "corner"]},
|
||||
_state(),
|
||||
)
|
||||
assert semantics["target_text"] == "Digit 1"
|
||||
assert semantics["target_label_source"] == "hit_test"
|
||||
per_point = semantics["points_semantics"]
|
||||
assert len(per_point) == 3
|
||||
assert per_point[0]["target_text"] == "Digit 1"
|
||||
assert per_point[1]["target_text"] == "Digit 2"
|
||||
assert per_point[2] == {"target_label_source": "none"}
|
||||
assert wire_args["sequence"] == [[500, 600], [690, 600], [10, 10]]
|
||||
assert "target_descriptions" not in wire_args
|
||||
assert recorded == {"target_descriptions": ["video body", "digit 2", "corner"]}
|
||||
|
||||
|
||||
def test_swipe_hit_tests_start_and_end_start_is_main_label():
|
||||
@pytest.mark.parametrize("index_entry", [2, "2", 2.0, [2]])
|
||||
def test_click_sequence_refuses_element_indices(index_entry):
|
||||
"""click_sequence takes coordinate pairs only. An index would be resolved
|
||||
against the element list and then recorded under the model's own
|
||||
description, reading back as a self-described coordinate target."""
|
||||
executor = _make_executor()
|
||||
state = _state_with_elements(
|
||||
[
|
||||
{
|
||||
"text": "Brightness slider",
|
||||
"bounds": [100, 1400, 900, 1480],
|
||||
"class": "android.widget.SeekBar",
|
||||
"is_ocr": False,
|
||||
}
|
||||
]
|
||||
)
|
||||
semantics = executor._target_semantics(
|
||||
"swipe", {"start": [200, 600], "end": [800, 600], "duration_ms": 400}, state
|
||||
)
|
||||
assert semantics["target_text"] == "Brightness slider"
|
||||
assert semantics["start_semantics"]["target_text"] == "Brightness slider"
|
||||
assert semantics["end_semantics"]["target_text"] == "Brightness slider"
|
||||
assert semantics["target_label_source"] == "hit_test"
|
||||
sequence = [[500, 600], index_entry, [10, 10]]
|
||||
|
||||
with pytest.raises(_ArgError) as excinfo:
|
||||
executor._translate(
|
||||
"click_sequence",
|
||||
{"sequence": sequence, "target_descriptions": ["video body", "digit 2", "corner"]},
|
||||
_state(),
|
||||
)
|
||||
message = str(excinfo.value)
|
||||
assert message.startswith("Error during click sequence")
|
||||
assert "entry 2" in message
|
||||
assert "coordinate" in message
|
||||
assert "ask_explorer" in message
|
||||
|
||||
|
||||
def test_long_press_uses_ocr_fallback():
|
||||
def test_click_sequence_accepts_a_serialized_pair_list():
|
||||
executor = _make_executor()
|
||||
state = _state_with_elements(
|
||||
[
|
||||
{
|
||||
"text": "OCR Word",
|
||||
"bounds": [520, 1420, 560, 1460],
|
||||
"class": None,
|
||||
"resource_id": None,
|
||||
"is_ocr": True,
|
||||
}
|
||||
]
|
||||
_, wire_args, _, _ = executor._translate(
|
||||
"click_sequence",
|
||||
{"sequence": "[[500, 600], [10, 10]]", "target_descriptions": ["video body", "corner"]},
|
||||
_state(),
|
||||
)
|
||||
semantics = executor._target_semantics("long_press", {"target": [500, 600]}, state)
|
||||
assert semantics["target_label_source"] == "ocr"
|
||||
assert semantics["target_text"] == "OCR Word"
|
||||
assert wire_args["sequence"] == [[500, 600], [10, 10]]
|
||||
Reference in new issue
Block a user