101 lines
3.7 KiB
Python
101 lines
3.7 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Tests for pixel safety-net target descriptions."""
|
|
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock
|
|
|
|
from artemis.agents.validator import precondition_pixel as pp
|
|
from langchain_core.messages import HumanMessage, SystemMessage
|
|
|
|
_PNG = b"fake-image-bytes"
|
|
|
|
|
|
def _texts(messages: list) -> list[str]:
|
|
human = messages[1]
|
|
return [part["text"] for part in human.content if part["type"] == "text"]
|
|
|
|
|
|
def test_named_control_target_block_precedes_images():
|
|
item = {
|
|
"action": "tap",
|
|
"coordinates": [1001, 718],
|
|
"target_text": "进入全屏模式",
|
|
"target_resource_id": "com.google.android.youtube:id/fullscreen_button",
|
|
"target_class": "android.widget.ImageView",
|
|
"target_bounds": [922, 655, 1080, 781],
|
|
}
|
|
state = MagicMock()
|
|
state.operator_native_thinking = "identified 进入全屏模式"
|
|
state.operator_raw_thinking = None
|
|
|
|
messages = pp._build_messages("SYSTEM", _PNG, _PNG, item, state)
|
|
|
|
assert isinstance(messages[0], SystemMessage)
|
|
assert isinstance(messages[1], HumanMessage)
|
|
first_text = messages[1].content[0]["text"]
|
|
assert first_text.startswith("[Target]")
|
|
assert "Kind: specific UI control" in first_text
|
|
assert "Label: 进入全屏模式" in first_text
|
|
assert "Resource ID: com.google.android.youtube:id/fullscreen_button" in first_text
|
|
assert "Class: android.widget.ImageView" in first_text
|
|
assert "[922, 655, 1080, 781]" in first_text
|
|
assert messages[1].content[1]["text"] == "[Image 1 (Reference)]"
|
|
assert any("Original Thinking" in t for t in _texts(messages))
|
|
assert any("Action: tap" in t for t in _texts(messages))
|
|
|
|
|
|
def test_coordinates_only_target_is_labelled_as_surface():
|
|
item = {"action": "tap", "coordinates": [540, 400]}
|
|
|
|
messages = pp._build_messages("SYSTEM", _PNG, _PNG, item, state=None)
|
|
|
|
first_text = messages[1].content[0]["text"]
|
|
assert "Kind: coordinates only" in first_text
|
|
assert "Label:" not in first_text
|
|
assert "Resource ID:" not in first_text
|
|
assert not any("Original Thinking" in t for t in _texts(messages))
|
|
|
|
|
|
def test_label_alone_counts_as_named_control():
|
|
item = {"action": "tap", "coordinates": [10, 10], "target_text": "Submit"}
|
|
text = pp._describe_target(item)
|
|
assert "Kind: specific UI control" in text
|
|
assert "Label: Submit" in text
|
|
assert "Resource ID" not in text
|
|
|
|
|
|
def test_unlabelled_hit_test_element_is_still_a_control():
|
|
item = {
|
|
"action": "tap",
|
|
"coordinates": [1001, 718],
|
|
"target_class": "android.widget.ImageView",
|
|
"target_bounds": [922, 655, 1080, 781],
|
|
"target_label_source": "hit_test",
|
|
}
|
|
text = pp._describe_target(item)
|
|
assert "Kind: specific UI control" in text
|
|
assert "unlabelled" in text
|
|
assert "Class: android.widget.ImageView" in text
|
|
assert "[922, 655, 1080, 781]" in text
|
|
|
|
|
|
def test_prompt_rules_are_keyed_on_target_kind():
|
|
prompt = Path(pp.__file__).with_name("pixel_safety_net.md").read_text(encoding="utf-8")
|
|
assert "`Kind: specific UI control`" in prompt
|
|
assert "`Kind: coordinates only`" in prompt
|
|
assert "hidden" in prompt
|
|
assert "Identify what appears under the red dot" in prompt
|