test: add comprehensive unit, integration, and e2e test suites
Introduce complete test coverage for the ARTEMIS platform across multiple layers: Unit tests: Add tests for agent nodes (Planner, Operator, Validator, Diagnoser, FlashRunner, Explorer), tools, utils (OCR/XML fusion, UI hierarchy), and data engine. Integration tests: Add live tool verification, reactive runner, IME, and standalone workflow tests. E2E tests: Add end-to-end scenario tests for media playback and failure recovery flows. Fixtures & mocks: Add mock inputs (audio, video, screenshot) and test utilities.
This commit is contained in:
1 parent
c1803fcb86
commit
5d2283deaa
115 files changed
+22813
No files matched your search
@@ -0,0 +1,148 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.utils.app_launch_utils import launch_app_with_retries
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.device = Mock()
|
||||
ctx.device.mobile_platform = "android"
|
||||
ctx.device.device_id = "emulator-5554"
|
||||
ctx.data_engine = None
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_success_immediate(mock_get_foreground, mock_controller_cls, mock_context):
|
||||
"""Test successful immediate app launch."""
|
||||
mock_controller = Mock()
|
||||
mock_controller.launch_app = AsyncMock(return_value=True)
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
|
||||
# App immediately in foreground
|
||||
mock_get_foreground.return_value = "com.google.android.youtube"
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context,
|
||||
"com.google.android.youtube",
|
||||
max_retries=1,
|
||||
max_poll_seconds=2,
|
||||
)
|
||||
|
||||
assert success is True
|
||||
assert error_msg is None
|
||||
mock_controller.launch_app.assert_called_once_with("com.google.android.youtube")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.get_focused_task_package")
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_success_permission_overlay(
|
||||
mock_get_foreground,
|
||||
mock_controller_cls,
|
||||
mock_get_focused_task,
|
||||
mock_context,
|
||||
):
|
||||
"""Test successful launch if overlayed by system permission controller and task is matching."""
|
||||
mock_controller = Mock()
|
||||
mock_controller.launch_app = AsyncMock(return_value=True)
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
|
||||
# System permission controller is focused foreground
|
||||
mock_get_foreground.return_value = "com.google.android.permissioncontroller"
|
||||
# Second layer check: Task stack shows com.google.android.youtube is focused
|
||||
mock_get_focused_task.return_value = "com.google.android.youtube"
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context,
|
||||
"com.google.android.youtube",
|
||||
max_retries=1,
|
||||
max_poll_seconds=2,
|
||||
)
|
||||
|
||||
assert success is True
|
||||
assert error_msg is None
|
||||
mock_controller.launch_app.assert_called_once_with("com.google.android.youtube")
|
||||
mock_get_focused_task.assert_called_once_with(mock_context)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.get_focused_task_package")
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_failure_permission_overlay_wrong_task(
|
||||
mock_get_foreground,
|
||||
mock_controller_cls,
|
||||
mock_get_focused_task,
|
||||
mock_context,
|
||||
):
|
||||
"""Test launch failure if overlayed by system permission controller but task does not match."""
|
||||
mock_controller = Mock()
|
||||
mock_controller.launch_app = AsyncMock(return_value=True)
|
||||
mock_controller.terminate_app = AsyncMock()
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
|
||||
# System permission controller is focused foreground
|
||||
mock_get_foreground.return_value = "com.google.android.permissioncontroller"
|
||||
# Second layer check: Task stack is some other app (e.g. com.android.settings)
|
||||
mock_get_focused_task.return_value = "com.android.settings"
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context,
|
||||
"com.google.android.youtube",
|
||||
max_retries=1,
|
||||
max_poll_seconds=1,
|
||||
)
|
||||
|
||||
assert success is False
|
||||
assert "Failed to launch com.google.android.youtube" in error_msg
|
||||
mock_get_focused_task.assert_called_once_with(mock_context)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.utils.app_launch_utils.UnifiedMobileController")
|
||||
@patch("artemis.utils.app_launch_utils.get_current_foreground_package_async")
|
||||
async def test_launch_app_failure_wrong_app(mock_get_foreground, mock_controller_cls, mock_context):
|
||||
"""Test launch failure and retry when wrong app remains in foreground."""
|
||||
mock_controller = Mock()
|
||||
mock_controller.launch_app = AsyncMock(return_value=True)
|
||||
mock_controller.terminate_app = AsyncMock()
|
||||
mock_controller_cls.return_value = mock_controller
|
||||
|
||||
# A different app (e.g., settings or home screen) is in foreground
|
||||
mock_get_foreground.return_value = "com.android.settings"
|
||||
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
mock_context,
|
||||
"com.google.android.youtube",
|
||||
max_retries=2,
|
||||
max_poll_seconds=1,
|
||||
)
|
||||
|
||||
assert success is False
|
||||
assert "Failed to launch com.google.android.youtube" in error_msg
|
||||
|
||||
# Should have attempted launching twice
|
||||
assert mock_controller.launch_app.call_count == 2
|
||||
# Should have attempted force-stopping once before retry
|
||||
assert mock_controller.terminate_app.call_count == 1
|
||||
@@ -0,0 +1,178 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import cv2
|
||||
from artemis.utils.image_diff import check_ui_change
|
||||
import numpy as np
|
||||
|
||||
|
||||
def test_no_change():
|
||||
img = np.zeros((1000, 1000), dtype=np.uint8)
|
||||
_, img_bytes = cv2.imencode(".jpg", img)
|
||||
|
||||
action = {"action": "tap", "coordinates": [500, 500]}
|
||||
assert not check_ui_change(img_bytes.tobytes(), img_bytes.tobytes(), action)
|
||||
print("test_no_change passed")
|
||||
|
||||
|
||||
def test_global_change():
|
||||
img1 = np.zeros((1000, 1000), dtype=np.uint8)
|
||||
img2 = np.ones((1000, 1000), dtype=np.uint8) * 255
|
||||
|
||||
_, img1_bytes = cv2.imencode(".jpg", img1)
|
||||
_, img2_bytes = cv2.imencode(".jpg", img2)
|
||||
|
||||
action = {"action": "swipe", "coordinates": [500, 500, 500, 200]}
|
||||
assert check_ui_change(img1_bytes.tobytes(), img2_bytes.tobytes(), action)
|
||||
print("test_global_change passed")
|
||||
|
||||
|
||||
def test_roi_change():
|
||||
img1 = np.zeros((1000, 1000), dtype=np.uint8)
|
||||
img2 = img1.copy()
|
||||
img2[480:520, 480:520] = 255
|
||||
|
||||
_, img1_bytes = cv2.imencode(".jpg", img1)
|
||||
_, img2_bytes = cv2.imencode(".jpg", img2)
|
||||
|
||||
action = {"action": "tap", "coordinates": [500, 500]}
|
||||
assert check_ui_change(img1_bytes.tobytes(), img2_bytes.tobytes(), action)
|
||||
print("test_roi_change passed")
|
||||
|
||||
|
||||
def test_status_bar_ignore():
|
||||
img1 = np.zeros((1000, 1000), dtype=np.uint8)
|
||||
img2 = img1.copy()
|
||||
img2[0:50, 0:1000] = 255
|
||||
|
||||
_, img1_bytes = cv2.imencode(".jpg", img1)
|
||||
_, img2_bytes = cv2.imencode(".jpg", img2)
|
||||
|
||||
action = {"action": "tap", "coordinates": [500, 500]}
|
||||
assert not check_ui_change(img1_bytes.tobytes(), img2_bytes.tobytes(), action)
|
||||
print("test_status_bar_ignore passed")
|
||||
|
||||
|
||||
def test_roi_threshold():
|
||||
# Use standard 1080x2400 resolution to keep scale_factor = 1.0 (roi_size = 50)
|
||||
img1 = np.zeros((2400, 1080), dtype=np.uint8)
|
||||
img2 = img1.copy()
|
||||
# Center of 1080x2400 is (540, 1200). Draw 10x10 square at center.
|
||||
img2[1195:1205, 535:545] = 255
|
||||
|
||||
_, img1_bytes = cv2.imencode(".jpg", img1)
|
||||
_, img2_bytes = cv2.imencode(".jpg", img2)
|
||||
|
||||
action = {"action": "tap", "coordinates": [540, 1200]}
|
||||
assert not check_ui_change(
|
||||
img1_bytes.tobytes(),
|
||||
img2_bytes.tobytes(),
|
||||
action,
|
||||
full_screen_threshold=0.001,
|
||||
roi_threshold=0.05,
|
||||
)
|
||||
print("test_roi_threshold passed")
|
||||
|
||||
|
||||
def test_wait_for_screen_stability():
|
||||
from artemis.utils.image_diff import wait_for_screen_stability
|
||||
|
||||
img_static = np.zeros((1000, 1000), dtype=np.uint8)
|
||||
_, static_bytes = cv2.imencode(".jpg", img_static)
|
||||
static_b64 = base64.b64encode(static_bytes).decode("utf-8")
|
||||
|
||||
# 1. Test immediate stability (already static)
|
||||
calls_immediate = 0
|
||||
|
||||
async def mock_screenshot_immediate():
|
||||
nonlocal calls_immediate
|
||||
calls_immediate += 1
|
||||
return static_b64
|
||||
|
||||
res_immediate = asyncio.run(
|
||||
wait_for_screen_stability(mock_screenshot_immediate, max_timeout=1.0, interval=0.05)
|
||||
)
|
||||
assert res_immediate == static_b64
|
||||
assert calls_immediate == 2
|
||||
print("test_wait_for_screen_stability: immediate stability subtest passed")
|
||||
|
||||
# 2. Test gradual stability (settles after 3 moving frames)
|
||||
calls_gradual = 0
|
||||
|
||||
async def mock_screenshot_gradual():
|
||||
nonlocal calls_gradual
|
||||
calls_gradual += 1
|
||||
if calls_gradual <= 3:
|
||||
img = np.zeros((1000, 1000), dtype=np.uint8)
|
||||
img[500:, :] = calls_gradual * 50
|
||||
_, bytes_data = cv2.imencode(".jpg", img)
|
||||
return base64.b64encode(bytes_data).decode("utf-8")
|
||||
return static_b64
|
||||
|
||||
res_gradual = asyncio.run(
|
||||
wait_for_screen_stability(mock_screenshot_gradual, max_timeout=1.0, interval=0.05)
|
||||
)
|
||||
assert res_gradual == static_b64
|
||||
assert calls_gradual > 3
|
||||
print("test_wait_for_screen_stability: gradual stability subtest passed")
|
||||
print("test_wait_for_screen_stability passed")
|
||||
|
||||
|
||||
def test_wait_for_screen_data_stability():
|
||||
from artemis.utils.image_diff import wait_for_screen_data_stability
|
||||
from artemis.controllers.device_controller import ScreenDataResponse
|
||||
|
||||
img_static = np.zeros((1000, 1000), dtype=np.uint8)
|
||||
_, static_bytes = cv2.imencode(".jpg", img_static)
|
||||
static_b64 = base64.b64encode(static_bytes).decode("utf-8")
|
||||
|
||||
# Test stability returning ScreenDataResponse
|
||||
calls = 0
|
||||
|
||||
async def mock_get_screen_data():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return ScreenDataResponse(
|
||||
base64=static_b64,
|
||||
elements=[],
|
||||
width=1000,
|
||||
height=1000,
|
||||
platform="android",
|
||||
)
|
||||
|
||||
res = asyncio.run(
|
||||
wait_for_screen_data_stability(mock_get_screen_data, max_timeout=1.0, interval=0.05)
|
||||
)
|
||||
assert res.base64 == static_b64
|
||||
assert res.width == 1000
|
||||
assert calls == 2
|
||||
print("test_wait_for_screen_data_stability passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
test_no_change()
|
||||
test_global_change()
|
||||
test_roi_change()
|
||||
test_status_bar_ignore()
|
||||
test_roi_threshold()
|
||||
test_wait_for_screen_stability()
|
||||
test_wait_for_screen_data_stability()
|
||||
print("All tests passed successfully!")
|
||||
except AssertionError as e:
|
||||
print(f"Test failed: {e}")
|
||||
except Exception as e:
|
||||
print(f"Error running tests: {e}")
|
||||
@@ -0,0 +1,165 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from artemis.utils.notes import (
|
||||
append_note_content,
|
||||
list_notes_info,
|
||||
read_note_content,
|
||||
save_note_content,
|
||||
update_note_content,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
def test_list_notes_info(tmp_path):
|
||||
base_dir = tmp_path
|
||||
|
||||
# Create some notes
|
||||
save_note_content(base_dir, "note1", "line1\nline2\nline3")
|
||||
save_note_content(base_dir, "note2", "line1\nline2")
|
||||
|
||||
info = list_notes_info(base_dir)
|
||||
assert info == {"note1": 3, "note2": 2}
|
||||
|
||||
|
||||
def test_read_note_content_range(tmp_path):
|
||||
base_dir = tmp_path
|
||||
save_note_content(base_dir, "note1", "line1\nline2\nline3\nline4")
|
||||
|
||||
# Read full
|
||||
assert read_note_content(base_dir, "note1") == "line1\nline2\nline3\nline4"
|
||||
|
||||
# Read range
|
||||
assert read_note_content(base_dir, "note1", start_line=2, end_line=3) == "line2\nline3"
|
||||
|
||||
# Read from start to line
|
||||
assert read_note_content(base_dir, "note1", end_line=2) == "line1\nline2"
|
||||
|
||||
# Read from line to end
|
||||
assert read_note_content(base_dir, "note1", start_line=3) == "line3\nline4"
|
||||
|
||||
|
||||
def test_update_note_exact_match(tmp_path):
|
||||
base_dir = tmp_path
|
||||
original_content = 'def demo():\n print("Hello, World!")\n return True\n'
|
||||
save_note_content(base_dir, "task_plan", original_content)
|
||||
|
||||
# Exact match
|
||||
warning = update_note_content(
|
||||
base_dir,
|
||||
"task_plan",
|
||||
' print("Hello, World!")',
|
||||
' print("Hello, Jetski!")',
|
||||
)
|
||||
assert warning is None
|
||||
|
||||
updated_content = read_note_content(base_dir, "task_plan")
|
||||
assert 'print("Hello, Jetski!")' in updated_content
|
||||
assert 'print("Hello, World!")' not in updated_content
|
||||
|
||||
|
||||
def test_update_note_relaxed_match(tmp_path):
|
||||
base_dir = tmp_path
|
||||
# Multi-line content with checkboxes and indentation
|
||||
original_content = (
|
||||
"- [ ] Step 1: Research the issue\n - [ ] Step 2: Write code\n - [ ] Step 3: Run tests\n"
|
||||
)
|
||||
save_note_content(base_dir, "task_plan", original_content)
|
||||
|
||||
# Relaxed match: case difference, quote difference, checkbox bullet difference, trailing whitespace
|
||||
target = " * [ ] step 2: write code "
|
||||
replacement = " - [/] Step 2: In progress"
|
||||
|
||||
warning = update_note_content(base_dir, "task_plan", target, replacement)
|
||||
|
||||
# Warning should be returned
|
||||
assert warning is not None
|
||||
assert "We did our best to apply changes despite some inaccuracies" in warning
|
||||
|
||||
updated_content = read_note_content(base_dir, "task_plan")
|
||||
assert "Step 2: In progress" in updated_content
|
||||
assert "Step 2: Write code" not in updated_content
|
||||
# Check indentation is preserved from replacement
|
||||
assert " - [/] Step 2: In progress\n" in updated_content
|
||||
|
||||
|
||||
def test_update_note_fuzzy_match(tmp_path):
|
||||
base_dir = tmp_path
|
||||
original_content = 'print("Alpha message")\nprint("Gamma message")\nprint("Delta message")\n'
|
||||
save_note_content(base_dir, "task_plan", original_content)
|
||||
|
||||
# Fuzzy match: typo (Gemma vs Gamma)
|
||||
target = 'print("Gemma message")'
|
||||
replacement = 'print("Gamma updated!")'
|
||||
|
||||
warning = update_note_content(base_dir, "task_plan", target, replacement)
|
||||
assert warning is not None
|
||||
assert "We did our best to apply changes despite some inaccuracies" in warning
|
||||
|
||||
updated_content = read_note_content(base_dir, "task_plan")
|
||||
assert 'print("Gamma updated!")' in updated_content
|
||||
assert 'print("Gamma message")' not in updated_content
|
||||
|
||||
|
||||
def test_update_note_ambiguous_match(tmp_path):
|
||||
base_dir = tmp_path
|
||||
# Multiple lines that could match target
|
||||
original_content = 'print("Beta message")\nprint("Zeta message")\nprint("Eta message")\n'
|
||||
save_note_content(base_dir, "task_plan", original_content)
|
||||
|
||||
# Target is highly ambiguous (Xeta is almost equally close to Beta and Zeta)
|
||||
target = 'print("Xeta message")'
|
||||
replacement = 'print("Ambiguous match!")'
|
||||
|
||||
# Should raise ValueError due to ambiguity or inability to find unique match
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
update_note_content(base_dir, "task_plan", target, replacement)
|
||||
|
||||
assert "Target string 'print(\"Xeta message\")' not found" in str(
|
||||
exc_info.value
|
||||
) or "not unique" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_update_note_not_found(tmp_path):
|
||||
base_dir = tmp_path
|
||||
original_content = 'print("Alpha message")\n'
|
||||
save_note_content(base_dir, "task_plan", original_content)
|
||||
|
||||
# Totally different target
|
||||
target = 'print("Something totally different")'
|
||||
replacement = 'print("Hello")'
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
update_note_content(base_dir, "task_plan", target, replacement)
|
||||
assert "not found" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_append_note_content(tmp_path):
|
||||
base_dir = tmp_path
|
||||
|
||||
# Case 1: File does not exist
|
||||
append_note_content(base_dir, "new_note", "hello")
|
||||
assert read_note_content(base_dir, "new_note") == "hello\n"
|
||||
|
||||
# Case 2: File exists and ends with a newline
|
||||
append_note_content(base_dir, "new_note", "world")
|
||||
assert read_note_content(base_dir, "new_note") == "hello\nworld\n"
|
||||
|
||||
# Case 3: File exists and does not end with a newline
|
||||
file_path = base_dir / "notes" / "no_newline.md"
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_path.write_text("no newline", encoding="utf-8")
|
||||
|
||||
append_note_content(base_dir, "no_newline", "added text")
|
||||
assert read_note_content(base_dir, "no_newline") == "no newline\nadded text\n"
|
||||
@@ -0,0 +1,248 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import base64
|
||||
from io import BytesIO
|
||||
import unittest
|
||||
from artemis.utils.ocr_xml_fusion import (
|
||||
_calculate_overlap_ratio,
|
||||
_crop_image_remove_status_bar,
|
||||
_detect_status_bar_height,
|
||||
_is_low_value_text,
|
||||
_map_coordinates_back,
|
||||
_parse_ocr_position,
|
||||
fuse_ocr_with_xml,
|
||||
)
|
||||
from PIL import Image
|
||||
|
||||
|
||||
class TestOCRFusion(unittest.TestCase):
|
||||
def test_is_low_value_text(self):
|
||||
# Test length 1
|
||||
self.assertTrue(_is_low_value_text("a"))
|
||||
self.assertFalse(_is_low_value_text("Yes"))
|
||||
self.assertFalse(_is_low_value_text("1"))
|
||||
|
||||
# Test placeholders
|
||||
self.assertTrue(_is_low_value_text("search"))
|
||||
|
||||
# Test default markers
|
||||
self.assertTrue(_is_low_value_text("[Icon]"))
|
||||
|
||||
# Test valid text
|
||||
self.assertFalse(_is_low_value_text("Click Me"))
|
||||
|
||||
def test_parse_ocr_position(self):
|
||||
position = [
|
||||
{"x": 10, "y": 20},
|
||||
{"x": 50, "y": 20},
|
||||
{"x": 50, "y": 60},
|
||||
{"x": 10, "y": 60},
|
||||
]
|
||||
bounds = _parse_ocr_position(position)
|
||||
self.assertEqual(bounds, {"left": 10, "top": 20, "right": 50, "bottom": 60})
|
||||
|
||||
def test_calculate_overlap_ratio(self):
|
||||
ocr_bounds = {"left": 10, "top": 10, "right": 30, "bottom": 30}
|
||||
xml_bounds = {"left": 0, "top": 0, "right": 40, "bottom": 40}
|
||||
# OCR is fully inside XML
|
||||
self.assertEqual(_calculate_overlap_ratio(ocr_bounds, xml_bounds), 1.0)
|
||||
|
||||
xml_bounds = {"left": 20, "top": 20, "right": 40, "bottom": 40}
|
||||
# Intersection is 10x10 = 100. OCR area is 20x20 = 400. Ratio = 0.25
|
||||
self.assertEqual(_calculate_overlap_ratio(ocr_bounds, xml_bounds), 0.25)
|
||||
|
||||
def test_detect_status_bar_height(self):
|
||||
xml = [
|
||||
{"package": "com.android.systemui", "bounds": "[0,0][1080,120]"},
|
||||
{"package": "com.example.app", "bounds": "[0,120][1080,2400]"},
|
||||
]
|
||||
# Pass screen_height=2400, should find 120 from XML
|
||||
self.assertEqual(_detect_status_bar_height(xml, screen_height=2400), 120)
|
||||
|
||||
# Fallback: should be 4% of 2400 = 96
|
||||
xml = [{"package": "com.example.app", "bounds": "[0,0][1080,2400]"}]
|
||||
self.assertEqual(_detect_status_bar_height(xml, screen_height=2400), 96)
|
||||
|
||||
def test_crop_image_remove_status_bar(self):
|
||||
# Create a dummy image
|
||||
img = Image.new("RGB", (100, 100), color="red")
|
||||
buffered = BytesIO()
|
||||
img.save(buffered, format="PNG")
|
||||
img_b64 = base64.b64encode(buffered.getvalue()).decode("utf-8")
|
||||
|
||||
cropped_b64, w, h = _crop_image_remove_status_bar(img_b64, crop_height=20)
|
||||
self.assertEqual(w, 100)
|
||||
self.assertEqual(h, 100)
|
||||
|
||||
# Verify cropped size
|
||||
cropped_img = Image.open(BytesIO(base64.b64decode(cropped_b64)))
|
||||
self.assertEqual(cropped_img.size, (100, 80))
|
||||
|
||||
def test_map_coordinates_back(self):
|
||||
ocr_results = [
|
||||
{
|
||||
"text": "test",
|
||||
"position": [{"x": 10, "y": 10}, {"x": 20, "y": 20}],
|
||||
}
|
||||
]
|
||||
mapped = _map_coordinates_back(ocr_results, status_bar_height=100)
|
||||
self.assertEqual(mapped[0]["position"][0]["y"], 110)
|
||||
self.assertEqual(mapped[0]["position"][1]["y"], 120)
|
||||
|
||||
def test_fuse_ocr_with_xml(self):
|
||||
xml = [
|
||||
{
|
||||
"class": "android.widget.Button",
|
||||
"bounds": "[10,100][200,150]",
|
||||
"text": "Submit",
|
||||
},
|
||||
{
|
||||
"class": "android.widget.TextView",
|
||||
"bounds": "[10,200][200,250]",
|
||||
"text": "",
|
||||
},
|
||||
]
|
||||
ocr = [
|
||||
# Should be discarded (similar to Submit)
|
||||
{
|
||||
"text": "Submit",
|
||||
"position": [{"x": 15, "y": 105}, {"x": 95, "y": 145}],
|
||||
},
|
||||
# Multiple OCRs matching the same node (TextView at [10,200])
|
||||
# They are on the same line, should be merged in order
|
||||
{
|
||||
"text": "Result",
|
||||
"position": [{"x": 15, "y": 205}, {"x": 95, "y": 245}],
|
||||
},
|
||||
{
|
||||
"text": "Found",
|
||||
"position": [{"x": 105, "y": 205}, {"x": 185, "y": 245}],
|
||||
},
|
||||
# Unmatched OCRs that are close to each other (should be merged into one virtual node)
|
||||
{
|
||||
"text": "Floating",
|
||||
"position": [{"x": 15, "y": 305}, {"x": 95, "y": 345}],
|
||||
},
|
||||
{
|
||||
"text": "Text",
|
||||
"position": [{"x": 105, "y": 305}, {"x": 185, "y": 345}],
|
||||
},
|
||||
# Unmatched OCR far away (should be separate virtual node)
|
||||
{
|
||||
"text": "Far",
|
||||
"position": [{"x": 15, "y": 405}, {"x": 95, "y": 445}],
|
||||
},
|
||||
]
|
||||
|
||||
fused = fuse_ocr_with_xml(xml, ocr)
|
||||
|
||||
# Check first node (Submit) - should NOT have ocr_text
|
||||
self.assertNotIn("ocr_text", fused[0])
|
||||
|
||||
# Check second node (Result Found) - should have aggregated ocr_elements
|
||||
self.assertEqual(fused[1]["ocr_elements"][0]["text"], "Result Found")
|
||||
self.assertIn("[OCR]", fused[1]["class"])
|
||||
|
||||
# We expect 0 virtual nodes now as per new strategy.
|
||||
# Total nodes should be just the original 2 XML nodes.
|
||||
self.assertEqual(len(fused), 2)
|
||||
|
||||
def test_banded_overlap_matching(self):
|
||||
# Screen is 1080x2400
|
||||
# XML has a giant container and a small correct button.
|
||||
xml = [
|
||||
# Giant container (FrameLayout, covers a lot of screen)
|
||||
{
|
||||
"class": "android.widget.FrameLayout",
|
||||
"bounds": "[0,80][1080,1000]",
|
||||
"text": "",
|
||||
},
|
||||
# Small correct button
|
||||
{
|
||||
"class": "android.widget.Button",
|
||||
"bounds": "[185,84][512,114]",
|
||||
"text": "",
|
||||
},
|
||||
]
|
||||
|
||||
# OCR Search is slightly shifted to the left of the Button.
|
||||
# Button: [185,84][512,114] -> area 327 * 30 = 9810
|
||||
# OCR: [174,89][326,109] -> area 152 * 20 = 3040
|
||||
# Overlap with Button:
|
||||
# Intersection: [185,89][326,109] -> area 141 * 20 = 2820
|
||||
# Ratio: 2820 / 3040 = 0.927 (Band 1)
|
||||
# Overlap with FrameLayout:
|
||||
# OCR is fully inside FrameLayout [0,80][1080,1000]
|
||||
# Ratio: 1.0 (Band 1)
|
||||
# Both are in Band 1 (>=0.9).
|
||||
# The Button has MUCH smaller area than FrameLayout (9810 < 993600).
|
||||
# So Button should win despite having lower overlap (0.927 < 1.0).
|
||||
ocr = [
|
||||
{
|
||||
"text": "Search",
|
||||
"position": [
|
||||
{"x": 174, "y": 89},
|
||||
{"x": 326, "y": 89},
|
||||
{"x": 326, "y": 109},
|
||||
{"x": 174, "y": 109},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
fused = fuse_ocr_with_xml(xml, ocr)
|
||||
|
||||
# Check that OCR text "Search" is mounted on the Button node (index 1)
|
||||
self.assertIn("ocr_elements", fused[1])
|
||||
self.assertEqual(fused[1]["ocr_elements"][0]["text"], "Search")
|
||||
|
||||
# And NOT on the FrameLayout node (index 0)
|
||||
self.assertNotIn("ocr_elements", fused[0])
|
||||
|
||||
def test_giant_node_exclusion(self):
|
||||
# Screen is 1080x2400. Area = 2,592,000.
|
||||
# 10% area = 259,200.
|
||||
# FrameLayout is 1080x1000 -> area 1,080,000 (>10%).
|
||||
# It should be excluded from matching entirely.
|
||||
# Since there is no other node, the OCR should be discarded.
|
||||
xml = [
|
||||
{
|
||||
"class": "android.widget.FrameLayout",
|
||||
"bounds": "[0,0][1080,1000]",
|
||||
"text": "",
|
||||
},
|
||||
]
|
||||
ocr = [
|
||||
# Some floating developer text "dX : 0.0"
|
||||
{
|
||||
"text": "dX : 0.0",
|
||||
"position": [
|
||||
{"x": 144, "y": 55},
|
||||
{"x": 215, "y": 55},
|
||||
{"x": 215, "y": 63},
|
||||
{"x": 144, "y": 63},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
fused = fuse_ocr_with_xml(xml, ocr)
|
||||
|
||||
# The giant FrameLayout should NOT have ocr_elements
|
||||
self.assertNotIn("ocr_elements", fused[0])
|
||||
# The class should NOT be updated with [OCR]
|
||||
self.assertNotIn("[OCR]", fused[0].get("class", ""))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,650 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import hashlib
|
||||
from artemis.utils.task_tree import (
|
||||
build_plan_and_history,
|
||||
get_active_subgoal_hashes,
|
||||
get_recent_subgoal_hashes,
|
||||
)
|
||||
|
||||
|
||||
def test_build_plan_and_history_separated_layout():
|
||||
plan = """- [ ] Open Settings app
|
||||
- [ ] Navigate to System settings"""
|
||||
|
||||
# No steps executed
|
||||
output = build_plan_and_history(plan, [], "default")
|
||||
assert plan in output
|
||||
assert "--- Execution History ---" in output
|
||||
assert "No steps executed yet." in output
|
||||
|
||||
# With steps
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Launched settings app",
|
||||
"operator_raw_thinking": "Thought 1",
|
||||
"action_taken": [{"action": "launch_app"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
assert plan in output
|
||||
assert "--- Execution History ---" in output
|
||||
assert "- **Step 1 (Most Recent Step, Start: 2.5s)**" in output
|
||||
assert " - Thought 1" in output
|
||||
assert " * [Planned Action]: Launched app 'None'" in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_granularities():
|
||||
plan = """- [x] Open Settings app
|
||||
- [/] Navigate to System settings
|
||||
- [ ] Navigate to Languages & input"""
|
||||
|
||||
subgoal_hash_1 = hashlib.md5(b"Open Settings app").hexdigest()
|
||||
subgoal_hash_2 = hashlib.md5(b"Navigate to System settings").hexdigest()
|
||||
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Launched settings",
|
||||
"operator_raw_thinking": "Thought 1",
|
||||
"action_taken": [{"action": "launch_app"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_1},
|
||||
},
|
||||
{
|
||||
"step_id": "step_2",
|
||||
"step_number": 2,
|
||||
"relative_time": "8.0s",
|
||||
"summary": "Swiped down",
|
||||
"operator_raw_thinking": "Thought 2",
|
||||
"action_taken": [{"action": "swipe"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_2},
|
||||
},
|
||||
{
|
||||
"step_id": "step_3",
|
||||
"step_number": 3,
|
||||
"relative_time": "15.0s",
|
||||
"summary": "Clicked System settings",
|
||||
"operator_raw_thinking": "Thought 3",
|
||||
"action_taken": [{"action": "click"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_2},
|
||||
},
|
||||
]
|
||||
|
||||
# Test last_n_detailed = 0 (high-level analysis context, all summarized)
|
||||
output_0 = build_plan_and_history(plan, steps, subgoal_hash_2, last_n_detailed=0)
|
||||
assert "- *Step 1 (Start: 2.5s): Launched settings*" in output_0
|
||||
assert "- *Step 2 (Start: 8.0s): Swiped down*" in output_0
|
||||
assert "- *Step 3 (Start: 15.0s): Clicked System settings*" in output_0
|
||||
assert "*Thought*: Thought 3" not in output_0
|
||||
|
||||
# Test last_n_detailed = 1 (normal execution context, only last detailed)
|
||||
output_1 = build_plan_and_history(plan, steps, subgoal_hash_2, last_n_detailed=1)
|
||||
assert "- *Step 1 (Start: 2.5s): Launched settings*" in output_1
|
||||
assert "- *Step 2 (Start: 8.0s): Swiped down*" in output_1
|
||||
assert "- **Step 3 (Most Recent Step, Start: 15.0s)**" in output_1
|
||||
assert " - Thought 3" in output_1
|
||||
|
||||
# Test last_n_detailed = 3 (committee context, last 3 detailed)
|
||||
output_3 = build_plan_and_history(plan, steps, subgoal_hash_2, last_n_detailed=3)
|
||||
assert "- **Step 1 (Start: 2.5s)**" in output_3
|
||||
assert " - Thought 1" in output_3
|
||||
assert "- **Step 2 (Start: 8.0s)**" in output_3
|
||||
assert " - Thought 2" in output_3
|
||||
assert "- **Step 3 (Most Recent Step, Start: 15.0s)**" in output_3
|
||||
assert " - Thought 3" in output_3
|
||||
|
||||
# Test keep_subgoal_hashes (milestone filtering for checker/diagnoser)
|
||||
# Only keep steps for subgoal_hash_2
|
||||
output_filter = build_plan_and_history(
|
||||
plan, steps, subgoal_hash_2, keep_subgoal_hashes={subgoal_hash_2}
|
||||
)
|
||||
assert "Launched settings" not in output_filter # Step 1 is filtered out!
|
||||
assert "Swiped down" in output_filter # Step 2 belongs to subgoal_hash_2, kept!
|
||||
assert " - Thought 3" in output_filter # Step 3 is the last step, kept detailed!
|
||||
|
||||
|
||||
def test_get_active_subgoal_hashes_fallback():
|
||||
# Scenario 1: Normal Active Subgoal (Level 1 [/] active)
|
||||
plan_normal = """- [x] Open Settings app
|
||||
- [/] Navigate to System settings
|
||||
- [ ] Navigate to Languages & input"""
|
||||
|
||||
h_settings = hashlib.md5(b"Open Settings app").hexdigest()
|
||||
h_system = hashlib.md5(b"Navigate to System settings").hexdigest()
|
||||
|
||||
parent, sub = get_active_subgoal_hashes(plan_normal)
|
||||
assert parent == h_system
|
||||
assert sub is None
|
||||
|
||||
# Scenario 2: Indented Sub-subgoal Active (Level 2 [/] active, consolidated to parent)
|
||||
plan_indented = """- [x] Open Settings app
|
||||
- [/] Navigate to System settings
|
||||
- [/] Scroll down to find System settings
|
||||
- [ ] Navigate to Languages & input"""
|
||||
|
||||
parent, sub = get_active_subgoal_hashes(plan_indented)
|
||||
assert parent == h_system
|
||||
assert sub is None
|
||||
|
||||
# Scenario 3: Safe Fallback (All pending "[ ]", no active subgoal)
|
||||
plan_fallback = """- [ ] Open Settings app
|
||||
- [ ] Navigate to System settings
|
||||
- [ ] Navigate to Languages & input"""
|
||||
|
||||
parent, sub = get_active_subgoal_hashes(plan_fallback)
|
||||
assert parent == h_settings # Successfully fell back to the first subgoal's hash!
|
||||
assert sub is None
|
||||
|
||||
# Scenario 4: All Completed (All "[x]", no active subgoal)
|
||||
plan_all_done = """- [x] Open Settings app
|
||||
- [x] Navigate to System settings
|
||||
- [x] Navigate to Languages & input"""
|
||||
|
||||
parent, sub = get_active_subgoal_hashes(plan_all_done)
|
||||
assert parent == "default"
|
||||
assert sub is None
|
||||
|
||||
|
||||
def test_build_plan_and_history_sliding_window():
|
||||
plan = """- [x] Subgoal 1
|
||||
- [x] Subgoal 2
|
||||
- [/] Subgoal 3"""
|
||||
|
||||
subgoal_hash_1 = hashlib.md5(b"Subgoal 1").hexdigest()
|
||||
subgoal_hash_2 = hashlib.md5(b"Subgoal 2").hexdigest()
|
||||
subgoal_hash_3 = hashlib.md5(b"Subgoal 3").hexdigest()
|
||||
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"summary": "Step 1 summary",
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_1},
|
||||
},
|
||||
{
|
||||
"step_id": "step_2",
|
||||
"step_number": 2,
|
||||
"summary": "Step 2 summary",
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_1},
|
||||
},
|
||||
{
|
||||
"step_id": "step_3",
|
||||
"step_number": 3,
|
||||
"summary": "Step 3 summary",
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_2},
|
||||
},
|
||||
{
|
||||
"step_id": "step_4",
|
||||
"step_number": 4,
|
||||
"summary": "Step 4 summary",
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_3},
|
||||
},
|
||||
{
|
||||
"step_id": "step_5",
|
||||
"step_number": 5,
|
||||
"summary": "Step 5 summary",
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_3},
|
||||
},
|
||||
{
|
||||
"step_id": "step_6",
|
||||
"step_number": 6,
|
||||
"summary": "Step 6 summary",
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_3},
|
||||
},
|
||||
{
|
||||
"step_id": "step_7",
|
||||
"step_number": 7,
|
||||
"summary": "Step 7 summary",
|
||||
"extra_metadata": {"subgoal_hash": subgoal_hash_3},
|
||||
},
|
||||
]
|
||||
|
||||
# Case 1: min_summaries = 2
|
||||
# subgoal 3 has 4 steps (4, 5, 6, 7). This already exceeds min_summaries=2.
|
||||
# So older steps (1, 2, 3) from completed subgoals should be compressed/hidden!
|
||||
output_c1 = build_plan_and_history(
|
||||
plan, steps, subgoal_hash_3, min_summaries=2, last_n_detailed=0
|
||||
)
|
||||
assert "Step 1 summary" not in output_c1
|
||||
assert "Step 2 summary" not in output_c1
|
||||
assert "Step 3 summary" not in output_c1
|
||||
assert "Step 4 summary" in output_c1
|
||||
assert "Step 5 summary" in output_c1
|
||||
assert "Step 6 summary" in output_c1
|
||||
assert "Step 7 summary" in output_c1
|
||||
|
||||
# Case 2: min_summaries = 5
|
||||
# subgoal 3 has 4 steps (4, 5, 6, 7), which translates to 3 non-last steps.
|
||||
# To reach min_summaries=5 non-last steps, the sliding window slides back to keep Step 3 and Step 2 visible!
|
||||
# But Step 1 should still be compressed/hidden.
|
||||
output_c2 = build_plan_and_history(
|
||||
plan, steps, subgoal_hash_3, min_summaries=5, last_n_detailed=0
|
||||
)
|
||||
assert "Step 1 summary" not in output_c2
|
||||
assert "Step 2 summary" in output_c2 # Kept visible by sliding window!
|
||||
assert "Step 3 summary" in output_c2 # Kept visible by sliding window!
|
||||
assert "Step 4 summary" in output_c2
|
||||
assert "Step 5 summary" in output_c2
|
||||
assert "Step 6 summary" in output_c2
|
||||
assert "Step 7 summary" in output_c2
|
||||
|
||||
|
||||
def test_build_plan_and_history_clean_success_result():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Swiped screen",
|
||||
"action_taken": [{"action": "swipe"}],
|
||||
"last_execution_result": {
|
||||
"executed_actions": [{"action": "swipe"}],
|
||||
"status": "success",
|
||||
},
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
|
||||
# The output should contain "status": "success" but NOT the redundant "executed_actions"
|
||||
# Success results are stripped under Factual Realism
|
||||
assert "* [Validator Execution Result]" not in output
|
||||
assert "executed_actions" not in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_with_tool_calls():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Read note and swiped",
|
||||
"action_taken": [{"action": "swipe"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "read_note",
|
||||
"payload": {
|
||||
"args": {"key": "other_note"},
|
||||
"result": "- [ ] Milestone",
|
||||
},
|
||||
"status": "success",
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
|
||||
assert "* [Operator Decision Loop]:" in output
|
||||
assert '`read_note({"key": "other_note"})` -> - [ ] Milestone' in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_with_interleaved_events():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Read note and thought",
|
||||
"action_taken": [{"action": "swipe"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": "I want to search for files."},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "list_notes",
|
||||
"args": {"dir": "/notes"},
|
||||
"result": ["notes.txt"],
|
||||
},
|
||||
{"type": "thought", "content": "I see notes.txt, let's read it."},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "read_note",
|
||||
"args": {"file": "notes.txt"},
|
||||
"result": "Milestone plan",
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
|
||||
# It should render interleaved thoughts and tool calls in natural order
|
||||
assert " - I want to search for files." in output
|
||||
assert ' - [Tool Call]: `list_notes({"dir": "/notes"})` -> [\'notes.txt\']' in output
|
||||
assert " - I see notes.txt, let's read it." in output
|
||||
assert ' - [Tool Call]: `read_note({"file": "notes.txt"})` -> Milestone plan' in output
|
||||
|
||||
# It should use the new section header
|
||||
assert "* [Operator Decision Loop]:" in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_concatenates_thoughts():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Step with multiple thoughts",
|
||||
"action_taken": [{"action": "click", "thought": "Action thought"}],
|
||||
"operator_raw_thinking": "Raw thought",
|
||||
"last_execution_result": {"status": "success"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": "Event thought"},
|
||||
],
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
|
||||
# Check that all thoughts are concatenated and visible in the output
|
||||
assert " - Action thought" in output
|
||||
assert " - Event thought" in output
|
||||
assert " - Raw thought" in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_chronological_last_step():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Completed search",
|
||||
"action_taken": [{"action": "click", "target_text": "Search"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": "I should find search icon."},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "ask_explorer",
|
||||
"args": {"query": "search icon"},
|
||||
"result": {"center": [500, 600]},
|
||||
},
|
||||
{
|
||||
"type": "native_thought",
|
||||
"content": "Clicking coordinates [500, 600] now.",
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(
|
||||
plan, steps, "default", last_n_detailed=1, chronological_last_step=True
|
||||
)
|
||||
|
||||
assert plan in output
|
||||
assert "--- Execution History ---" in output
|
||||
assert "- **Step 1 (Most Recent Step, Start: 2.5s)**" in output
|
||||
assert "* [Operator Decision Loop]:" in output
|
||||
assert " - I should find search icon." in output
|
||||
assert (
|
||||
'- [Tool Call]: `ask_explorer({"query": "search icon"})` -> {"center":'
|
||||
" [500, 600]}" in output
|
||||
)
|
||||
assert " - Clicking coordinates [500, 600] now." in output
|
||||
assert "* [Planned Action]: Tapped 'Search' at None" in output
|
||||
assert "* [Validator Execution Result]" not in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_safety_net_and_failure_analyzer():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "5.0s",
|
||||
"summary": "Repaired click search",
|
||||
"action_taken": [{"action": "click", "target_text": "Search"}],
|
||||
"last_execution_result": {"status": "success"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": "Let's click search."},
|
||||
# Safety net failure
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "safety_net_validation",
|
||||
"args": {"target": "Search"},
|
||||
"result": [
|
||||
False,
|
||||
"TARGET_DISAPPEARED",
|
||||
"Target button is not visible",
|
||||
],
|
||||
},
|
||||
# Failure Analyzer intervention
|
||||
{
|
||||
"type": "failure_analyzer_thought",
|
||||
"content": ("The search button is covered by keyboard, need to go BACK first."),
|
||||
},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "_exec_press_key",
|
||||
"args": {"key": "BACK", "state": "<State>"},
|
||||
"result": ["Pressed key BACK", "<Bytes>", "/path/to/shot.jpg"],
|
||||
},
|
||||
{
|
||||
"type": "failure_analyzer_thought",
|
||||
"content": "Keyboard is closed, now re-executing search tap.",
|
||||
},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "_exec_click",
|
||||
"args": {"target": [500, 600]},
|
||||
"result": ["Tapped search button successfully", "<Bytes>"],
|
||||
},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "report_failure_analysis",
|
||||
"args": {
|
||||
"status": "fixed",
|
||||
"analysis": "Closed keyboard and tapped search.",
|
||||
},
|
||||
"result": "Success",
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(
|
||||
plan, steps, "default", last_n_detailed=1, chronological_last_step=True
|
||||
)
|
||||
|
||||
assert "- **Step 1 (Most Recent Step, Start: 5.0s)**" in output
|
||||
assert "* [Operator Decision Loop]:" in output
|
||||
assert " - Let's click search." in output
|
||||
assert (
|
||||
"* [Planned Action]: Tapped 'Search' at None (Intercepted by"
|
||||
" Pre-Execution Safety Net)" in output
|
||||
)
|
||||
assert "* [Pre-Execution Safety Net]:" in output
|
||||
assert "- [Safety Net Check]: (Action not executed: Target button is not visible)" in output
|
||||
assert "* [Failure Analyzer Recovery Loop]:" in output
|
||||
assert " - [Tool Call]: Pressed key 'BACK'" in output
|
||||
assert " - [Tool Call]: Tapped element at [500, 600]" in output
|
||||
|
||||
|
||||
def test_build_plan_and_history_repaired_result():
|
||||
plan = "- [ ] Subgoal 1"
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "2.5s",
|
||||
"summary": "Tapped button with repair",
|
||||
"action_taken": [{"action": "click"}],
|
||||
"last_execution_result": {
|
||||
"execution": [
|
||||
{
|
||||
"action": "click",
|
||||
"attempts": ["pre-validation failed"],
|
||||
"repair": ("Tapped center to wake controls and then tapped target"),
|
||||
}
|
||||
],
|
||||
"status": "success",
|
||||
"repair_status": "fixed",
|
||||
},
|
||||
}
|
||||
]
|
||||
output = build_plan_and_history(plan, steps, "default", last_n_detailed=1)
|
||||
assert " * [Validator Execution Result]: Action executed." not in output
|
||||
assert " * [Result]: Repaired: Tapped center to wake controls and then tapped target" in output
|
||||
|
||||
|
||||
def test_get_recent_subgoal_hashes_robust_milestone_exclusion(tmp_path):
|
||||
# Scenario:
|
||||
# 1. Milestone 1 is completed (hash_completed_1).
|
||||
# 2. We had a failed attempt 'hash_failed_old' during Milestone 1.
|
||||
# 3. Milestone 2 is completed (hash_completed_2).
|
||||
# 4. We are now working on Milestone C (hash_active_c).
|
||||
# 5. Milestone C had two previous failed renamed versions: Failed Attempt A and Failed Attempt B.
|
||||
# 6. subgoal_hash_chain.json maps: A -> B -> C.
|
||||
|
||||
base_dir = tmp_path
|
||||
notes_dir = base_dir / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
chain_path = notes_dir / "subgoal_hash_chain.json"
|
||||
|
||||
hash_completed_1 = hashlib.md5(b"Completed Milestone 1").hexdigest()
|
||||
hash_completed_2 = hashlib.md5(b"Completed Milestone 2").hexdigest()
|
||||
hash_active_c = hashlib.md5(b"Active Milestone C").hexdigest()
|
||||
|
||||
hash_failed_old = hashlib.md5(b"Failed Attempt Old").hexdigest()
|
||||
hash_failed_a = hashlib.md5(b"Failed Attempt A").hexdigest()
|
||||
hash_failed_b = hashlib.md5(b"Failed Attempt B").hexdigest()
|
||||
|
||||
# Write the hash chain representing: A -> B -> C
|
||||
import json
|
||||
|
||||
chain_data = {hash_failed_a: hash_failed_b, hash_failed_b: hash_active_c}
|
||||
chain_path.write_text(json.dumps(chain_data), encoding="utf-8")
|
||||
|
||||
steps = [
|
||||
# Steps from Completed Milestone 1
|
||||
{"step_id": "s1", "extra_metadata": {"subgoal_hash": hash_completed_1}},
|
||||
# Old failed attempt during Milestone 1
|
||||
{"step_id": "s2", "extra_metadata": {"subgoal_hash": hash_failed_old}},
|
||||
# Steps from Completed Milestone 2 (Most recent completed milestone)
|
||||
{"step_id": "s3", "extra_metadata": {"subgoal_hash": hash_completed_2}},
|
||||
{"step_id": "s4", "extra_metadata": {"subgoal_hash": hash_completed_2}},
|
||||
# Steps from Failed Attempt A (Part of the active task slot's past renames)
|
||||
{"step_id": "s5", "extra_metadata": {"subgoal_hash": hash_failed_a}},
|
||||
{"step_id": "s6", "extra_metadata": {"subgoal_hash": hash_failed_a}},
|
||||
# Steps from Failed Attempt B (Part of the active task slot's past renames)
|
||||
{"step_id": "s7", "extra_metadata": {"subgoal_hash": hash_failed_b}},
|
||||
{"step_id": "s8", "extra_metadata": {"subgoal_hash": hash_failed_b}},
|
||||
# Steps from Active Milestone C
|
||||
{"step_id": "s9", "extra_metadata": {"subgoal_hash": hash_active_c}},
|
||||
]
|
||||
|
||||
# Test raw transitive alias resolver
|
||||
from artemis.utils.task_tree import get_all_subgoal_aliases
|
||||
|
||||
aliases = get_all_subgoal_aliases(hash_active_c, base_dir)
|
||||
assert aliases == {hash_active_c, hash_failed_b, hash_failed_a}
|
||||
|
||||
keep_hashes = get_recent_subgoal_hashes(steps, hash_active_c, base_dir)
|
||||
|
||||
# Should keep:
|
||||
# 1. The current active subgoal (hash_active_c)
|
||||
assert hash_active_c in keep_hashes
|
||||
|
||||
# 2. The failed/renamed subgoals during active slot (hash_failed_b, hash_failed_a) - resolved transitively via chain
|
||||
assert hash_failed_b in keep_hashes
|
||||
assert hash_failed_a in keep_hashes
|
||||
|
||||
# 3. The single most recent completed subgoal (hash_completed_2) for continuity context
|
||||
assert hash_completed_2 in keep_hashes
|
||||
|
||||
# Should prune:
|
||||
# 4. Older completed subgoals (hash_completed_1)
|
||||
assert hash_completed_1 not in keep_hashes
|
||||
|
||||
# 5. Older failed subgoals from previously completed milestones (hash_failed_old)
|
||||
assert hash_failed_old not in keep_hashes
|
||||
|
||||
|
||||
def test_build_plan_and_history_for_failure_analyzer():
|
||||
plan_with_subgoals = """- [/] Goal 1
|
||||
- [x] Subtask 1.1
|
||||
- [/] Subtask 1.2"""
|
||||
|
||||
steps = [
|
||||
{
|
||||
"step_id": "step_1",
|
||||
"step_number": 1,
|
||||
"relative_time": "12.3s",
|
||||
"summary": "Tapped full screen",
|
||||
"action_taken": [{"action": "click", "target_text": "Fullscreen"}],
|
||||
"operator_raw_thinking": "Let's tap it",
|
||||
"operator_native_thinking": "Thinking to tap",
|
||||
"last_execution_result": {"status": "failed"},
|
||||
"interleaved_events": [
|
||||
{"type": "thought", "content": "I should find fullscreen icon."},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"name": "safety_net_validation",
|
||||
"args": {"target": "Fullscreen"},
|
||||
"result": [
|
||||
False,
|
||||
"TARGET_DISAPPEARED",
|
||||
"Target button is not visible",
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
# Test with subgoals and for_failure_analyzer=True
|
||||
output_analyzer_with_plan = build_plan_and_history(
|
||||
plan_with_subgoals,
|
||||
steps,
|
||||
"default",
|
||||
last_n_detailed=1,
|
||||
for_failure_analyzer=True,
|
||||
)
|
||||
assert "--- Task Plan ---" in output_analyzer_with_plan
|
||||
assert (
|
||||
"*(Note: Provided for context only. Do not execute the remaining"
|
||||
" plan.)*" in output_analyzer_with_plan
|
||||
)
|
||||
assert "--- Execution History ---" in output_analyzer_with_plan
|
||||
assert (
|
||||
"- **Step 1 (Most Recent Step (Failed to execute, this is the step you"
|
||||
" need to focus on and repair), Start: 12.3s)**" in output_analyzer_with_plan
|
||||
)
|
||||
assert "[Operator Decision Loop]" in output_analyzer_with_plan
|
||||
assert "I should find fullscreen icon." in output_analyzer_with_plan
|
||||
assert (
|
||||
"* [Planned Action]: Tapped 'Fullscreen' at None (Intercepted by"
|
||||
" Pre-Execution Safety Net)" in output_analyzer_with_plan
|
||||
)
|
||||
|
||||
# Test with empty plan and for_failure_analyzer=True (should completely omit plan block)
|
||||
output_analyzer_empty_plan = build_plan_and_history(
|
||||
"", steps, "default", last_n_detailed=1, for_failure_analyzer=True
|
||||
)
|
||||
assert "--- Task Plan ---" not in output_analyzer_empty_plan
|
||||
assert (
|
||||
"*(Note: Provided for context only. Do not execute the remaining"
|
||||
" plan.)*" not in output_analyzer_empty_plan
|
||||
)
|
||||
assert "--- Execution History ---" in output_analyzer_empty_plan
|
||||
@@ -0,0 +1,349 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from artemis.utils.ui_filter import (
|
||||
_clip_bounds,
|
||||
_is_semantic_empty,
|
||||
filter_node,
|
||||
filter_ui_hierarchy,
|
||||
)
|
||||
|
||||
|
||||
def test_is_semantic_empty():
|
||||
# Interactive or textual nodes are NOT empty
|
||||
assert not _is_semantic_empty({"text": "Hello"})
|
||||
assert not _is_semantic_empty({"content-desc": "Button"})
|
||||
assert not _is_semantic_empty({"clickable": True})
|
||||
assert not _is_semantic_empty({"clickable": "true"})
|
||||
assert not _is_semantic_empty({"focusable": True})
|
||||
assert not _is_semantic_empty({"className": "android.widget.ImageView"})
|
||||
|
||||
# Empty containers ARE semantic empty
|
||||
assert _is_semantic_empty(
|
||||
{
|
||||
"text": "",
|
||||
"clickable": False,
|
||||
"className": "android.widget.FrameLayout",
|
||||
}
|
||||
)
|
||||
assert _is_semantic_empty({})
|
||||
|
||||
|
||||
def test_clip_bounds():
|
||||
node = {"bounds": "[10,-10][1200,100]"}
|
||||
bounds = {"left": 10, "top": -10, "right": 1200, "bottom": 100}
|
||||
ancestor_bounds = {"left": 0, "top": 0, "right": 1080, "bottom": 2400}
|
||||
clipped = _clip_bounds(node, bounds, ancestor_bounds)
|
||||
|
||||
assert clipped["left"] == 10
|
||||
assert clipped["top"] == 0
|
||||
assert clipped["right"] == 1080
|
||||
assert clipped["bottom"] == 100
|
||||
assert node["bounds"] == "[10,0][1080,100]"
|
||||
assert "is_clipped" not in node
|
||||
|
||||
|
||||
def test_clip_bounds_fully_clipped():
|
||||
node = {"bounds": "[1200,2500][1300,2600]"}
|
||||
bounds = {"left": 1200, "top": 2500, "right": 1300, "bottom": 2600}
|
||||
ancestor_bounds = {"left": 0, "top": 0, "right": 1080, "bottom": 2400}
|
||||
clipped = _clip_bounds(node, bounds, ancestor_bounds)
|
||||
|
||||
assert clipped["left"] == 1080
|
||||
assert clipped["top"] == 2400
|
||||
assert clipped["right"] == 1080
|
||||
assert clipped["bottom"] == 2400
|
||||
assert node["is_clipped"] is True
|
||||
assert node["bounds"] == "[1080,2400][1080,2400]"
|
||||
|
||||
|
||||
def test_semantic_hoisting():
|
||||
hierarchy = [
|
||||
{
|
||||
"bounds": "[0,0][100,100]",
|
||||
"text": "",
|
||||
"clickable": False,
|
||||
"children": [
|
||||
{
|
||||
"bounds": "[10,10][90,90]",
|
||||
"text": "Child Button",
|
||||
"clickable": True,
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
# Mode: hoist
|
||||
res_hoist = filter_ui_hierarchy(hierarchy, semantic_pruning="hoist")
|
||||
assert len(res_hoist) == 1
|
||||
assert res_hoist[0]["text"] == "Child Button"
|
||||
|
||||
# Mode: safe
|
||||
res_safe = filter_ui_hierarchy(hierarchy, semantic_pruning="safe")
|
||||
assert len(res_safe) == 1
|
||||
assert res_safe[0]["text"] == ""
|
||||
assert len(res_safe[0]["children"]) == 1
|
||||
|
||||
|
||||
def test_dynamic_min_size():
|
||||
# Short side is 1000. 0.5% of 1000 is 5px.
|
||||
# Element size is 4x4 -> filtered out.
|
||||
node = {"bounds": "[0,0][4,4]", "text": "Tiny", "clickable": True}
|
||||
res = filter_node(node, 1000, 2000, min_element_size=2, use_dynamic_min_size=True)
|
||||
assert res is None
|
||||
|
||||
# Element size is 6x6 -> kept.
|
||||
node = {"bounds": "[0,0][6,6]", "text": "Good", "clickable": True}
|
||||
res = filter_node(node, 1000, 2000, min_element_size=2, use_dynamic_min_size=True)
|
||||
assert res is not None
|
||||
|
||||
|
||||
def test_ancestor_bounds_propagation():
|
||||
hierarchy = {
|
||||
"bounds": "[0,0][100,100]",
|
||||
"text": "Parent",
|
||||
"clickable": True,
|
||||
"children": [{"bounds": "[50,50][150,150]", "text": "Child", "clickable": True}],
|
||||
}
|
||||
|
||||
# The child is at [50,50][150,150], but parent is [0,0][100,100].
|
||||
# So child should be clipped to [50,50][100,100].
|
||||
res = filter_node(hierarchy, 1000, 1000, clip_bounds=True)
|
||||
assert res is not None
|
||||
assert res["children"][0]["bounds"] == "[50,50][100,100]"
|
||||
|
||||
|
||||
def test_sibling_merging():
|
||||
hierarchy = {
|
||||
"clickable": True,
|
||||
"bounds": "[0,0][200,200]",
|
||||
"children": [
|
||||
{
|
||||
"className": "android.widget.ImageView",
|
||||
"bounds": "[10,10][50,50]",
|
||||
"clickable": False,
|
||||
"focusable": False,
|
||||
},
|
||||
{
|
||||
"className": "android.widget.TextView",
|
||||
"text": "Settings",
|
||||
"bounds": "[60,10][150,50]",
|
||||
"clickable": False,
|
||||
"focusable": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
# The parent is clickable, has 1 text child and 1 image child without text.
|
||||
# They should be merged!
|
||||
res = filter_node(hierarchy, 1000, 1000)
|
||||
assert res is not None
|
||||
assert "children" in res
|
||||
assert len(res["children"]) == 1
|
||||
assert res["children"][0]["text"] == "Settings"
|
||||
assert res["children"][0]["content-desc"] == "[Icon]"
|
||||
assert res["children"][0]["bounds"] == "[10,10][150,50]"
|
||||
|
||||
|
||||
def test_sibling_merging_not_triggered_if_child_clickable():
|
||||
hierarchy = {
|
||||
"clickable": True,
|
||||
"bounds": "[0,0][200,200]",
|
||||
"children": [
|
||||
{
|
||||
"className": "android.widget.ImageView",
|
||||
"bounds": "[10,10][50,50]",
|
||||
"clickable": True, # Child is clickable
|
||||
"focusable": False,
|
||||
},
|
||||
{
|
||||
"className": "android.widget.TextView",
|
||||
"text": "Settings",
|
||||
"bounds": "[60,10][150,50]",
|
||||
"clickable": False,
|
||||
"focusable": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
res = filter_node(hierarchy, 1000, 1000)
|
||||
assert res is not None
|
||||
assert "children" in res
|
||||
assert len(res["children"]) == 2 # No merge
|
||||
|
||||
|
||||
def test_sibling_merging_not_triggered_if_form_control():
|
||||
hierarchy = {
|
||||
"clickable": True,
|
||||
"bounds": "[0,0][200,200]",
|
||||
"children": [
|
||||
{
|
||||
"className": "android.widget.CheckBox", # Form control
|
||||
"bounds": "[10,10][50,50]",
|
||||
"clickable": True, # Make it clickable so it's not pruned
|
||||
"focusable": False,
|
||||
},
|
||||
{
|
||||
"className": "android.widget.TextView",
|
||||
"text": "Enable",
|
||||
"bounds": "[60,10][150,50]",
|
||||
"clickable": False,
|
||||
"focusable": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
res = filter_node(hierarchy, 1000, 1000)
|
||||
assert res is not None
|
||||
assert "children" in res
|
||||
assert len(res["children"]) == 2 # No merge
|
||||
|
||||
|
||||
def test_parent_child_redundancy():
|
||||
# Case 1: Redundant TextView removed
|
||||
hierarchy = {
|
||||
"text": "Confirm",
|
||||
"clickable": True,
|
||||
"bounds": "[0,0][200,200]",
|
||||
"children": [
|
||||
{
|
||||
"className": "android.widget.TextView",
|
||||
"text": "Confirm",
|
||||
"bounds": "[10,10][190,190]",
|
||||
"clickable": False,
|
||||
"focusable": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
res = filter_node(hierarchy, 1000, 1000)
|
||||
assert res is not None
|
||||
assert "children" in res
|
||||
assert len(res["children"]) == 0 # Child removed
|
||||
|
||||
# Case 2: Redundant CheckBox text cleared
|
||||
hierarchy2 = {
|
||||
"text": "Confirm",
|
||||
"clickable": True,
|
||||
"bounds": "[0,0][200,200]",
|
||||
"children": [
|
||||
{
|
||||
"className": "android.widget.CheckBox",
|
||||
"text": "Confirm",
|
||||
"bounds": "[10,10][190,190]",
|
||||
"clickable": False,
|
||||
"focusable": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
res2 = filter_node(hierarchy2, 1000, 1000)
|
||||
assert res2 is not None
|
||||
assert "children" in res2
|
||||
assert len(res2["children"]) == 1
|
||||
assert res2["children"][0]["text"] == ""
|
||||
|
||||
# Case 3: Redundant interactive node text cleared
|
||||
hierarchy3 = {
|
||||
"text": "Confirm",
|
||||
"clickable": True,
|
||||
"bounds": "[0,0][200,200]",
|
||||
"children": [
|
||||
{
|
||||
"className": "android.widget.TextView",
|
||||
"text": "Confirm",
|
||||
"bounds": "[10,10][190,190]",
|
||||
"clickable": True,
|
||||
"focusable": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
res3 = filter_node(hierarchy3, 1000, 1000)
|
||||
assert res3 is not None
|
||||
assert "children" in res3
|
||||
assert len(res3["children"]) == 1
|
||||
assert res3["children"][0]["text"] == ""
|
||||
|
||||
# Case 4: No match
|
||||
hierarchy4 = {
|
||||
"text": "Confirm",
|
||||
"clickable": True,
|
||||
"bounds": "[0,0][200,200]",
|
||||
"children": [
|
||||
{
|
||||
"className": "android.widget.TextView",
|
||||
"text": "Cancel",
|
||||
"bounds": "[10,10][190,190]",
|
||||
"clickable": False,
|
||||
"focusable": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
res4 = filter_node(hierarchy4, 1000, 1000)
|
||||
assert res4 is not None
|
||||
assert "children" in res4
|
||||
assert len(res4["children"]) == 1
|
||||
assert res4["children"][0]["text"] == "Cancel"
|
||||
|
||||
|
||||
def test_fixed_system_bars_clamping_and_discarding():
|
||||
# Simulate a full-width fixed bottom navigation bar at [0, 2250][1080, 2400]
|
||||
# and two candidates: one slightly occluded (should be safely clamped) and one mostly covered (should be discarded).
|
||||
hierarchy = [
|
||||
{
|
||||
"resource-id": "com.google.android.youtube:id/bottom_navigation",
|
||||
"bounds": "[0,2250][1080,2400]",
|
||||
"scrollable": False,
|
||||
"clickable": False,
|
||||
},
|
||||
{
|
||||
"text": "Partially occluded video item",
|
||||
"bounds": "[0,2100][1080,2300]",
|
||||
"scrollable": False,
|
||||
"clickable": True,
|
||||
},
|
||||
{
|
||||
"text": "Almost entirely covered sliver item",
|
||||
"bounds": "[0,2245][1080,2300]",
|
||||
"scrollable": False,
|
||||
"clickable": True,
|
||||
},
|
||||
]
|
||||
|
||||
filtered = filter_ui_hierarchy(
|
||||
hierarchy, screen_width=1080, screen_height=2400, min_element_size=20
|
||||
)
|
||||
# The bottom_navigation bar itself is kept (or filtered depending on semantics, here kept if not empty / checked).
|
||||
# Specifically check the clamped item and discarded item:
|
||||
texts = [el.get("text") for el in filtered]
|
||||
assert "Partially occluded video item" in texts
|
||||
assert "Almost entirely covered sliver item" not in texts
|
||||
|
||||
clamped_item = next(el for el in filtered if el.get("text") == "Partially occluded video item")
|
||||
# Its original bottom was 2300, clamped safely to the bar's top at 2250
|
||||
assert clamped_item["bounds"] == "[0,2100][1080,2250]"
|
||||
|
||||
|
||||
def test_mutual_occlusion_warning_injection():
|
||||
from artemis.utils.visualization import format_minimal_list_with_elements
|
||||
|
||||
# Simulate two overlapping non-ancestor interactive items with >= 50% overlap
|
||||
fused_xml = [
|
||||
{"text": "Search Video Result", "bounds": "[50,1800][1000,2000]"},
|
||||
{"text": "Create Floating Button", "bounds": "[750,1850][1000,2000]"},
|
||||
]
|
||||
|
||||
minimal_list, elements, labels = format_minimal_list_with_elements(
|
||||
fused_xml, width=1080, height=2400
|
||||
)
|
||||
assert "WARNING: may overlap with [2], possible occlusion" in minimal_list
|
||||
assert "WARNING: may overlap with [1], possible occlusion" in minimal_list
|
||||
@@ -0,0 +1,243 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from artemis.utils.ui_hierarchy import (
|
||||
ElementBounds,
|
||||
Point,
|
||||
find_element_by_resource_id,
|
||||
get_bounds_for_element,
|
||||
get_element_text,
|
||||
is_element_focused,
|
||||
text_input_is_empty,
|
||||
)
|
||||
|
||||
|
||||
def test_text_input_is_empty():
|
||||
assert text_input_is_empty(text=None, hint_text=None)
|
||||
assert text_input_is_empty(text="", hint_text=None)
|
||||
assert text_input_is_empty(text="", hint_text="")
|
||||
assert text_input_is_empty(text="text", hint_text="text")
|
||||
|
||||
assert not text_input_is_empty(text="text", hint_text=None)
|
||||
assert not text_input_is_empty(text="text", hint_text="")
|
||||
|
||||
|
||||
def test_find_element_by_resource_id():
|
||||
ui_hierarchy = [
|
||||
{
|
||||
"resourceId": "com.example:id/button1",
|
||||
"text": "Button 1",
|
||||
"children": [],
|
||||
},
|
||||
{
|
||||
"resourceId": "com.example:id/container",
|
||||
"children": [
|
||||
{
|
||||
"resourceId": "com.example:id/nested_button",
|
||||
"text": "Nested Button",
|
||||
"children": [],
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button1")
|
||||
assert result is not None
|
||||
assert result["resourceId"] == "com.example:id/button1"
|
||||
assert result["text"] == "Button 1"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nested_button")
|
||||
assert result is not None
|
||||
assert result["resourceId"] == "com.example:id/nested_button"
|
||||
assert result["text"] == "Nested Button"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nonexistent")
|
||||
assert result is None
|
||||
|
||||
result = find_element_by_resource_id([], "com.example:id/button1")
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_find_element_by_resource_id_with_index():
|
||||
ui_hierarchy = [
|
||||
{
|
||||
"resourceId": "com.example:id/button",
|
||||
"text": "Button 1",
|
||||
"children": [],
|
||||
},
|
||||
{
|
||||
"resourceId": "com.example:id/button",
|
||||
"text": "Button 2",
|
||||
"children": [],
|
||||
},
|
||||
{
|
||||
"resourceId": "com.example:id/container",
|
||||
"children": [
|
||||
{
|
||||
"resourceId": "com.example:id/button",
|
||||
"text": "Button 3",
|
||||
"children": [],
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=0)
|
||||
assert result is not None
|
||||
assert result["text"] == "Button 1"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=1)
|
||||
assert result is not None
|
||||
assert result["text"] == "Button 2"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=2)
|
||||
assert result is not None
|
||||
assert result["text"] == "Button 3"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=3)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_find_element_by_resource_id_rich_hierarchy():
|
||||
rich_hierarchy = [
|
||||
{
|
||||
"attributes": {"resource-id": "com.example:id/button1"},
|
||||
"children": [],
|
||||
},
|
||||
{
|
||||
"attributes": {"resource-id": "com.example:id/container"},
|
||||
"children": [
|
||||
{
|
||||
"attributes": {"resource-id": "com.example:id/nested_button"},
|
||||
"children": [],
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = find_element_by_resource_id(
|
||||
rich_hierarchy, "com.example:id/button1", is_rich_hierarchy=True
|
||||
)
|
||||
assert result is not None
|
||||
assert result["resource-id"] == "com.example:id/button1"
|
||||
|
||||
result = find_element_by_resource_id(
|
||||
rich_hierarchy, "com.example:id/nested_button", is_rich_hierarchy=True
|
||||
)
|
||||
assert result is not None
|
||||
assert result["resource-id"] == "com.example:id/nested_button"
|
||||
|
||||
result = find_element_by_resource_id(
|
||||
rich_hierarchy, "com.example:id/nonexistent", is_rich_hierarchy=True
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_is_element_focused():
|
||||
focused_element = {"focused": "true"}
|
||||
assert is_element_focused(focused_element)
|
||||
|
||||
non_focused_element = {"focused": "false"}
|
||||
assert not is_element_focused(non_focused_element)
|
||||
|
||||
no_focused_element = {"text": "some text"}
|
||||
assert not is_element_focused(no_focused_element)
|
||||
|
||||
none_focused_element = {"focused": None}
|
||||
assert not is_element_focused(none_focused_element)
|
||||
|
||||
|
||||
def test_get_element_text():
|
||||
element = {"text": "Button Text", "hintText": "Hint Text"}
|
||||
assert get_element_text(element) == "Button Text"
|
||||
assert get_element_text(element, hint_text=False) == "Button Text"
|
||||
assert get_element_text(element, hint_text=True) == "Hint Text"
|
||||
|
||||
element_no_text = {"hintText": "Hint Text"}
|
||||
assert get_element_text(element_no_text) is None
|
||||
assert get_element_text(element_no_text, hint_text=True) == "Hint Text"
|
||||
element_no_hint = {"text": "Button Text"}
|
||||
assert get_element_text(element_no_hint) == "Button Text"
|
||||
assert get_element_text(element_no_hint, hint_text=True) is None
|
||||
|
||||
empty_element = {}
|
||||
assert get_element_text(empty_element) is None
|
||||
assert get_element_text(empty_element, hint_text=True) is None
|
||||
|
||||
|
||||
def test_get_bounds_for_element():
|
||||
element_with_bounds = {"bounds": {"x": 10, "y": 20, "width": 100, "height": 50}}
|
||||
bounds = get_bounds_for_element(element_with_bounds)
|
||||
assert bounds is not None
|
||||
assert isinstance(bounds, ElementBounds)
|
||||
assert bounds.x == 10
|
||||
assert bounds.y == 20
|
||||
assert bounds.width == 100
|
||||
assert bounds.height == 50
|
||||
|
||||
element_no_bounds = {"text": "Button"}
|
||||
bounds = get_bounds_for_element(element_no_bounds)
|
||||
assert bounds is None
|
||||
|
||||
# Suppress logger output for the invalid bounds test case
|
||||
with patch("artemis.utils.ui_hierarchy.logger.error"):
|
||||
element_invalid_bounds = {
|
||||
"bounds": {
|
||||
"x": "invalid", # Should be int
|
||||
"y": 20,
|
||||
"width": 100,
|
||||
"height": 50,
|
||||
}
|
||||
}
|
||||
bounds = get_bounds_for_element(element_invalid_bounds)
|
||||
assert bounds is None
|
||||
|
||||
|
||||
def test_element_bounds():
|
||||
bounds = ElementBounds(x=10, y=20, width=100, height=50)
|
||||
|
||||
center = bounds.get_center()
|
||||
assert isinstance(center, Point)
|
||||
assert center.x == 60
|
||||
assert center.y == 45
|
||||
|
||||
center_point = bounds.get_relative_point(0.5, 0.5)
|
||||
assert isinstance(center_point, Point)
|
||||
assert center_point.x == 60
|
||||
assert center_point.y == 45
|
||||
|
||||
top_left = bounds.get_relative_point(0.0, 0.0)
|
||||
assert top_left.x == 10
|
||||
assert top_left.y == 20
|
||||
|
||||
bottom_right = bounds.get_relative_point(1.0, 1.0)
|
||||
assert bottom_right.x == 110
|
||||
assert bottom_right.y == 70
|
||||
custom_point = bounds.get_relative_point(0.95, 0.95)
|
||||
assert custom_point.x == 105
|
||||
assert custom_point.y == 67
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_text_input_is_empty()
|
||||
test_find_element_by_resource_id()
|
||||
test_find_element_by_resource_id_with_index()
|
||||
test_find_element_by_resource_id_rich_hierarchy()
|
||||
test_is_element_focused()
|
||||
test_get_element_text()
|
||||
test_get_bounds_for_element()
|
||||
test_element_bounds()
|
||||
print("All tests passed")
|
||||
@@ -0,0 +1,77 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
from artemis.utils.verification import (
|
||||
append_verification_chat,
|
||||
get_verification_chat_path,
|
||||
get_verification_chat_rounds,
|
||||
read_verification_chat,
|
||||
)
|
||||
|
||||
|
||||
def test_get_verification_chat_path(tmp_path):
|
||||
subgoal_hash = "test_hash"
|
||||
expected_path = tmp_path / "notes" / f"verification_chat_{subgoal_hash}.json"
|
||||
path = get_verification_chat_path(tmp_path, subgoal_hash)
|
||||
assert path == expected_path
|
||||
|
||||
|
||||
def test_read_verification_chat_not_exists(tmp_path):
|
||||
path = tmp_path / "non_existent.json"
|
||||
turns = read_verification_chat(path)
|
||||
assert turns == []
|
||||
|
||||
|
||||
def test_read_verification_chat_exists(tmp_path):
|
||||
path = tmp_path / "chat.json"
|
||||
expected_turns = [{"role": "operator", "round": 1, "content": "hello"}]
|
||||
path.write_text(json.dumps(expected_turns), encoding="utf-8")
|
||||
turns = read_verification_chat(path)
|
||||
assert turns == expected_turns
|
||||
|
||||
|
||||
def test_get_verification_chat_rounds():
|
||||
turns = [
|
||||
{"role": "operator", "round": 1, "content": "op1"},
|
||||
{"role": "checker", "round": 1, "content": "chk1"},
|
||||
{"role": "operator", "round": 2, "content": "op2"},
|
||||
]
|
||||
max_op, max_chk = get_verification_chat_rounds(turns)
|
||||
assert max_op == 2
|
||||
assert max_chk == 1
|
||||
|
||||
|
||||
def test_get_verification_chat_rounds_empty():
|
||||
max_op, max_chk = get_verification_chat_rounds([])
|
||||
assert max_op == 0
|
||||
assert max_chk == 0
|
||||
|
||||
|
||||
def test_append_verification_chat(tmp_path):
|
||||
path = tmp_path / "chat.json"
|
||||
|
||||
# Append to empty/non-existent
|
||||
success = append_verification_chat(path, "operator", "hello", 1)
|
||||
assert success
|
||||
assert path.exists()
|
||||
turns = json.loads(path.read_text(encoding="utf-8"))
|
||||
assert turns == [{"role": "operator", "round": 1, "content": "hello"}]
|
||||
|
||||
# Append again
|
||||
success = append_verification_chat(path, "checker", "hi", 2)
|
||||
assert success
|
||||
turns = json.loads(path.read_text(encoding="utf-8"))
|
||||
assert len(turns) == 2
|
||||
assert turns[1] == {"role": "checker", "round": 2, "content": "hi"}
|
||||
Reference in new issue
Block a user