refactor: clean up legacy code, rename hopper to entity_extractor, and add third-party notices
- Add NOTICE file and README license references for mobile-use and finalrun-agent - Add derivative attribution headers to 17 files derived from mobile-use - Rename hopper agent to entity_extractor across prompts, tools, and configs - Remove obsolete adb_tunnel, ApiBaseUrl, and legacy CLI utility scripts - Replace legacy examples with native Artemis diagnostic and workflow examples - Update unit and tool tests to validate entity_extractor and SDK builders
This commit is contained in:
1 parent
0860788192
commit
54fcef9887
83 files changed
+1996
-3660
No files matched your search
+1
-1
@@ -14,6 +14,6 @@
|
||||
|
||||
"""This file is intentionally empty.
|
||||
|
||||
The hopper agent does not expose any tools directly. It is invoked
|
||||
The entity_extractor agent does not expose any tools directly. It is invoked
|
||||
programmatically by other tools (e.g., launch_app_tool) as an LLM utility.
|
||||
"""
|
||||
@@ -0,0 +1,76 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for Artemis entity extraction and package resolver."""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import pytest
|
||||
|
||||
from artemis.agents.entity_extractor.entity_extractor import (
|
||||
ExtractionResult,
|
||||
extract_entity,
|
||||
)
|
||||
|
||||
|
||||
def test_extraction_result_model():
|
||||
"""Verify ExtractionResult attributes and validation."""
|
||||
res = ExtractionResult(found=True, output="com.google.android.apps.nexuslauncher", reason="Exact match")
|
||||
assert res.found is True
|
||||
assert res.output == "com.google.android.apps.nexuslauncher"
|
||||
assert res.reason == "Exact match"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_entity_success():
|
||||
"""Verify successful LLM entity extraction invocation."""
|
||||
mock_ctx = MagicMock()
|
||||
mock_llm = MagicMock()
|
||||
expected_result = ExtractionResult(
|
||||
found=True,
|
||||
output="com.google.android.calculator",
|
||||
reason="Found calculator package in list",
|
||||
)
|
||||
mock_llm.ainvoke = AsyncMock(return_value=expected_result)
|
||||
|
||||
with patch("artemis.agents.entity_extractor.entity_extractor.get_llm") as mock_get_llm:
|
||||
mock_get_llm.return_value.with_structured_output.return_value = mock_llm
|
||||
result = await extract_entity(
|
||||
request="Calculator",
|
||||
data="com.google.android.calculator\ncom.google.android.calendar",
|
||||
ctx=mock_ctx,
|
||||
)
|
||||
|
||||
assert result.found is True
|
||||
assert result.output == "com.google.android.calculator"
|
||||
assert result.reason == "Found calculator package in list"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_entity_handles_exception():
|
||||
"""Verify extraction returns a graceful failure result when LLM throws an exception."""
|
||||
mock_ctx = MagicMock()
|
||||
mock_llm = MagicMock()
|
||||
mock_llm.ainvoke = AsyncMock(side_effect=RuntimeError("LLM connection timed out"))
|
||||
|
||||
with patch("artemis.agents.entity_extractor.entity_extractor.get_llm") as mock_get_llm:
|
||||
mock_get_llm.return_value.with_structured_output.return_value = mock_llm
|
||||
result = await extract_entity(
|
||||
request="Settings",
|
||||
data="com.android.settings",
|
||||
ctx=mock_ctx,
|
||||
)
|
||||
|
||||
assert result.found is False
|
||||
assert result.output is None
|
||||
assert "LLM extraction error" in result.reason
|
||||
@@ -26,26 +26,7 @@ from artemis.utils.logger import get_logger # noqa: E402
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class _CyFunctionDetectorMeta(type):
|
||||
def __instancecheck__(self, instance):
|
||||
name = type(instance).__name__
|
||||
return (
|
||||
name
|
||||
in (
|
||||
"cyfunction",
|
||||
"cython_function_or_method",
|
||||
"builtin_function_or_method",
|
||||
)
|
||||
or "cyfunction" in name.lower()
|
||||
)
|
||||
|
||||
|
||||
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
|
||||
pass
|
||||
|
||||
|
||||
class MockPydanticSchema(BaseModel):
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
color: str
|
||||
price: float
|
||||
currency_symbol: str
|
||||
|
||||
@@ -9,6 +9,8 @@ from mcp_server.utils import device_utils
|
||||
def test_ensure_emulator_uses_windows_creation_flags(monkeypatch) -> None:
|
||||
popen = MagicMock()
|
||||
monkeypatch.setattr(device_utils.sys, "platform", "win32")
|
||||
monkeypatch.setattr(device_utils.subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200, raising=False)
|
||||
monkeypatch.setattr(device_utils.subprocess, "DETACHED_PROCESS", 0x00000008, raising=False)
|
||||
monkeypatch.setattr(device_utils, "is_emulator_running", lambda _adb: False)
|
||||
monkeypatch.setattr(device_utils.os.path, "exists", lambda _path: True)
|
||||
monkeypatch.setattr(device_utils.subprocess, "Popen", popen)
|
||||
@@ -20,7 +22,7 @@ def test_ensure_emulator_uses_windows_creation_flags(monkeypatch) -> None:
|
||||
monkeypatch.setattr(device_utils.time, "sleep", lambda _seconds: None)
|
||||
|
||||
assert device_utils.ensure_emulator(
|
||||
adb_path="adb.exe", emulator_path="emulator.exe", timeout_seconds=1
|
||||
adb_path="adb.exe", emulator_path="emulator.exe", timeout_seconds=10
|
||||
)
|
||||
|
||||
kwargs = popen.call_args.kwargs
|
||||
@@ -43,7 +45,7 @@ def test_ensure_emulator_starts_new_session_on_posix(monkeypatch) -> None:
|
||||
)
|
||||
monkeypatch.setattr(device_utils.time, "sleep", lambda _seconds: None)
|
||||
|
||||
assert device_utils.ensure_emulator(adb_path="adb", emulator_path="emulator", timeout_seconds=1)
|
||||
assert device_utils.ensure_emulator(adb_path="adb", emulator_path="emulator", timeout_seconds=10)
|
||||
|
||||
kwargs = popen.call_args.kwargs
|
||||
assert kwargs["start_new_session"] is True
|
||||
|
||||
@@ -24,11 +24,8 @@ import queue
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.clients import ui_automator_client
|
||||
from artemis.runtime.awake_lease import ScreenAwakeLease
|
||||
from artemis.runtime.awake_service import _run_awake_adb_command
|
||||
@@ -204,16 +201,12 @@ def test_device_utils_isolates_stdin():
|
||||
|
||||
|
||||
def test_ui_automator_client_isolates_stdin():
|
||||
"""Verify ui_automator_client helper commands isolate stdin."""
|
||||
"""Verify ui_automator_client screencap commands isolate stdin."""
|
||||
client = ui_automator_client.UIAutomatorClient("dev-1")
|
||||
with patch("artemis.clients.ui_automator_client.subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout="package:com.test\n", stderr="")
|
||||
ui_automator_client._is_package_installed("dev-1", "com.test")
|
||||
assert mock_run.called
|
||||
assert mock_run.call_args.kwargs.get("stdin") == subprocess.DEVNULL
|
||||
|
||||
with patch("artemis.clients.ui_automator_client.subprocess.run") as mock_run:
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
|
||||
ui_automator_client._uninstall_package("dev-1", "com.test")
|
||||
mock_run.return_value = MagicMock(returncode=0, stdout=b"", stderr="")
|
||||
with patch("artemis.clients.ui_automator_client.Image.open"):
|
||||
client.get_screenshot()
|
||||
assert mock_run.called
|
||||
assert mock_run.call_args.kwargs.get("stdin") == subprocess.DEVNULL
|
||||
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests validating Artemis SDK exception hierarchy and diagnostics."""
|
||||
|
||||
from artemis.sdk.types.exceptions import (
|
||||
AgentError,
|
||||
AgentNotInitializedError,
|
||||
AgentProfileNotFoundError,
|
||||
AgentTaskRequestError,
|
||||
ArtemisError,
|
||||
DeviceError,
|
||||
DeviceNotFoundError,
|
||||
ExecutableNotFoundError,
|
||||
ServerError,
|
||||
ServerStartupError,
|
||||
)
|
||||
|
||||
|
||||
def test_exception_hierarchy():
|
||||
"""Verify all custom exceptions inherit properly from ArtemisError."""
|
||||
assert issubclass(DeviceError, ArtemisError)
|
||||
assert issubclass(DeviceNotFoundError, DeviceError)
|
||||
assert issubclass(ServerError, ArtemisError)
|
||||
assert issubclass(ServerStartupError, ServerError)
|
||||
assert issubclass(AgentError, ArtemisError)
|
||||
assert issubclass(AgentNotInitializedError, AgentError)
|
||||
assert issubclass(AgentTaskRequestError, AgentError)
|
||||
assert issubclass(AgentProfileNotFoundError, AgentTaskRequestError)
|
||||
assert issubclass(ExecutableNotFoundError, ArtemisError)
|
||||
|
||||
|
||||
def test_executable_not_found_error_adb_guidance():
|
||||
"""Verify executable error formats download instructions for known tools."""
|
||||
err = ExecutableNotFoundError("adb")
|
||||
assert "adb" in str(err)
|
||||
assert "developer.android.com" in str(err)
|
||||
assert err.executable_name == "adb"
|
||||
|
||||
|
||||
def test_server_startup_error_formatting():
|
||||
"""Verify server startup error encapsulates service identifier."""
|
||||
err = ServerStartupError(server_name="mcp_daemon")
|
||||
assert "mcp_daemon" in str(err)
|
||||
assert err.server_name == "mcp_daemon"
|
||||
|
||||
|
||||
def test_agent_profile_not_found_error():
|
||||
"""Verify profile error retains profile name."""
|
||||
err = AgentProfileNotFoundError("nonexistent_profile")
|
||||
assert "nonexistent_profile" in str(err)
|
||||
assert err.profile_name == "nonexistent_profile"
|
||||
@@ -0,0 +1,111 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for Artemis SDK TaskRequestBuilder."""
|
||||
|
||||
from pathlib import Path
|
||||
from pydantic import BaseModel
|
||||
import pytest
|
||||
|
||||
from artemis.sdk.builders.task_request_builder import TaskRequestBuilder, TaskRequestCommonBuilder
|
||||
from artemis.sdk.types.task import TaskRequest
|
||||
|
||||
|
||||
class DummyOutputSchema(BaseModel):
|
||||
summary: str
|
||||
item_count: int
|
||||
|
||||
|
||||
def test_task_request_builder_fluent_chaining():
|
||||
"""Verify basic fluent configuration and build."""
|
||||
req = (
|
||||
TaskRequestBuilder(goal="Open Settings and verify battery level")
|
||||
.with_max_steps(25)
|
||||
.with_trace_recording(True, path="custom_traces")
|
||||
.with_locked_app_package("com.android.settings")
|
||||
.with_app_path("build/app.apk")
|
||||
.using_profile("flash")
|
||||
.with_name("Battery Check")
|
||||
.build()
|
||||
)
|
||||
|
||||
assert isinstance(req, TaskRequest)
|
||||
assert req.goal == "Open Settings and verify battery level"
|
||||
assert req.max_steps == 25
|
||||
assert req.record_trace is True
|
||||
assert req.trace_path == Path("custom_traces")
|
||||
assert req.locked_app_package == "com.android.settings"
|
||||
assert req.app_path == Path("build/app.apk")
|
||||
assert req.profile == "flash"
|
||||
assert req.task_name == "Battery Check"
|
||||
|
||||
|
||||
def test_task_request_builder_with_output_format():
|
||||
"""Verify builder binds structured schema properly."""
|
||||
req = (
|
||||
TaskRequestBuilder(goal="Extract items")
|
||||
.with_output_format(DummyOutputSchema)
|
||||
.build()
|
||||
)
|
||||
|
||||
assert req.output_format is DummyOutputSchema
|
||||
assert req.output_description is None
|
||||
|
||||
|
||||
def test_task_request_builder_with_output_description():
|
||||
"""Verify builder accepts unformatted string deliverables."""
|
||||
req = (
|
||||
TaskRequestBuilder(goal="Extract items")
|
||||
.with_output_description("Return a numbered list")
|
||||
.build()
|
||||
)
|
||||
|
||||
assert req.output_description == "Return a numbered list"
|
||||
assert req.output_format is None
|
||||
|
||||
|
||||
def test_task_request_builder_conflicting_outputs_raises():
|
||||
"""Verify that specifying both output_format and output_description raises ValueError."""
|
||||
builder = (
|
||||
TaskRequestBuilder(goal="Extract items")
|
||||
.with_output_format(DummyOutputSchema)
|
||||
.with_output_description("Description text")
|
||||
)
|
||||
with pytest.raises(ValueError, match="Cannot specify both output_format and output_description"):
|
||||
builder.build()
|
||||
|
||||
|
||||
def test_task_request_builder_empty_goal_raises():
|
||||
"""Verify empty or whitespace-only goal raises ValueError."""
|
||||
with pytest.raises(ValueError, match="Task goal must be a non-empty"):
|
||||
TaskRequestBuilder(goal=" ").build()
|
||||
|
||||
|
||||
def test_task_request_builder_from_common():
|
||||
"""Verify task request initialization derived from common base settings."""
|
||||
common_settings = (
|
||||
TaskRequestCommonBuilder()
|
||||
.with_max_steps(42)
|
||||
.with_trace_recording(False)
|
||||
.with_locked_app_package("org.artemis.target")
|
||||
.with_app_path(Path("/tmp/target.apk"))
|
||||
.build()
|
||||
)
|
||||
|
||||
derived = TaskRequestBuilder.from_common("Derived goal", common_settings).build()
|
||||
assert derived.goal == "Derived goal"
|
||||
assert derived.max_steps == 42
|
||||
assert derived.record_trace is False
|
||||
assert derived.locked_app_package == "org.artemis.target"
|
||||
assert derived.app_path == Path("/tmp/target.apk")
|
||||
@@ -154,7 +154,7 @@ def test_llm_config_parsing_and_merging():
|
||||
assert isinstance(llm_cfg, LLMConfig)
|
||||
assert llm_cfg.planner.provider in ("google", "openai", "openrouter", "xai", "vertexai")
|
||||
assert llm_cfg.get_agent("planner") is not None
|
||||
assert llm_cfg.get_utils("hopper") is not None
|
||||
assert llm_cfg.get_utils("entity_extractor") is not None
|
||||
|
||||
# Deep merge overrides
|
||||
overrides = {
|
||||
|
||||
@@ -11,16 +11,14 @@ from artemis.drivers.android.adb_driver import AndroidAdbDriver
|
||||
"artemis.clients.ui_automator_client.ensure_device_awake",
|
||||
return_value="host_heartbeat",
|
||||
)
|
||||
@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed")
|
||||
def test_new_ui_connection_enrolls_device_in_shared_awake_strategy(
|
||||
mock_remove_maestro, mock_ensure_awake, mock_connect
|
||||
mock_ensure_awake, mock_connect
|
||||
):
|
||||
client = UIAutomatorClient("device-123")
|
||||
|
||||
client.connect()
|
||||
client.connect()
|
||||
|
||||
mock_remove_maestro.assert_called_once_with("device-123")
|
||||
mock_ensure_awake.assert_called_once_with("device-123")
|
||||
mock_connect.assert_called_once_with("device-123")
|
||||
|
||||
@@ -30,9 +28,8 @@ def test_new_ui_connection_enrolls_device_in_shared_awake_strategy(
|
||||
"artemis.clients.ui_automator_client.ensure_device_awake",
|
||||
return_value="host_heartbeat",
|
||||
)
|
||||
@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed")
|
||||
def test_failed_ui_connection_does_not_stop_process_awake_service(
|
||||
_mock_remove_maestro, mock_ensure_awake, _mock_connect
|
||||
mock_ensure_awake, _mock_connect
|
||||
):
|
||||
client = UIAutomatorClient("device-123")
|
||||
|
||||
@@ -48,9 +45,8 @@ def test_failed_ui_connection_does_not_stop_process_awake_service(
|
||||
"artemis.clients.ui_automator_client.ensure_device_awake",
|
||||
return_value="host_heartbeat",
|
||||
)
|
||||
@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed")
|
||||
def test_client_disconnect_does_not_send_power_cleanup_commands(
|
||||
_mock_remove_maestro, mock_ensure_awake, _mock_connect
|
||||
mock_ensure_awake, _mock_connect
|
||||
):
|
||||
client = UIAutomatorClient("device-123")
|
||||
client.connect()
|
||||
|
||||
@@ -13,10 +13,11 @@
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
from artemis.agents.hopper.hopper import HopperOutput
|
||||
import pytest
|
||||
|
||||
from artemis.agents.entity_extractor.entity_extractor import ExtractionResult
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.tools.mobile.launch_app import find_package
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -28,63 +29,63 @@ def mock_context():
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.tools.mobile.launch_app.list_packages_async")
|
||||
@patch("artemis.tools.mobile.launch_app.hopper")
|
||||
async def test_find_package_cache_hit(mock_hopper, mock_list_packages, mock_context):
|
||||
@patch("artemis.tools.mobile.launch_app.extract_entity")
|
||||
async def test_find_package_cache_hit(mock_extract_entity, mock_list_packages, mock_context):
|
||||
mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
|
||||
|
||||
# First call: not in cache, calls hopper
|
||||
mock_hopper.return_value = HopperOutput(
|
||||
# First call: not in cache, calls extract_entity
|
||||
mock_extract_entity.return_value = ExtractionResult(
|
||||
found=True, output="com.example.app1", reason="Found it"
|
||||
)
|
||||
|
||||
pkg1 = await find_package(mock_context, "App 1")
|
||||
assert pkg1 == "com.example.app1"
|
||||
assert mock_hopper.call_count == 1
|
||||
assert mock_extract_entity.call_count == 1
|
||||
assert mock_context.package_cache["App 1"] == "com.example.app1"
|
||||
|
||||
# Second call: cache hit, does not call hopper or list_packages again
|
||||
mock_hopper.reset_mock()
|
||||
# Second call: cache hit, does not call extract_entity or list_packages again
|
||||
mock_extract_entity.reset_mock()
|
||||
mock_list_packages.reset_mock()
|
||||
|
||||
pkg2 = await find_package(mock_context, "App 1")
|
||||
assert pkg2 == "com.example.app1"
|
||||
mock_hopper.assert_not_called()
|
||||
mock_extract_entity.assert_not_called()
|
||||
mock_list_packages.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.tools.mobile.launch_app.list_packages_async")
|
||||
@patch("artemis.tools.mobile.launch_app.hopper")
|
||||
async def test_find_package_not_found_caches_none(mock_hopper, mock_list_packages, mock_context):
|
||||
@patch("artemis.tools.mobile.launch_app.extract_entity")
|
||||
async def test_find_package_not_found_caches_none(mock_extract_entity, mock_list_packages, mock_context):
|
||||
mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
|
||||
|
||||
# Hopper doesn't find it
|
||||
mock_hopper.return_value = HopperOutput(found=False, output=None, reason="Not found")
|
||||
# Extractor doesn't find it
|
||||
mock_extract_entity.return_value = ExtractionResult(found=False, output=None, reason="Not found")
|
||||
|
||||
pkg1 = await find_package(mock_context, "App 3")
|
||||
assert pkg1 is None
|
||||
assert mock_hopper.call_count == 1
|
||||
assert mock_extract_entity.call_count == 1
|
||||
assert mock_context.package_cache["App 3"] is None
|
||||
|
||||
# Second call: cache hit (None), does not call hopper again
|
||||
mock_hopper.reset_mock()
|
||||
# Second call: cache hit (None), does not call extract_entity again
|
||||
mock_extract_entity.reset_mock()
|
||||
pkg2 = await find_package(mock_context, "App 3")
|
||||
assert pkg2 is None
|
||||
mock_hopper.assert_not_called()
|
||||
mock_extract_entity.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.tools.mobile.launch_app.list_packages_async")
|
||||
@patch("artemis.tools.mobile.launch_app.hopper")
|
||||
async def test_find_package_passes_use_fallback(mock_hopper, mock_list_packages, mock_context):
|
||||
@patch("artemis.tools.mobile.launch_app.extract_entity")
|
||||
async def test_find_package_passes_use_fallback(mock_extract_entity, mock_list_packages, mock_context):
|
||||
mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
|
||||
mock_hopper.return_value = HopperOutput(
|
||||
mock_extract_entity.return_value = ExtractionResult(
|
||||
found=True, output="com.example.app1", reason="Found it"
|
||||
)
|
||||
|
||||
# Pass use_fallback=False
|
||||
await find_package(mock_context, "App 1", use_fallback=False)
|
||||
mock_hopper.assert_called_once_with(
|
||||
mock_extract_entity.assert_called_once_with(
|
||||
ctx=mock_context,
|
||||
request=("I'm looking for the package name of the following app: 'App 1'"),
|
||||
data="com.example.app1\ncom.example.app2",
|
||||
|
||||
+248
-354
@@ -12,446 +12,340 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from unittest.mock import Mock, patch
|
||||
"""Unit tests for ARTEMIS tool utility functions and element interaction helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# Mock the problematic langgraph import at module level
|
||||
_orig_chat_agent_executor = sys.modules.get("langgraph.prebuilt.chat_agent_executor")
|
||||
_orig_state = sys.modules.get("artemis.graph.state")
|
||||
|
||||
sys.modules["langgraph.prebuilt.chat_agent_executor"] = Mock()
|
||||
sys.modules["artemis.graph.state"] = Mock()
|
||||
|
||||
from artemis.context import DeviceContext, DevicePlatform, ArtemisContext # noqa: E402
|
||||
from artemis.tools.types import Target # noqa: E402
|
||||
from artemis.tools.utils import ( # noqa: E402
|
||||
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
|
||||
from artemis.controllers.types import CoordinatesSelectorRequest
|
||||
from artemis.tools.types import Target
|
||||
from artemis.tools.utils import (
|
||||
IdSelectorRequest,
|
||||
IdWithTextSelectorRequest,
|
||||
SelectorRequestWithCoordinates,
|
||||
SelectorRequestWithPercentages,
|
||||
TextSelectorRequest,
|
||||
_extract_resource_id_and_text_from_selector,
|
||||
find_element_by_text,
|
||||
focus_element_if_needed,
|
||||
has_valid_selectors,
|
||||
move_cursor_to_end_if_bounds,
|
||||
tap_bottom_right_of_element,
|
||||
validate_coordinates_bounds,
|
||||
)
|
||||
from artemis.utils.ui_hierarchy import ElementBounds # noqa: E402
|
||||
|
||||
# Restore original modules to prevent contaminating other tests
|
||||
if _orig_chat_agent_executor is not None:
|
||||
sys.modules["langgraph.prebuilt.chat_agent_executor"] = _orig_chat_agent_executor
|
||||
else:
|
||||
del sys.modules["langgraph.prebuilt.chat_agent_executor"]
|
||||
|
||||
if _orig_state is not None:
|
||||
sys.modules["artemis.graph.state"] = _orig_state
|
||||
else:
|
||||
del sys.modules["artemis.graph.state"]
|
||||
from artemis.utils.ui_hierarchy import ElementBounds
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context():
|
||||
"""Create a mock ArtemisContext for testing."""
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
|
||||
# Create device context with necessary attributes
|
||||
ctx.device = Mock(spec=DeviceContext)
|
||||
def dummy_context() -> MagicMock:
|
||||
"""Fixture providing an ARTEMIS runtime context configured for Android."""
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.device = MagicMock(spec=DeviceContext)
|
||||
ctx.device.mobile_platform = DevicePlatform.ANDROID
|
||||
ctx.device.device_id = "test_device_123"
|
||||
ctx.device.device_id = "test-device-pixel8"
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2340
|
||||
ctx.device.host_platform = "LINUX"
|
||||
|
||||
ctx.ui_adb_client = Mock()
|
||||
|
||||
# Mock the ADB client for Android
|
||||
ctx.adb_client = Mock()
|
||||
mock_device = Mock()
|
||||
mock_device.shell = Mock(return_value="")
|
||||
ctx.adb_client.device = Mock(return_value=mock_device)
|
||||
|
||||
# Mock the ADB client for Android
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {"elements": []}
|
||||
ctx.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
ctx.ui_adb_client.get_hierarchy = Mock(
|
||||
return_value=(
|
||||
b'<hierarchy><node bounds="[0,0][1000,1000]"><node'
|
||||
b' bounds="[0,0][100,100]"/></node></hierarchy>'
|
||||
)
|
||||
)
|
||||
|
||||
ctx.device.device_height = 2400
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_state():
|
||||
"""Create a mock State for testing."""
|
||||
state = Mock()
|
||||
def mock_graph_state() -> MagicMock:
|
||||
"""Fixture providing a mock LangGraph state container."""
|
||||
state = MagicMock()
|
||||
state.latest_ui_hierarchy = []
|
||||
return state
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_element():
|
||||
"""Create a sample UI element for testing."""
|
||||
def sample_hierarchy_node() -> dict:
|
||||
"""Fixture providing a standard flat hierarchy node."""
|
||||
return {
|
||||
"resourceId": "com.example:id/text_input",
|
||||
"text": "Sample text",
|
||||
"bounds": {"x": 100, "y": 200, "width": 300, "height": 50},
|
||||
"resourceId": "com.google.android.apps:id/search_query",
|
||||
"text": "Search Google",
|
||||
"bounds": {"x": 120, "y": 250, "width": 400, "height": 80},
|
||||
"focused": "false",
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_rich_element():
|
||||
"""Create a sample rich UI element for testing."""
|
||||
def sample_nested_node() -> dict:
|
||||
"""Fixture providing a rich nested hierarchy node."""
|
||||
return {
|
||||
"attributes": {
|
||||
"resource-id": "com.example:id/text_input",
|
||||
"resource-id": "com.google.android.apps:id/search_query",
|
||||
"text": "Search Google",
|
||||
"bounds": {"x": 120, "y": 250, "width": 400, "height": 80},
|
||||
"focused": "false",
|
||||
"text": "Sample text",
|
||||
"bounds": {"x": 100, "y": 200, "width": 300, "height": 50},
|
||||
},
|
||||
"children": [],
|
||||
}
|
||||
|
||||
|
||||
class TestMoveCursorToEndIfBounds:
|
||||
"""Test cases for move_cursor_to_end_if_bounds function."""
|
||||
# ==============================================================================
|
||||
# Cursor Placement & Boundary Alignment Tests
|
||||
# ==============================================================================
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
|
||||
class TestCursorAlignment:
|
||||
"""Validates precise cursor placement against interactive targets."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.tools.utils.tap", new_callable=AsyncMock)
|
||||
@patch("artemis.tools.utils.find_element_by_resource_id")
|
||||
def test_move_cursor_with_resource_id(
|
||||
async def test_cursor_routes_to_end_via_resource_id(
|
||||
self,
|
||||
mock_find_element,
|
||||
mock_tap,
|
||||
mock_context,
|
||||
mock_state,
|
||||
sample_element,
|
||||
mock_find_elem: MagicMock,
|
||||
mock_tap_fn: AsyncMock,
|
||||
dummy_context: MagicMock,
|
||||
mock_graph_state: MagicMock,
|
||||
sample_hierarchy_node: dict,
|
||||
):
|
||||
"""Test moving cursor using resource_id (highest priority)."""
|
||||
mock_state.latest_ui_hierarchy = [sample_element]
|
||||
mock_find_element.return_value = sample_element
|
||||
"""Resource ID locator should calculate end-boundary tap (99% width/height offset)."""
|
||||
mock_graph_state.latest_ui_hierarchy = [sample_hierarchy_node]
|
||||
mock_find_elem.return_value = sample_hierarchy_node
|
||||
|
||||
target = Target(
|
||||
resource_id="com.example:id/text_input",
|
||||
resource_id_index=None,
|
||||
text=None,
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(
|
||||
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
|
||||
target = Target(resource_id="com.google.android.apps:id/search_query")
|
||||
elem = await move_cursor_to_end_if_bounds(
|
||||
ctx=dummy_context, state=mock_graph_state, target=target
|
||||
)
|
||||
|
||||
mock_find_element.assert_called_once_with(
|
||||
ui_hierarchy=[sample_element],
|
||||
resource_id="com.example:id/text_input",
|
||||
mock_find_elem.assert_called_once_with(
|
||||
ui_hierarchy=[sample_hierarchy_node],
|
||||
resource_id="com.google.android.apps:id/search_query",
|
||||
index=0,
|
||||
)
|
||||
mock_tap.assert_called_once()
|
||||
call_args = mock_tap.call_args[1]
|
||||
selector_request = call_args["selector_request"]
|
||||
assert isinstance(selector_request, SelectorRequestWithCoordinates)
|
||||
coords = selector_request.coordinates
|
||||
assert coords.x == 397 # 100 + 300 * 0.99
|
||||
assert coords.y == 249 # 200 + 50 * 0.99
|
||||
assert result == sample_element
|
||||
mock_tap_fn.assert_awaited_once()
|
||||
kwargs = mock_tap_fn.await_args.kwargs
|
||||
selector_req = kwargs["selector_request"]
|
||||
assert isinstance(selector_req, SelectorRequestWithCoordinates)
|
||||
assert selector_req.coordinates.x == 516
|
||||
assert selector_req.coordinates.y == 329
|
||||
assert elem == sample_hierarchy_node
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.tools.utils.tap", new_callable=AsyncMock)
|
||||
@patch("artemis.tools.utils.find_element_by_resource_id")
|
||||
def test_move_cursor_with_coordinates_only(
|
||||
self, mock_find_element, mock_tap, mock_context, mock_state
|
||||
async def test_cursor_routes_using_direct_bounds(
|
||||
self,
|
||||
mock_find_elem: MagicMock,
|
||||
mock_tap_fn: AsyncMock,
|
||||
dummy_context: MagicMock,
|
||||
mock_graph_state: MagicMock,
|
||||
):
|
||||
"""Test moving cursor when only coordinates are provided."""
|
||||
bounds = ElementBounds(x=50, y=150, width=200, height=40)
|
||||
target = Target(
|
||||
resource_id=None,
|
||||
resource_id_index=None,
|
||||
text=None,
|
||||
text_index=None,
|
||||
bounds=bounds,
|
||||
"""Direct bounding box coordinates should bypass hierarchy inspection."""
|
||||
bounds = ElementBounds(x=80, y=100, width=200, height=60)
|
||||
target = Target(bounds=bounds)
|
||||
|
||||
result = await move_cursor_to_end_if_bounds(
|
||||
ctx=dummy_context, state=mock_graph_state, target=target
|
||||
)
|
||||
|
||||
result = asyncio.run(
|
||||
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
|
||||
)
|
||||
mock_find_elem.assert_not_called()
|
||||
mock_tap_fn.assert_awaited_once()
|
||||
kwargs = mock_tap_fn.await_args.kwargs
|
||||
selector_req = kwargs["selector_request"]
|
||||
assert selector_req.coordinates.x == 278
|
||||
assert selector_req.coordinates.y == 159
|
||||
assert result is None
|
||||
|
||||
mock_find_element.assert_not_called()
|
||||
mock_tap.assert_called_once()
|
||||
call_args = mock_tap.call_args[1]
|
||||
selector_request = call_args["selector_request"]
|
||||
coords = selector_request.coordinates
|
||||
assert coords.x == 248 # 50 + 200 * 0.99
|
||||
assert coords.y == 189 # 150 + 40 * 0.99
|
||||
assert result is None # No element is returned when using coords directly
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.tools.utils.tap", new_callable=AsyncMock)
|
||||
@patch("artemis.tools.utils.find_element_by_text")
|
||||
def test_move_cursor_with_text_only_success(
|
||||
self, mock_find_text, mock_tap, mock_context, mock_state, sample_element
|
||||
async def test_cursor_routes_by_matched_text(
|
||||
self,
|
||||
mock_find_text: MagicMock,
|
||||
mock_tap_fn: AsyncMock,
|
||||
dummy_context: MagicMock,
|
||||
mock_graph_state: MagicMock,
|
||||
sample_hierarchy_node: dict,
|
||||
):
|
||||
"""Test moving cursor when only text is provided and succeeds."""
|
||||
mock_state.latest_ui_hierarchy = [sample_element]
|
||||
mock_find_text.return_value = sample_element
|
||||
"""Visible text locator should successfully resolve bounds and trigger cursor placement."""
|
||||
mock_graph_state.latest_ui_hierarchy = [sample_hierarchy_node]
|
||||
mock_find_text.return_value = sample_hierarchy_node
|
||||
|
||||
target = Target(
|
||||
resource_id=None,
|
||||
resource_id_index=None,
|
||||
text="Sample text",
|
||||
text_index=0,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(
|
||||
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
|
||||
target = Target(text="Search Google", text_index=0)
|
||||
result = await move_cursor_to_end_if_bounds(
|
||||
ctx=dummy_context, state=mock_graph_state, target=target
|
||||
)
|
||||
|
||||
mock_find_text.assert_called_once_with([sample_element], "Sample text", index=0)
|
||||
mock_tap.assert_called_once()
|
||||
assert result == sample_element
|
||||
mock_find_text.assert_called_once_with([sample_hierarchy_node], "Search Google", index=0)
|
||||
mock_tap_fn.assert_awaited_once()
|
||||
assert result == sample_hierarchy_node
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.tools.utils.tap", new_callable=AsyncMock)
|
||||
@patch("artemis.tools.utils.find_element_by_text")
|
||||
def test_move_cursor_with_text_only_element_not_found(
|
||||
self, mock_find_text, mock_tap, mock_context, mock_state
|
||||
async def test_cursor_skips_when_text_unmatched(
|
||||
self,
|
||||
mock_find_text: MagicMock,
|
||||
mock_tap_fn: AsyncMock,
|
||||
dummy_context: MagicMock,
|
||||
mock_graph_state: MagicMock,
|
||||
):
|
||||
"""Test when searching by text finds no element."""
|
||||
mock_state.latest_ui_hierarchy = []
|
||||
"""Unresolved text target returns None without tapping."""
|
||||
mock_find_text.return_value = None
|
||||
target = Target(text="Unmatched Query")
|
||||
|
||||
target = Target(
|
||||
resource_id=None,
|
||||
resource_id_index=None,
|
||||
text="Nonexistent text",
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
result = await move_cursor_to_end_if_bounds(
|
||||
ctx=dummy_context, state=mock_graph_state, target=target
|
||||
)
|
||||
result = asyncio.run(
|
||||
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
|
||||
)
|
||||
|
||||
mock_tap.assert_not_called()
|
||||
assert result is None
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@patch("artemis.tools.utils.find_element_by_text")
|
||||
def test_move_cursor_with_text_only_no_bounds(
|
||||
self, mock_find_text, mock_tap, mock_context, mock_state
|
||||
):
|
||||
"""Test when element is found by text but has no bounds."""
|
||||
element_no_bounds = {"text": "Text without bounds"}
|
||||
mock_state.latest_ui_hierarchy = [element_no_bounds]
|
||||
mock_find_text.return_value = element_no_bounds
|
||||
|
||||
target = Target(
|
||||
resource_id=None,
|
||||
resource_id_index=None,
|
||||
text="Text without bounds",
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(
|
||||
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
|
||||
)
|
||||
|
||||
mock_tap.assert_not_called()
|
||||
assert result is None # Should return None as no action was taken
|
||||
|
||||
@patch("artemis.tools.utils.find_element_by_resource_id")
|
||||
def test_move_cursor_element_not_found_by_id(self, mock_find_element, mock_context, mock_state):
|
||||
"""Test when element is not found by resource_id."""
|
||||
mock_find_element.return_value = None
|
||||
|
||||
target = Target(
|
||||
resource_id="com.example:id/nonexistent",
|
||||
resource_id_index=None,
|
||||
text=None,
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(
|
||||
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
|
||||
)
|
||||
|
||||
mock_tap_fn.assert_not_awaited()
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestFocusElementIfNeeded:
|
||||
"""Test cases for focus_element_if_needed function."""
|
||||
# ==============================================================================
|
||||
# Element Focus Resolution Tests
|
||||
# ==============================================================================
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@patch("artemis.tools.utils.find_element_by_resource_id")
|
||||
def test_focus_element_already_focused(
|
||||
self, mock_find_element, mock_tap, mock_context, sample_rich_element
|
||||
|
||||
class TestElementFocusing:
|
||||
"""Validates conditional focusing behavior prior to keyboard inputs."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.tools.utils.tap", new_callable=AsyncMock)
|
||||
@patch("artemis.tools.utils.UnifiedMobileController")
|
||||
async def test_skips_tap_when_already_focused(
|
||||
self,
|
||||
mock_controller_cls: MagicMock,
|
||||
mock_tap_fn: AsyncMock,
|
||||
dummy_context: MagicMock,
|
||||
sample_hierarchy_node: dict,
|
||||
):
|
||||
"""Test when element is already focused."""
|
||||
focused_element = sample_rich_element.copy()
|
||||
focused_element["attributes"]["focused"] = "true"
|
||||
"""Elements reporting focused state true should avoid redundant tap interactions."""
|
||||
focused_elem = sample_hierarchy_node.copy()
|
||||
focused_elem["focused"] = "true"
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {"elements": [focused_element]}
|
||||
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
mock_find_element.return_value = focused_element["attributes"]
|
||||
mock_ctrl = MagicMock()
|
||||
mock_ctrl.get_ui_elements = AsyncMock(return_value=[focused_elem])
|
||||
mock_controller_cls.return_value = mock_ctrl
|
||||
|
||||
target = Target(
|
||||
resource_id="com.example:id/text_input",
|
||||
resource_id_index=None,
|
||||
text=None,
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
|
||||
target = Target(resource_id="com.google.android.apps:id/search_query")
|
||||
focus_strategy = await focus_element_if_needed(ctx=dummy_context, target=target)
|
||||
|
||||
mock_tap.assert_not_called()
|
||||
assert result == "resource_id"
|
||||
mock_context.ui_adb_client.get_hierarchy.assert_called_once()
|
||||
mock_tap_fn.assert_not_awaited()
|
||||
assert focus_strategy == "resource_id"
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@patch("artemis.tools.utils.find_element_by_resource_id")
|
||||
def test_focus_element_needs_focus_success(
|
||||
self, mock_find_element, mock_tap, mock_context, sample_rich_element
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.tools.utils.tap", new_callable=AsyncMock)
|
||||
@patch("artemis.tools.utils.UnifiedMobileController")
|
||||
async def test_focus_executes_tap_for_unfocused_element(
|
||||
self,
|
||||
mock_controller_cls: MagicMock,
|
||||
mock_tap_fn: AsyncMock,
|
||||
dummy_context: MagicMock,
|
||||
sample_hierarchy_node: dict,
|
||||
):
|
||||
"""Test when element needs focus and focusing succeeds."""
|
||||
unfocused_element = sample_rich_element
|
||||
focused_element = {
|
||||
"attributes": {
|
||||
"resource-id": "com.example:id/text_input",
|
||||
"focused": "true",
|
||||
},
|
||||
"children": [],
|
||||
}
|
||||
"""Unfocused element triggers focus tap and confirms state change."""
|
||||
unfocused = sample_hierarchy_node.copy()
|
||||
unfocused["focused"] = "false"
|
||||
focused = sample_hierarchy_node.copy()
|
||||
focused["focused"] = "true"
|
||||
|
||||
mock_find_element.side_effect = [
|
||||
unfocused_element["attributes"],
|
||||
focused_element["attributes"],
|
||||
]
|
||||
mock_ctrl = MagicMock()
|
||||
mock_ctrl.get_ui_elements = AsyncMock(side_effect=[[unfocused], [focused]])
|
||||
mock_controller_cls.return_value = mock_ctrl
|
||||
|
||||
target = Target(
|
||||
resource_id="com.example:id/text_input",
|
||||
resource_id_index=None,
|
||||
text=None,
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
|
||||
target = Target(resource_id="com.google.android.apps:id/search_query")
|
||||
strategy = await focus_element_if_needed(ctx=dummy_context, target=target)
|
||||
|
||||
mock_tap.assert_called_once_with(
|
||||
ctx=mock_context,
|
||||
selector_request=IdSelectorRequest(id="com.example:id/text_input"),
|
||||
mock_tap_fn.assert_awaited_once_with(
|
||||
ctx=dummy_context,
|
||||
selector_request=IdSelectorRequest(id="com.google.android.apps:id/search_query"),
|
||||
index=0,
|
||||
)
|
||||
assert mock_context.ui_adb_client.get_hierarchy.call_count == 2
|
||||
assert result == "resource_id"
|
||||
assert strategy == "resource_id"
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@patch("artemis.tools.utils.logger")
|
||||
@patch("artemis.tools.utils.find_element_by_resource_id")
|
||||
def test_focus_id_and_text_mismatch_fallback_to_text(
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.tools.utils.tap", new_callable=AsyncMock)
|
||||
@patch("artemis.tools.utils.UnifiedMobileController")
|
||||
async def test_focus_falls_back_to_text_coordinates(
|
||||
self,
|
||||
mock_find_id,
|
||||
mock_logger,
|
||||
mock_tap,
|
||||
mock_context,
|
||||
sample_rich_element,
|
||||
mock_controller_cls: MagicMock,
|
||||
mock_tap_fn: AsyncMock,
|
||||
dummy_context: MagicMock,
|
||||
sample_hierarchy_node: dict,
|
||||
):
|
||||
"""Test fallback when resource_id and text point to different elements."""
|
||||
element_from_id = sample_rich_element["attributes"].copy()
|
||||
element_from_id["text"] = "Different text"
|
||||
"""When ID is unavailable, focusing falls back to text center tap coordinates."""
|
||||
elem = sample_hierarchy_node.copy()
|
||||
elem["bounds"] = {"x": 20, "y": 40, "width": 200, "height": 60}
|
||||
|
||||
element_from_text = sample_rich_element.copy()
|
||||
element_from_text["attributes"]["bounds"] = {
|
||||
"x": 10,
|
||||
"y": 20,
|
||||
"width": 100,
|
||||
"height": 30,
|
||||
}
|
||||
mock_ctrl = MagicMock()
|
||||
mock_ctrl.get_ui_elements = AsyncMock(return_value=[elem])
|
||||
mock_controller_cls.return_value = mock_ctrl
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {"elements": [element_from_text]}
|
||||
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
mock_find_id.return_value = element_from_id
|
||||
target = Target(text="Search Google")
|
||||
strategy = await focus_element_if_needed(ctx=dummy_context, target=target)
|
||||
|
||||
with patch("artemis.tools.utils.find_element_by_text") as mock_find_text:
|
||||
mock_find_text.return_value = element_from_text["attributes"]
|
||||
|
||||
target = Target(
|
||||
resource_id="com.example:id/text_input",
|
||||
resource_id_index=None,
|
||||
text="Sample text",
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
|
||||
|
||||
mock_logger.warning.assert_called_once()
|
||||
mock_tap.assert_called_once()
|
||||
assert result == "text"
|
||||
|
||||
@patch("artemis.tools.utils.tap")
|
||||
@patch("artemis.tools.utils.find_element_by_text")
|
||||
def test_focus_fallback_to_text(
|
||||
self, mock_find_text, mock_tap, mock_context, sample_rich_element
|
||||
):
|
||||
"""Test fallback to focusing using text."""
|
||||
element_with_bounds = sample_rich_element.copy()
|
||||
element_with_bounds["attributes"]["bounds"] = {
|
||||
"x": 10,
|
||||
"y": 20,
|
||||
"width": 100,
|
||||
"height": 30,
|
||||
}
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {"elements": [element_with_bounds]}
|
||||
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
mock_find_text.return_value = element_with_bounds["attributes"]
|
||||
|
||||
target = Target(
|
||||
resource_id=None,
|
||||
resource_id_index=None,
|
||||
text="Sample text",
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
|
||||
|
||||
mock_find_text.assert_called_once()
|
||||
mock_tap.assert_called_once()
|
||||
call_args = mock_tap.call_args[1]
|
||||
selector = call_args["selector_request"]
|
||||
assert isinstance(selector, SelectorRequestWithCoordinates)
|
||||
assert selector.coordinates.x == 60 # 10 + 100/2
|
||||
assert selector.coordinates.y == 35 # 20 + 30/2
|
||||
assert result == "text"
|
||||
mock_tap_fn.assert_awaited_once()
|
||||
req = mock_tap_fn.await_args.kwargs["selector_request"]
|
||||
assert isinstance(req, SelectorRequestWithCoordinates)
|
||||
assert req.coordinates.x == 120 # 20 + 200/2
|
||||
assert req.coordinates.y == 70 # 40 + 60/2
|
||||
assert strategy == "text"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.tools.utils.logger")
|
||||
def test_focus_all_locators_fail(self, mock_logger, mock_context):
|
||||
"""Test failure when no locator can find an element."""
|
||||
@patch("artemis.tools.utils.UnifiedMobileController")
|
||||
async def test_focus_aborts_when_no_locators_resolve(
|
||||
self,
|
||||
mock_controller_cls: MagicMock,
|
||||
mock_logger: MagicMock,
|
||||
dummy_context: MagicMock,
|
||||
):
|
||||
"""When none of the declared locators match an active element, returns None and logs error."""
|
||||
mock_ctrl = MagicMock()
|
||||
mock_ctrl.get_ui_elements = AsyncMock(return_value=[])
|
||||
mock_controller_cls.return_value = mock_ctrl
|
||||
|
||||
mock_response = Mock()
|
||||
mock_response.json.return_value = {"elements": []}
|
||||
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
|
||||
with (
|
||||
patch("artemis.tools.utils.find_element_by_resource_id") as mock_find_id,
|
||||
patch("artemis.tools.utils.find_element_by_text") as mock_find_text,
|
||||
):
|
||||
mock_find_id.return_value = None
|
||||
mock_find_text.return_value = None
|
||||
target = Target(resource_id="unknown_id", text="unknown_text")
|
||||
strategy = await focus_element_if_needed(ctx=dummy_context, target=target)
|
||||
|
||||
target = Target(
|
||||
resource_id="nonexistent",
|
||||
resource_id_index=None,
|
||||
text="nonexistent",
|
||||
text_index=None,
|
||||
bounds=None,
|
||||
)
|
||||
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
|
||||
mock_logger.error.assert_called_once()
|
||||
assert strategy is None
|
||||
|
||||
mock_logger.error.assert_called_once_with(
|
||||
"Failed to focus element."
|
||||
+ " No valid locator (resource_id, coordinates, or text) succeeded."
|
||||
|
||||
# ==============================================================================
|
||||
# Helper Verification & Selector Validation Tests
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
class TestSelectorValidation:
|
||||
"""Verifies target completeness and boundary constraint checks."""
|
||||
|
||||
def test_has_valid_selectors_matrix(self):
|
||||
"""Verifies selector detection across valid and invalid combinations."""
|
||||
assert has_valid_selectors(Target(resource_id="elem_id"))
|
||||
assert has_valid_selectors(Target(text="Label"))
|
||||
assert has_valid_selectors(Target(bounds=ElementBounds(x=0, y=0, width=10, height=10)))
|
||||
assert not has_valid_selectors(Target())
|
||||
|
||||
def test_validate_coordinates_bounds(self):
|
||||
"""Checks bounds containment against physical device display limits."""
|
||||
valid_target = Target(bounds=ElementBounds(x=100, y=200, width=100, height=100))
|
||||
assert validate_coordinates_bounds(valid_target, 1080, 2400) is None
|
||||
|
||||
out_width = Target(bounds=ElementBounds(x=1050, y=200, width=100, height=100))
|
||||
assert validate_coordinates_bounds(out_width, 1080, 2400) is not None
|
||||
|
||||
out_height = Target(bounds=ElementBounds(x=100, y=2380, width=100, height=100))
|
||||
assert validate_coordinates_bounds(out_height, 1080, 2400) is not None
|
||||
|
||||
def test_extract_selector_attributes(self):
|
||||
"""Extracts resource ID and text from heterogeneous selector requests."""
|
||||
req_id = IdSelectorRequest(id="view_item")
|
||||
assert _extract_resource_id_and_text_from_selector(req_id) == ("view_item", None)
|
||||
|
||||
req_text = TextSelectorRequest(text="Confirm")
|
||||
assert _extract_resource_id_and_text_from_selector(req_text) == (None, "Confirm")
|
||||
|
||||
req_both = IdWithTextSelectorRequest(id="submit_btn", text="Submit")
|
||||
assert _extract_resource_id_and_text_from_selector(req_both) == ("submit_btn", "Submit")
|
||||
|
||||
req_coords = SelectorRequestWithCoordinates(
|
||||
coordinates=CoordinatesSelectorRequest(x=50, y=50)
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__])
|
||||
assert _extract_resource_id_and_text_from_selector(req_coords) == (None, None)
|
||||
@@ -11,6 +11,11 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
|
||||
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
|
||||
|
||||
"""Unit tests validating Artemis UI hierarchy parsing and geometry computation."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -29,215 +34,160 @@ def test_text_input_is_empty():
|
||||
assert text_input_is_empty(text=None, hint_text=None)
|
||||
assert text_input_is_empty(text="", hint_text=None)
|
||||
assert text_input_is_empty(text="", hint_text="")
|
||||
assert text_input_is_empty(text="text", hint_text="text")
|
||||
assert text_input_is_empty(text="search", hint_text="search")
|
||||
|
||||
assert not text_input_is_empty(text="text", hint_text=None)
|
||||
assert not text_input_is_empty(text="text", hint_text="")
|
||||
assert not text_input_is_empty(text="search", hint_text=None)
|
||||
assert not text_input_is_empty(text="search", hint_text="")
|
||||
|
||||
|
||||
def test_find_element_by_resource_id():
|
||||
ui_hierarchy = [
|
||||
tree = [
|
||||
{
|
||||
"resourceId": "com.example:id/button1",
|
||||
"text": "Button 1",
|
||||
"resourceId": "com.google.android.settings:id/search_bar",
|
||||
"text": "Search Settings",
|
||||
"children": [],
|
||||
},
|
||||
{
|
||||
"resourceId": "com.example:id/container",
|
||||
"resourceId": "com.google.android.settings:id/content_frame",
|
||||
"children": [
|
||||
{
|
||||
"resourceId": "com.example:id/nested_button",
|
||||
"text": "Nested Button",
|
||||
"resourceId": "com.google.android.settings:id/network_item",
|
||||
"text": "Network & Internet",
|
||||
"children": [],
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button1")
|
||||
assert result is not None
|
||||
assert result["resourceId"] == "com.example:id/button1"
|
||||
assert result["text"] == "Button 1"
|
||||
target = find_element_by_resource_id(tree, "com.google.android.settings:id/search_bar")
|
||||
assert target is not None
|
||||
assert target["resourceId"] == "com.google.android.settings:id/search_bar"
|
||||
assert target["text"] == "Search Settings"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nested_button")
|
||||
assert result is not None
|
||||
assert result["resourceId"] == "com.example:id/nested_button"
|
||||
assert result["text"] == "Nested Button"
|
||||
nested = find_element_by_resource_id(tree, "com.google.android.settings:id/network_item")
|
||||
assert nested is not None
|
||||
assert nested["resourceId"] == "com.google.android.settings:id/network_item"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nonexistent")
|
||||
assert result is None
|
||||
|
||||
result = find_element_by_resource_id([], "com.example:id/button1")
|
||||
assert result is None
|
||||
assert find_element_by_resource_id(tree, "com.google.android.settings:id/missing") is None
|
||||
assert find_element_by_resource_id([], "com.google.android.settings:id/search_bar") is None
|
||||
|
||||
|
||||
def test_find_element_by_resource_id_with_index():
|
||||
ui_hierarchy = [
|
||||
duplicate_items = [
|
||||
{"resourceId": "com.artemis.ui:id/row_item", "text": "Item Alpha", "children": []},
|
||||
{"resourceId": "com.artemis.ui:id/row_item", "text": "Item Beta", "children": []},
|
||||
{
|
||||
"resourceId": "com.example:id/button",
|
||||
"text": "Button 1",
|
||||
"children": [],
|
||||
},
|
||||
{
|
||||
"resourceId": "com.example:id/button",
|
||||
"text": "Button 2",
|
||||
"children": [],
|
||||
},
|
||||
{
|
||||
"resourceId": "com.example:id/container",
|
||||
"resourceId": "com.artemis.ui:id/section_group",
|
||||
"children": [
|
||||
{
|
||||
"resourceId": "com.example:id/button",
|
||||
"text": "Button 3",
|
||||
"children": [],
|
||||
}
|
||||
{"resourceId": "com.artemis.ui:id/row_item", "text": "Item Gamma", "children": []}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=0)
|
||||
assert result is not None
|
||||
assert result["text"] == "Button 1"
|
||||
first = find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=0)
|
||||
assert first is not None
|
||||
assert first["text"] == "Item Alpha"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=1)
|
||||
assert result is not None
|
||||
assert result["text"] == "Button 2"
|
||||
second = find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=1)
|
||||
assert second is not None
|
||||
assert second["text"] == "Item Beta"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=2)
|
||||
assert result is not None
|
||||
assert result["text"] == "Button 3"
|
||||
third = find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=2)
|
||||
assert third is not None
|
||||
assert third["text"] == "Item Gamma"
|
||||
|
||||
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=3)
|
||||
assert result is None
|
||||
assert find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=3) is None
|
||||
|
||||
|
||||
def test_find_element_by_resource_id_rich_hierarchy():
|
||||
rich_hierarchy = [
|
||||
rich_nodes = [
|
||||
{
|
||||
"attributes": {"resource-id": "com.example:id/button1"},
|
||||
"attributes": {"resource-id": "com.android.systemui:id/clock"},
|
||||
"children": [],
|
||||
},
|
||||
{
|
||||
"attributes": {"resource-id": "com.example:id/container"},
|
||||
"attributes": {"resource-id": "com.android.systemui:id/status_bar"},
|
||||
"children": [
|
||||
{
|
||||
"attributes": {"resource-id": "com.example:id/nested_button"},
|
||||
"attributes": {"resource-id": "com.android.systemui:id/battery_meter"},
|
||||
"children": [],
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
result = find_element_by_resource_id(
|
||||
rich_hierarchy, "com.example:id/button1", is_rich_hierarchy=True
|
||||
match = find_element_by_resource_id(
|
||||
rich_nodes, "com.android.systemui:id/clock", is_rich_hierarchy=True
|
||||
)
|
||||
assert result is not None
|
||||
assert result["resource-id"] == "com.example:id/button1"
|
||||
assert match is not None
|
||||
assert match["resource-id"] == "com.android.systemui:id/clock"
|
||||
|
||||
result = find_element_by_resource_id(
|
||||
rich_hierarchy, "com.example:id/nested_button", is_rich_hierarchy=True
|
||||
nested_match = find_element_by_resource_id(
|
||||
rich_nodes, "com.android.systemui:id/battery_meter", is_rich_hierarchy=True
|
||||
)
|
||||
assert result is not None
|
||||
assert result["resource-id"] == "com.example:id/nested_button"
|
||||
assert nested_match is not None
|
||||
assert nested_match["resource-id"] == "com.android.systemui:id/battery_meter"
|
||||
|
||||
result = find_element_by_resource_id(
|
||||
rich_hierarchy, "com.example:id/nonexistent", is_rich_hierarchy=True
|
||||
)
|
||||
assert result is None
|
||||
assert find_element_by_resource_id(
|
||||
rich_nodes, "com.android.systemui:id/unknown", is_rich_hierarchy=True
|
||||
) is None
|
||||
|
||||
|
||||
def test_is_element_focused():
|
||||
focused_element = {"focused": "true"}
|
||||
assert is_element_focused(focused_element)
|
||||
|
||||
non_focused_element = {"focused": "false"}
|
||||
assert not is_element_focused(non_focused_element)
|
||||
|
||||
no_focused_element = {"text": "some text"}
|
||||
assert not is_element_focused(no_focused_element)
|
||||
|
||||
none_focused_element = {"focused": None}
|
||||
assert not is_element_focused(none_focused_element)
|
||||
assert is_element_focused({"focused": "true"})
|
||||
assert is_element_focused({"focused": True})
|
||||
assert not is_element_focused({"focused": "false"})
|
||||
assert not is_element_focused({"focused": False})
|
||||
assert not is_element_focused({"text": "sample"})
|
||||
assert not is_element_focused({"focused": None})
|
||||
|
||||
|
||||
def test_get_element_text():
|
||||
element = {"text": "Button Text", "hintText": "Hint Text"}
|
||||
assert get_element_text(element) == "Button Text"
|
||||
assert get_element_text(element, hint_text=False) == "Button Text"
|
||||
assert get_element_text(element, hint_text=True) == "Hint Text"
|
||||
node = {"text": "Battery", "hintText": "Search settings"}
|
||||
assert get_element_text(node) == "Battery"
|
||||
assert get_element_text(node, hint_text=False) == "Battery"
|
||||
assert get_element_text(node, hint_text=True) == "Search settings"
|
||||
|
||||
element_no_text = {"hintText": "Hint Text"}
|
||||
assert get_element_text(element_no_text) is None
|
||||
assert get_element_text(element_no_text, hint_text=True) == "Hint Text"
|
||||
element_no_hint = {"text": "Button Text"}
|
||||
assert get_element_text(element_no_hint) == "Button Text"
|
||||
assert get_element_text(element_no_hint, hint_text=True) is None
|
||||
node_no_text = {"hintText": "Placeholder"}
|
||||
assert get_element_text(node_no_text) is None
|
||||
assert get_element_text(node_no_text, hint_text=True) == "Placeholder"
|
||||
|
||||
empty_element = {}
|
||||
assert get_element_text(empty_element) is None
|
||||
assert get_element_text(empty_element, hint_text=True) is None
|
||||
assert get_element_text({}) is None
|
||||
|
||||
|
||||
def test_get_bounds_for_element():
|
||||
element_with_bounds = {"bounds": {"x": 10, "y": 20, "width": 100, "height": 50}}
|
||||
bounds = get_bounds_for_element(element_with_bounds)
|
||||
assert bounds is not None
|
||||
assert isinstance(bounds, ElementBounds)
|
||||
assert bounds.x == 10
|
||||
assert bounds.y == 20
|
||||
assert bounds.width == 100
|
||||
assert bounds.height == 50
|
||||
node_with_box = {"bounds": {"x": 24, "y": 48, "width": 400, "height": 80}}
|
||||
box = get_bounds_for_element(node_with_box)
|
||||
assert box is not None
|
||||
assert isinstance(box, ElementBounds)
|
||||
assert box.x == 24
|
||||
assert box.y == 48
|
||||
assert box.width == 400
|
||||
assert box.height == 80
|
||||
|
||||
element_no_bounds = {"text": "Button"}
|
||||
bounds = get_bounds_for_element(element_no_bounds)
|
||||
assert bounds is None
|
||||
assert get_bounds_for_element({"text": "Click"}) is None
|
||||
|
||||
# Suppress logger output for the invalid bounds test case
|
||||
with patch("artemis.utils.ui_hierarchy.logger.error"):
|
||||
element_invalid_bounds = {
|
||||
"bounds": {
|
||||
"x": "invalid", # Should be int
|
||||
"y": 20,
|
||||
"width": 100,
|
||||
"height": 50,
|
||||
}
|
||||
}
|
||||
bounds = get_bounds_for_element(element_invalid_bounds)
|
||||
assert bounds is None
|
||||
bad_box = {"bounds": {"x": "corrupted", "y": 48, "width": 400, "height": 80}}
|
||||
assert get_bounds_for_element(bad_box) is None
|
||||
|
||||
|
||||
def test_element_bounds():
|
||||
bounds = ElementBounds(x=10, y=20, width=100, height=50)
|
||||
def test_element_bounds_geometry():
|
||||
box = ElementBounds(x=100, y=200, width=500, height=100)
|
||||
|
||||
center = bounds.get_center()
|
||||
center = box.get_center()
|
||||
assert isinstance(center, Point)
|
||||
assert center.x == 60
|
||||
assert center.y == 45
|
||||
assert center.x == 350
|
||||
assert center.y == 250
|
||||
|
||||
center_point = bounds.get_relative_point(0.5, 0.5)
|
||||
assert isinstance(center_point, Point)
|
||||
assert center_point.x == 60
|
||||
assert center_point.y == 45
|
||||
mid = box.get_relative_point(0.5, 0.5)
|
||||
assert mid.x == 350
|
||||
assert mid.y == 250
|
||||
|
||||
top_left = bounds.get_relative_point(0.0, 0.0)
|
||||
assert top_left.x == 10
|
||||
assert top_left.y == 20
|
||||
top_left = box.get_relative_point(0.0, 0.0)
|
||||
assert top_left.x == 100
|
||||
assert top_left.y == 200
|
||||
|
||||
bottom_right = bounds.get_relative_point(1.0, 1.0)
|
||||
assert bottom_right.x == 110
|
||||
assert bottom_right.y == 70
|
||||
custom_point = bounds.get_relative_point(0.95, 0.95)
|
||||
assert custom_point.x == 105
|
||||
assert custom_point.y == 67
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_text_input_is_empty()
|
||||
test_find_element_by_resource_id()
|
||||
test_find_element_by_resource_id_with_index()
|
||||
test_find_element_by_resource_id_rich_hierarchy()
|
||||
test_is_element_focused()
|
||||
test_get_element_text()
|
||||
test_get_bounds_for_element()
|
||||
test_element_bounds()
|
||||
print("All tests passed")
|
||||
bottom_right = box.get_relative_point(1.0, 1.0)
|
||||
assert bottom_right.x == 600
|
||||
assert bottom_right.y == 300
|
||||
Reference in new issue
Block a user