refactor: clean up legacy code, rename hopper to entity_extractor, and add third-party notices
- Add NOTICE file and README license references for mobile-use and finalrun-agent - Add derivative attribution headers to 17 files derived from mobile-use - Rename hopper agent to entity_extractor across prompts, tools, and configs - Remove obsolete adb_tunnel, ApiBaseUrl, and legacy CLI utility scripts - Replace legacy examples with native Artemis diagnostic and workflow examples - Update unit and tool tests to validate entity_extractor and SDK builders
This commit is contained in:
somew1nd committed
1 parent
0860788192
commit
54fcef9887
83 files changed
+1996
-3660
No files matched your search
+128
-132
@@ -12,155 +12,151 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""UI hierarchy traversal and element geometry primitives for ARTEMIS.
|
||||
|
||||
Provides structured spatial coordinates, bounding box metrics, and recursive
|
||||
node querying across Android accessibility hierarchies and UIAutomator tree dumps.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from artemis.utils.logger import get_logger
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def __find_element_by_resource_id_in_rich_hierarchy(
|
||||
hierarchy: list[dict], resource_id: str
|
||||
) -> dict | None:
|
||||
"""Retrieves all the sibling elements for a given resource ID from a nested dictionary.
|
||||
|
||||
Args:
|
||||
hierarchy (dict): The nested dictionary representing the UI hierarchy.
|
||||
resource_id (str): The resource-id to find.
|
||||
|
||||
Returns:
|
||||
list: A list of the sibling elements, or None if the resource_id is not
|
||||
found.
|
||||
"""
|
||||
if not hierarchy:
|
||||
return None
|
||||
|
||||
for child in hierarchy:
|
||||
if child.get("attributes", {}).get("resource-id") == resource_id:
|
||||
return child.get("attributes", {})
|
||||
|
||||
for child in hierarchy:
|
||||
result = __find_element_by_resource_id_in_rich_hierarchy(
|
||||
child.get("children", []), resource_id
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def text_input_is_empty(text: str | None, hint_text: str | None) -> bool:
|
||||
return not text or text == hint_text
|
||||
|
||||
|
||||
def find_element_by_resource_id(
|
||||
ui_hierarchy: list[dict],
|
||||
resource_id: str,
|
||||
index: int | None = None,
|
||||
is_rich_hierarchy: bool = False,
|
||||
) -> dict | None:
|
||||
"""Find a UI element by its resource-id in the UI hierarchy.
|
||||
|
||||
Args:
|
||||
ui_hierarchy: List of UI element dictionaries
|
||||
resource_id: The resource-id to search for (e.g.,
|
||||
"com.google.android.settings.intelligence:id/open_search_view_edit_text")
|
||||
|
||||
Returns:
|
||||
The complete UI element dictionary if found, None otherwise
|
||||
"""
|
||||
if is_rich_hierarchy:
|
||||
return __find_element_by_resource_id_in_rich_hierarchy(ui_hierarchy, resource_id)
|
||||
|
||||
target_idx = index or 0
|
||||
|
||||
def search_recursive(elements: list[dict]) -> dict | None:
|
||||
nonlocal target_idx
|
||||
for element in elements:
|
||||
if isinstance(element, dict):
|
||||
if element.get("resourceId") == resource_id:
|
||||
if target_idx == 0:
|
||||
return element
|
||||
target_idx -= 1
|
||||
continue
|
||||
|
||||
children = element.get("children", [])
|
||||
if children:
|
||||
result = search_recursive(children)
|
||||
if result:
|
||||
return result
|
||||
return None
|
||||
|
||||
return search_recursive(ui_hierarchy)
|
||||
|
||||
|
||||
def is_element_focused(element: dict) -> bool:
|
||||
return element.get("focused", None) == "true"
|
||||
|
||||
|
||||
def get_element_text(element: dict, hint_text: bool = False) -> str | None:
|
||||
if hint_text:
|
||||
return element.get("hintText", None)
|
||||
return element.get("text", None)
|
||||
|
||||
|
||||
class _CyFunctionDetectorMeta(type):
|
||||
def __instancecheck__(self, instance):
|
||||
name = type(instance).__name__
|
||||
return (
|
||||
name
|
||||
in (
|
||||
"cyfunction",
|
||||
"cython_function_or_method",
|
||||
"builtin_function_or_method",
|
||||
)
|
||||
or "cyfunction" in name.lower()
|
||||
)
|
||||
|
||||
|
||||
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
|
||||
pass
|
||||
|
||||
|
||||
class Point(BaseModel):
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
"""Discrete pixel coordinate on screen."""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
x: int
|
||||
y: int
|
||||
|
||||
|
||||
class ElementBounds(BaseModel):
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
x: int = Field(description="The x coordinate of the top-left corner of the element.")
|
||||
y: int = Field(description="The y coordinate of the top-left corner of the element.")
|
||||
width: int = Field(description="The width of the element.")
|
||||
height: int = Field(description="The height of the element.")
|
||||
"""Axis-aligned rectangular boundary defining an on-screen element."""
|
||||
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
x: int = Field(description="Horizontal offset of the top-left corner in pixels.")
|
||||
y: int = Field(description="Vertical offset of the top-left corner in pixels.")
|
||||
width: int = Field(description="Horizontal extent of the bounding box.")
|
||||
height: int = Field(description="Vertical extent of the bounding box.")
|
||||
|
||||
def get_center(self) -> Point:
|
||||
return Point(x=self.x + self.width // 2, y=self.y + self.height // 2)
|
||||
"""Compute the centroid of the element boundary."""
|
||||
return Point(x=self.x + (self.width // 2), y=self.y + (self.height // 2))
|
||||
|
||||
def get_relative_point(self, x_percent: float, y_percent: float) -> Point:
|
||||
"""Returns the coordinates of the point at x_percent of the width and y_percent
|
||||
|
||||
of the height of the element.
|
||||
|
||||
Ex if x_percent = 0.95 and y_percent = 0.95,
|
||||
the point is at the bottom right of the element:
|
||||
<------>
|
||||
| |
|
||||
| x|
|
||||
<------>
|
||||
"""
|
||||
"""Compute an internal pixel coordinate by fractional offsets [0.0, 1.0]."""
|
||||
clamped_x = max(0.0, min(1.0, float(x_percent)))
|
||||
clamped_y = max(0.0, min(1.0, float(y_percent)))
|
||||
return Point(
|
||||
x=int(self.x + self.width * x_percent),
|
||||
y=int(self.y + self.height * y_percent),
|
||||
x=int(self.x + (self.width * clamped_x)),
|
||||
y=int(self.y + (self.height * clamped_y)),
|
||||
)
|
||||
|
||||
|
||||
def get_bounds_for_element(element: dict) -> ElementBounds | None:
|
||||
bounds = element.get("bounds", None)
|
||||
if bounds:
|
||||
try:
|
||||
return ElementBounds(**bounds)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to validate bounds: {e}")
|
||||
return None
|
||||
def get_bounds_for_element(element: dict[str, Any]) -> ElementBounds | None:
|
||||
"""Extract and validate the bounding box from a node descriptor."""
|
||||
raw_bounds = element.get("bounds")
|
||||
if not isinstance(raw_bounds, dict):
|
||||
return None
|
||||
|
||||
try:
|
||||
return ElementBounds(
|
||||
x=int(raw_bounds["x"]),
|
||||
y=int(raw_bounds["y"]),
|
||||
width=int(raw_bounds["width"]),
|
||||
height=int(raw_bounds["height"]),
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
logger.error(f"Failed to validate element bounds ({raw_bounds}): {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def text_input_is_empty(text: str | None, hint_text: str | None) -> bool:
|
||||
"""Determine whether an editable field contains user-entered text."""
|
||||
if not text:
|
||||
return True
|
||||
return text == hint_text
|
||||
|
||||
|
||||
def is_element_focused(element: dict[str, Any]) -> bool:
|
||||
"""Check if the given node currently holds system focus."""
|
||||
focused_val = element.get("focused")
|
||||
if isinstance(focused_val, bool):
|
||||
return focused_val
|
||||
return str(focused_val).strip().lower() == "true"
|
||||
|
||||
|
||||
def get_element_text(element: dict[str, Any], hint_text: bool = False) -> str | None:
|
||||
"""Retrieve textual label or placeholder from an element node."""
|
||||
if hint_text:
|
||||
return element.get("hintText")
|
||||
return element.get("text")
|
||||
|
||||
|
||||
def find_element_by_resource_id(
|
||||
ui_hierarchy: list[dict[str, Any]],
|
||||
resource_id: str,
|
||||
index: int | None = None,
|
||||
is_rich_hierarchy: bool = False,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Locate an element in the hierarchy tree matching the specified resource ID.
|
||||
|
||||
Supports both flat attribute mappings and rich accessibility tree hierarchies.
|
||||
"""
|
||||
if not ui_hierarchy or not resource_id:
|
||||
return None
|
||||
|
||||
if is_rich_hierarchy:
|
||||
return _search_rich_tree(ui_hierarchy, resource_id)
|
||||
|
||||
target_index = max(0, index or 0)
|
||||
current_match = 0
|
||||
|
||||
def _walk(nodes: list[dict[str, Any]]) -> dict[str, Any] | None:
|
||||
nonlocal current_match
|
||||
for node in nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
|
||||
if node.get("resourceId") == resource_id:
|
||||
if current_match == target_index:
|
||||
return node
|
||||
current_match += 1
|
||||
|
||||
children = node.get("children")
|
||||
if isinstance(children, list) and children:
|
||||
found = _walk(children)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
return _walk(ui_hierarchy)
|
||||
|
||||
|
||||
def _search_rich_tree(
|
||||
nodes: list[dict[str, Any]], resource_id: str
|
||||
) -> dict[str, Any] | None:
|
||||
"""Recursively traverse a rich XML attribute tree for resource ID match."""
|
||||
for node in nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
|
||||
attributes = node.get("attributes")
|
||||
if isinstance(attributes, dict) and attributes.get("resource-id") == resource_id:
|
||||
return attributes
|
||||
|
||||
children = node.get("children")
|
||||
if isinstance(children, list) and children:
|
||||
sub = _search_rich_tree(children, resource_id)
|
||||
if sub is not None:
|
||||
return sub
|
||||
|
||||
return None
|
||||
Reference in new issue
Block a user