somew1nd
54fcef9887
- Add NOTICE file and README license references for mobile-use and finalrun-agent - Add derivative attribution headers to 17 files derived from mobile-use - Rename hopper agent to entity_extractor across prompts, tools, and configs - Remove obsolete adb_tunnel, ApiBaseUrl, and legacy CLI utility scripts - Replace legacy examples with native Artemis diagnostic and workflow examples - Update unit and tool tests to validate entity_extractor and SDK builders
320 lines
11 KiB
Python
320 lines
11 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
|
|
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
|
|
|
|
"""Spatial positioning, focus resolution, and interaction helpers for ARTEMIS tools.
|
|
|
|
Coordinates locator resolution across hierarchy trees, validates bounded screen areas,
|
|
and routes focus and cursor placement to target elements.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any, Literal
|
|
from pydantic import BaseModel, ConfigDict
|
|
|
|
from artemis.context import ArtemisContext
|
|
from artemis.controllers.types import CoordinatesSelectorRequest, PercentagesSelectorRequest
|
|
from artemis.controllers.unified_controller import UnifiedMobileController
|
|
from artemis.graph.state import State
|
|
from artemis.tools.types import Target
|
|
from artemis.utils.logger import get_logger
|
|
from artemis.utils.ui_hierarchy import (
|
|
ElementBounds,
|
|
find_element_by_resource_id,
|
|
get_bounds_for_element,
|
|
get_element_text,
|
|
is_element_focused,
|
|
)
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
class IdSelectorRequest(BaseModel):
|
|
"""Target selector driven by Android resource ID."""
|
|
|
|
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
|
|
id: str
|
|
|
|
def to_dict(self) -> dict[str, str | int]:
|
|
return {"id": self.id}
|
|
|
|
|
|
class TextSelectorRequest(BaseModel):
|
|
"""Target selector driven by exact or case-insensitive element label."""
|
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
text: str
|
|
|
|
def to_dict(self) -> dict[str, str | int]:
|
|
return {"text": self.text}
|
|
|
|
|
|
class SelectorRequestWithCoordinates(BaseModel):
|
|
"""Target selector pointing to an absolute pixel coordinate."""
|
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
coordinates: CoordinatesSelectorRequest
|
|
|
|
def to_dict(self) -> dict[str, str | int]:
|
|
return {"point": self.coordinates.to_str()}
|
|
|
|
|
|
class SelectorRequestWithPercentages(BaseModel):
|
|
"""Target selector pointing to normalized fractional [0.0, 1.0] coordinates."""
|
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
percentages: PercentagesSelectorRequest
|
|
|
|
def to_dict(self) -> dict[str, str | int]:
|
|
return {"point": self.percentages.to_str()}
|
|
|
|
|
|
class IdWithTextSelectorRequest(BaseModel):
|
|
"""Composite selector pairing resource ID with expected text."""
|
|
|
|
model_config = ConfigDict(extra="forbid")
|
|
id: str
|
|
text: str
|
|
|
|
def to_dict(self) -> dict[str, str | int]:
|
|
return {"id": self.id, "text": self.text}
|
|
|
|
|
|
SelectorRequest = (
|
|
IdSelectorRequest
|
|
| SelectorRequestWithCoordinates
|
|
| SelectorRequestWithPercentages
|
|
| TextSelectorRequest
|
|
| IdWithTextSelectorRequest
|
|
)
|
|
|
|
|
|
def _extract_resource_id_and_text_from_selector(
|
|
selector: SelectorRequest,
|
|
) -> tuple[str | None, str | None]:
|
|
"""Unpack identifier and textual query from any supported SelectorRequest."""
|
|
if isinstance(selector, IdSelectorRequest):
|
|
return selector.id, None
|
|
if isinstance(selector, TextSelectorRequest):
|
|
return None, selector.text
|
|
if isinstance(selector, IdWithTextSelectorRequest):
|
|
return selector.id, selector.text
|
|
return None, None
|
|
|
|
|
|
async def tap(
|
|
ctx: ArtemisContext,
|
|
selector_request: SelectorRequest,
|
|
index: int | None = None,
|
|
) -> Any:
|
|
"""Dispatch an on-screen tap to an element or coordinate selector."""
|
|
controller = UnifiedMobileController(ctx)
|
|
if isinstance(selector_request, SelectorRequestWithCoordinates):
|
|
res = await controller.tap_at(
|
|
x=selector_request.coordinates.x,
|
|
y=selector_request.coordinates.y,
|
|
)
|
|
return res.error if res.error else None
|
|
|
|
if isinstance(selector_request, SelectorRequestWithPercentages):
|
|
coords = selector_request.percentages.to_coords(
|
|
width=ctx.device.device_width,
|
|
height=ctx.device.device_height,
|
|
)
|
|
return await controller.tap_at(coords.x, coords.y)
|
|
|
|
res_id, txt = _extract_resource_id_and_text_from_selector(selector_request)
|
|
return await controller.tap_element(
|
|
resource_id=res_id,
|
|
text=txt,
|
|
index=index if index is not None else 0,
|
|
)
|
|
|
|
|
|
def find_element_by_text(
|
|
ui_hierarchy: list[dict[str, Any]], text: str, index: int | None = None
|
|
) -> dict[str, Any] | None:
|
|
"""Search for elements matching label text across nested hierarchy tree nodes."""
|
|
target_text = (text or "").strip().lower()
|
|
if not target_text:
|
|
return None
|
|
|
|
matched_nodes: list[dict[str, Any]] = []
|
|
|
|
def _traverse(nodes: list[dict[str, Any]]) -> None:
|
|
for node in nodes:
|
|
if not isinstance(node, dict):
|
|
continue
|
|
attrs = node.get("attributes", node)
|
|
val = attrs.get("text")
|
|
if isinstance(val, str) and val.strip().lower() == target_text:
|
|
matched_nodes.append(attrs if "attributes" in node else node)
|
|
|
|
children = node.get("children")
|
|
if isinstance(children, list) and children:
|
|
_traverse(children)
|
|
|
|
_traverse(ui_hierarchy)
|
|
if not matched_nodes:
|
|
return None
|
|
|
|
target_idx = max(0, index or 0)
|
|
if target_idx < len(matched_nodes):
|
|
return matched_nodes[target_idx]
|
|
return None
|
|
|
|
|
|
async def tap_bottom_right_of_element(bounds: ElementBounds, ctx: ArtemisContext) -> None:
|
|
"""Position pointer near the trailing boundary of an input element."""
|
|
corner_point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99)
|
|
await tap(
|
|
ctx=ctx,
|
|
selector_request=SelectorRequestWithCoordinates(
|
|
coordinates=CoordinatesSelectorRequest(x=corner_point.x, y=corner_point.y)
|
|
),
|
|
)
|
|
|
|
|
|
async def move_cursor_to_end_if_bounds(
|
|
ctx: ArtemisContext,
|
|
state: State,
|
|
target: Target,
|
|
elt: dict[str, Any] | None = None,
|
|
) -> dict[str, Any] | None:
|
|
"""Position insertion point at the end of an input element boundary."""
|
|
current_tree = state.latest_ui_hierarchy or []
|
|
|
|
if target.resource_id:
|
|
element_node = elt or find_element_by_resource_id(
|
|
ui_hierarchy=current_tree,
|
|
resource_id=target.resource_id,
|
|
index=target.resource_id_index,
|
|
)
|
|
if not element_node:
|
|
return None
|
|
box = get_bounds_for_element(element_node)
|
|
if not box:
|
|
return element_node
|
|
await tap_bottom_right_of_element(bounds=box, ctx=ctx)
|
|
return element_node
|
|
|
|
if target.bounds:
|
|
await tap_bottom_right_of_element(bounds=target.bounds, ctx=ctx)
|
|
return elt
|
|
|
|
if target.text:
|
|
matched = find_element_by_text(current_tree, target.text, index=target.text_index)
|
|
if matched:
|
|
box = get_bounds_for_element(matched)
|
|
if box:
|
|
await tap_bottom_right_of_element(bounds=box, ctx=ctx)
|
|
return matched
|
|
return None
|
|
|
|
return None
|
|
|
|
|
|
async def focus_element_if_needed(
|
|
ctx: ArtemisContext, target: Target
|
|
) -> Literal["resource_id", "coordinates", "text", "already_focused"] | None:
|
|
"""Ensure element possesses active input focus, falling back through available selectors."""
|
|
if not target.resource_id and not target.bounds and not target.text:
|
|
return "already_focused"
|
|
|
|
controller = UnifiedMobileController(ctx)
|
|
hierarchy = await controller.get_ui_elements()
|
|
|
|
element_by_id = None
|
|
if target.resource_id:
|
|
element_by_id = find_element_by_resource_id(
|
|
ui_hierarchy=hierarchy,
|
|
resource_id=target.resource_id,
|
|
index=target.resource_id_index,
|
|
is_rich_hierarchy=False,
|
|
)
|
|
|
|
if element_by_id and target.text:
|
|
observed_label = get_element_text(element_by_id)
|
|
if not observed_label or target.text.strip().lower() != observed_label.strip().lower():
|
|
logger.warning(
|
|
f"ID '{target.resource_id}' and text '{target.text}' do not align to the same node."
|
|
" Falling back to secondary selectors."
|
|
)
|
|
element_by_id = None
|
|
|
|
if element_by_id:
|
|
if not is_element_focused(element_by_id):
|
|
await tap(
|
|
ctx=ctx,
|
|
selector_request=IdSelectorRequest(id=target.resource_id), # type: ignore[arg-type]
|
|
index=target.resource_id_index,
|
|
)
|
|
hierarchy = await controller.get_ui_elements()
|
|
element_by_id = find_element_by_resource_id(
|
|
ui_hierarchy=hierarchy,
|
|
resource_id=target.resource_id, # type: ignore[arg-type]
|
|
index=target.resource_id_index,
|
|
is_rich_hierarchy=False,
|
|
)
|
|
if element_by_id and is_element_focused(element_by_id):
|
|
return "resource_id"
|
|
|
|
if target.bounds:
|
|
centroid = target.bounds.get_center()
|
|
await tap(
|
|
ctx=ctx,
|
|
selector_request=SelectorRequestWithCoordinates(
|
|
coordinates=CoordinatesSelectorRequest(x=centroid.x, y=centroid.y)
|
|
),
|
|
)
|
|
return "coordinates"
|
|
|
|
if target.text:
|
|
matched_text_node = find_element_by_text(hierarchy, target.text, index=target.text_index)
|
|
if matched_text_node:
|
|
text_box = get_bounds_for_element(matched_text_node)
|
|
if text_box:
|
|
centroid = text_box.get_center()
|
|
await tap(
|
|
ctx=ctx,
|
|
selector_request=SelectorRequestWithCoordinates(
|
|
coordinates=CoordinatesSelectorRequest(x=centroid.x, y=centroid.y)
|
|
),
|
|
)
|
|
return "text"
|
|
|
|
logger.error("Failed to focus element. No valid locator (resource_id, coordinates, or text) succeeded.")
|
|
return None
|
|
|
|
|
|
def validate_coordinates_bounds(target: Target, screen_width: int, screen_height: int) -> str | None:
|
|
"""Verify target centroid resides within active viewport boundaries."""
|
|
if not target.bounds:
|
|
return None
|
|
c = target.bounds.get_center()
|
|
errors: list[str] = []
|
|
if not (0 <= c.x < screen_width):
|
|
errors.append(f"x={c.x} is outside screen width (0-{screen_width})")
|
|
if not (0 <= c.y < screen_height):
|
|
errors.append(f"y={c.y} is outside screen height (0-{screen_height})")
|
|
return "; ".join(errors) if errors else None
|
|
|
|
|
|
def has_valid_selectors(target: Target) -> bool:
|
|
"""Verify target defines at least one valid locator criterion."""
|
|
return bool(target.bounds is not None or target.resource_id or target.text)
|