refactor: clean up legacy code, rename hopper to entity_extractor, and add third-party notices

- Add NOTICE file and README license references for mobile-use and finalrun-agent
- Add derivative attribution headers to 17 files derived from mobile-use
- Rename hopper agent to entity_extractor across prompts, tools, and configs
- Remove obsolete adb_tunnel, ApiBaseUrl, and legacy CLI utility scripts
- Replace legacy examples with native Artemis diagnostic and workflow examples
- Update unit and tool tests to validate entity_extractor and SDK builders
This commit is contained in:
somew1nd committed 2026-09-11 16:19:42 -07:00
1 parent 0860788192
commit 54fcef9887
83 files changed
+1996 -3660

No files matched your search

-3
View File
@@ -46,7 +46,6 @@ from artemis.tools.tool_wrapper import (
invoke_tool_with_injection,
tool_result_messages,
)
from artemis.tools.types import CyFunctionDetector
from artemis.tools.video_tool import get_video_analyzer_tool
from artemis.utils.logger import get_logger
from artemis.utils.notes import get_note_file_path, get_notes_dir
@@ -61,8 +60,6 @@ logger = get_logger(__name__)
class AskCommitteeArgs(BaseModel):
"""Arguments schema for invoking the committee tool."""
model_config = {"ignored_types": (CyFunctionDetector,)}
avatar_directive: str = Field(
...,
description="Directive to frame and steer the debate for your avatar.",
-3
View File
@@ -27,7 +27,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool
from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
@@ -36,8 +35,6 @@ logger = get_logger(__name__)
# pylint: disable=too-few-public-methods
class AskDiagnoserArgs(BaseModel):
"""Arguments schema for invoking the diagnostic subagent."""
model_config = {"ignored_types": (CyFunctionDetector,)}
query: str = Field(
...,
description=(
-3
View File
@@ -58,7 +58,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolCategory
from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.element_hit_test import find_element_at_point
from artemis.utils.logger import get_logger
from artemis.utils.visualization import draw_dots
@@ -94,8 +93,6 @@ CANDIDATE_DOT_COLOR = "magenta"
class AskExplorerArgs(BaseModel):
"""Arguments of ``ask_explorer`` as seen by every calling agent."""
model_config = {"ignored_types": (CyFunctionDetector,)}
query: str = Field(..., description=ASK_EXPLORER_QUERY_DESCRIPTION)
context_feedback: str = Field("", description=ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION)
-3
View File
@@ -25,7 +25,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool
from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
@@ -34,8 +33,6 @@ logger = get_logger(__name__)
# pylint: disable=too-few-public-methods
class AskVisionCoderArgs(BaseModel):
"""Arguments schema for invoking the Image Processor subagent."""
model_config = {"ignored_types": (CyFunctionDetector,)}
instruction: str = Field(
...,
description=(
-3
View File
@@ -27,7 +27,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool
from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
@@ -36,8 +35,6 @@ logger = get_logger(__name__)
# pylint: disable=too-few-public-methods
class AnalyzeLogsArgs(BaseModel):
"""Arguments schema for invoking the Log Analyzer subagent."""
model_config = {"ignored_types": (CyFunctionDetector,)}
specific_query: str = Field(
...,
description=(
+8 -8
View File
@@ -18,7 +18,10 @@ from langchain_core.messages import ToolMessage
from langchain_core.tools import BaseTool
from pydantic import BaseModel, Field
from artemis.agents.hopper.hopper import HopperOutput, hopper
from artemis.agents.entity_extractor.entity_extractor import (
ExtractionResult,
extract_entity,
)
from artemis.context import ArtemisContext
from artemis.controllers.platform_specific_commands_controller import (
list_packages_async,
@@ -28,7 +31,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolCategory
from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.app_launch_utils import launch_app_with_retries
from artemis.utils.logger import get_logger
@@ -37,8 +39,6 @@ logger = get_logger(__name__)
class LaunchAppArgs(BaseModel):
"""Arguments schema for launching an application."""
model_config = {"ignored_types": (CyFunctionDetector,)}
app_name: str = Field(
...,
description="The natural language name of the application to launch.",
@@ -77,21 +77,21 @@ async def find_package(ctx: ArtemisContext, app_name: str, use_fallback: bool =
package_cache[app_name] = app_name
return app_name
hopper_output: HopperOutput = await hopper(
extractor_output: ExtractionResult = await extract_entity(
ctx=ctx,
request=(f"I'm looking for the package name of the following app: '{app_name}'"),
data=all_packages,
use_fallback=use_fallback,
)
if not hopper_output.found or not hopper_output.output:
if not extractor_output.found or not extractor_output.output:
if isinstance(package_cache, dict):
package_cache[app_name] = None
return None
package_name = hopper_output.output.strip()
package_name = extractor_output.output.strip()
if package_name not in package_set:
logger.warning(
f"Hopper returned package '{package_name}' for '{app_name}', "
f"Entity extractor returned package '{package_name}' for '{app_name}', "
"but it is NOT physically installed on the device!"
)
if isinstance(package_cache, dict):
-5
View File
@@ -25,7 +25,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolCategory
from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
@@ -33,10 +32,6 @@ logger = get_logger(__name__)
class GetUiHierarchyArgs(BaseModel):
"""Arguments schema for getting UI hierarchy."""
model_config = {"ignored_types": (CyFunctionDetector,)}
GET_UI_HIERARCHY_DOCSTRING = (
"[DIAGNOSTIC] Retrieves the current screen UI hierarchy XML using a persistent"
" UI Automator client.\n\nUse this instead of running 'uiautomator dump'"
-5
View File
@@ -35,7 +35,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolCategory
from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
@@ -44,8 +43,6 @@ logger = get_logger(__name__)
# pylint: disable=too-few-public-methods
class ObjectDetectionArgs(BaseModel):
"""Arguments schema for object detection with explicit screenshot path."""
model_config = {"ignored_types": (CyFunctionDetector,)}
image_path: str = Field(..., description="Absolute path to the screenshot file.")
queries: list[str] = Field(..., description="A list of single query strings to find.")
@@ -53,8 +50,6 @@ class ObjectDetectionArgs(BaseModel):
# pylint: disable=too-few-public-methods
class OperatorObjectDetectionArgs(BaseModel):
"""Arguments schema for operator object detection using latest screenshot from state."""
model_config = {"ignored_types": (CyFunctionDetector,)}
queries: list[str] = Field(..., description="A list of single query strings to find.")
-13
View File
@@ -25,7 +25,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool
from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger
from artemis.utils.notes import (
APPEND_NOTE_ARG_CONTENT_DESC,
@@ -57,24 +56,18 @@ logger = get_logger(__name__)
class SaveNoteArgs(BaseModel):
"""Arguments schema for saving notes."""
model_config = {"ignored_types": (CyFunctionDetector,)}
key: str = Field(..., description=SAVE_NOTE_ARG_KEY_DESC)
content: str = Field(..., description=SAVE_NOTE_ARG_CONTENT_DESC)
class AppendNoteArgs(BaseModel):
"""Arguments schema for appending notes."""
model_config = {"ignored_types": (CyFunctionDetector,)}
key: str = Field(..., description=APPEND_NOTE_ARG_KEY_DESC)
content: str = Field(..., description=APPEND_NOTE_ARG_CONTENT_DESC)
class UpdateNoteArgs(BaseModel):
"""Arguments schema for updating notes."""
model_config = {"ignored_types": (CyFunctionDetector,)}
key: str = Field(..., description=UPDATE_NOTE_ARG_KEY_DESC)
target: str = Field(..., description=UPDATE_NOTE_ARG_TARGET_DESC)
replacement: str = Field(..., description=UPDATE_NOTE_ARG_REPLACEMENT_DESC)
@@ -82,8 +75,6 @@ class UpdateNoteArgs(BaseModel):
class ReadNoteArgs(BaseModel):
"""Arguments schema for reading notes."""
model_config = {"ignored_types": (CyFunctionDetector,)}
key: str = Field(..., description=READ_NOTE_ARG_KEY_DESC)
start_line: int | None = Field(None, description="Start line to read (1-indexed, inclusive)")
end_line: int | None = Field(None, description="End line to read (1-indexed, inclusive)")
@@ -91,10 +82,6 @@ class ReadNoteArgs(BaseModel):
class ListNotesArgs(BaseModel):
"""Arguments schema for listing notes."""
model_config = {"ignored_types": (CyFunctionDetector,)}
class SaveNoteTool(ArtemisTool):
"""Universal tool for saving notes to persistent memory."""
-5
View File
@@ -31,13 +31,8 @@ from artemis.context import ArtemisContext
from artemis.data_engine import engine as engine_mod
from artemis.data_engine.trace import CURRENT_TRACE_ID, smart_serialize
from artemis.graph.state import State
from artemis.tools.types import CyFunctionDetector
class ToolWrapper(BaseModel):
"""Wrapper holding a tool factory and lifecycle callbacks."""
model_config = {"ignored_types": (CyFunctionDetector,)}
tool_fn_getter: Callable[[ArtemisContext], BaseTool]
on_success_fn: Callable[..., str]
on_failure_fn: Callable[..., str]
+24 -38
View File
@@ -12,61 +12,47 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from pydantic import BaseModel, Field, model_validator
"""Multi-strategy target element descriptors for ARTEMIS actions.
Encapsulates semantic identifiers, textual queries, and spatial bounding boxes
into a prioritized target resolution contract.
"""
from __future__ import annotations
from pydantic import BaseModel, ConfigDict, Field, model_validator
from artemis.utils.ui_hierarchy import ElementBounds
class _CyFunctionDetectorMeta(type):
"""Metaclass for detecting cython compiled functions."""
def __instancecheck__(cls, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
# pylint: disable=too-few-public-methods
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
"""Detector for cython compiled functions."""
class Target(BaseModel):
"""A comprehensive locator for a UI element, supporting a fallback mechanism."""
"""Composite locator descriptor resolving UI nodes across dynamic and visual channels."""
model_config = {"ignored_types": (CyFunctionDetector,)}
model_config = ConfigDict(extra="ignore")
resource_id: str | None = Field(None, description="The resource-id of the element.")
resource_id: str | None = Field(default=None, description="Android resource identifier token.")
resource_id_index: int | None = Field(
None,
description=("The zero-based index if multiple elements share the same resource-id."),
default=None,
description="Zero-based ordinal index when multiple nodes match resource_id.",
)
text: str | None = Field(
None,
description=("The text content of the element (e.g., a label or placeholder)."),
default=None,
description="Exact or case-insensitive element label or content descriptor.",
)
text_index: int | None = Field(
None,
description=("The zero-based index if multiple elements share the same text."),
default=None,
description="Zero-based ordinal index when multiple nodes match text query.",
)
bounds: ElementBounds | None = Field(
None, description="The x, y, width, and height of the element."
default=None,
description="Spatial boundary metrics (x, y, width, height) of the element.",
)
@model_validator(mode="after")
def _default_indices(self):
# Treat empty strings like “not provided”
if (
self.resource_id is not None and self.resource_id != ""
) and self.resource_id_index is None:
def _normalize_indices(self) -> Target:
"""Coerce missing indices to zero if primary selector string is populated."""
if self.resource_id and self.resource_id.strip() and self.resource_id_index is None:
self.resource_id_index = 0
if (self.text is not None and self.text != "") and self.text_index is None:
if self.text and self.text.strip() and self.text_index is None:
self.text_index = 0
return self
+207 -277
View File
@@ -11,25 +11,29 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
"""Spatial positioning, focus resolution, and interaction helpers for ARTEMIS tools.
Coordinates locator resolution across hierarchy trees, validates bounded screen areas,
and routes focus and cursor placement to target elements.
"""
from __future__ import annotations
from typing import Literal
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict
from artemis.context import ArtemisContext
from artemis.controllers.types import (
CoordinatesSelectorRequest,
PercentagesSelectorRequest,
)
from artemis.controllers.types import CoordinatesSelectorRequest, PercentagesSelectorRequest
from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.graph.state import State
from artemis.tools.types import Target
from artemis.utils.logger import get_logger
from artemis.utils.ui_hierarchy import (
ElementBounds,
Point,
find_element_by_resource_id,
get_bounds_for_element,
get_element_text,
@@ -39,293 +43,54 @@ from artemis.utils.ui_hierarchy import (
logger = get_logger(__name__)
def find_element_by_text(
ui_hierarchy: list[dict], text: str, index: int | None = None
) -> dict | None:
"""Find a UI element by its text content (adapted to both flat and rich hierarchy)
This function performs a recursive, case-insensitive search.
Args:
ui_hierarchy: List of UI element dictionaries.
text: The text content to search for.
index: Optional index to select nth matching element.
Returns:
The complete UI element dictionary if found, None otherwise.
"""
matches: list[dict] = []
def search_recursive(elements: list[dict]) -> None:
for element in elements:
if isinstance(element, dict):
src = element.get("attributes", element)
element_text = src.get("text", "")
# Guard against non-string values
if not isinstance(element_text, str):
element_text = ""
text_lower = text.lower()
if element_text and text_lower == element_text.lower():
matches.append(element)
if children := element.get("children", []):
search_recursive(children)
search_recursive(ui_hierarchy)
if not matches:
return None
if index is None:
idx = 0
elif index < 0:
return None
else:
idx = index
if idx < len(matches):
return matches[idx]
return None
async def tap_bottom_right_of_element(bounds: ElementBounds, ctx: ArtemisContext):
"""Tap the bottom-right corner of the specified element bounds."""
bottom_right: Point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99)
await tap(
ctx=ctx,
selector_request=SelectorRequestWithCoordinates(
coordinates=CoordinatesSelectorRequest(
x=bottom_right.x,
y=bottom_right.y,
),
),
)
# pylint: disable=too-many-return-statements
async def move_cursor_to_end_if_bounds(
ctx: ArtemisContext,
state: State,
target: Target,
elt: dict | None = None,
) -> dict | None:
"""Best-effort move of the text cursor near the end of the input by tapping the
bottom-right area of the focused element (if bounds are available).
"""
if target.resource_id:
if not elt:
elt = find_element_by_resource_id(
ui_hierarchy=state.latest_ui_hierarchy or [],
resource_id=target.resource_id,
index=target.resource_id_index,
)
if not elt:
return None
bounds = get_bounds_for_element(elt)
if not bounds:
return elt
logger.debug("Tapping near the end of the input to move the cursor")
await tap_bottom_right_of_element(bounds=bounds, ctx=ctx)
logger.debug(f"Tapped end of input {target.resource_id}")
return elt
if target.bounds:
await tap_bottom_right_of_element(target.bounds, ctx=ctx)
logger.debug("Tapped end of input by coordinates")
return elt
if target.text:
text_elt = find_element_by_text(
state.latest_ui_hierarchy or [],
target.text,
index=target.text_index,
)
if text_elt:
bounds = get_bounds_for_element(text_elt)
if bounds:
await tap_bottom_right_of_element(bounds=bounds, ctx=ctx)
logger.debug(f"Tapped end of input that had text'{target.text}'")
return text_elt
return None
return None
async def focus_element_if_needed(
ctx: ArtemisContext, target: Target
) -> Literal["resource_id", "coordinates", "text", "already_focused"] | None:
"""Ensures the element is focused, with a sanity check to prevent trusting misleading IDs.
If no target locator is provided (empty target), assumes the field is
already focused
and returns "already_focused" to allow typing without re-focusing.
"""
# If no locator is provided, assume already focused (e.g., keyboard is visible)
if not target.resource_id and not target.bounds and not target.text:
logger.debug("No target locator provided, assuming element is already focused")
return "already_focused"
controller = UnifiedMobileController(ctx)
rich_hierarchy = await controller.get_ui_elements()
elt_from_id = None
if target.resource_id:
elt_from_id = find_element_by_resource_id(
ui_hierarchy=rich_hierarchy,
resource_id=target.resource_id,
index=target.resource_id_index,
is_rich_hierarchy=False,
)
if elt_from_id and target.text:
text_from_id_elt = get_element_text(elt_from_id)
if not text_from_id_elt or target.text.lower() != text_from_id_elt.lower():
logger.warning(
f"ID '{target.resource_id}' and text '{target.text}' seem to be"
" on different elements. Ignoring the resource_id and falling"
" back to other locators."
)
elt_from_id = None
if elt_from_id:
if not is_element_focused(elt_from_id):
await tap(
ctx=ctx,
selector_request=IdSelectorRequest(id=target.resource_id), # type: ignore
index=target.resource_id_index,
)
logger.debug(f"Focused (tap) on resource_id={target.resource_id}")
rich_hierarchy = await controller.get_ui_elements()
elt_from_id = find_element_by_resource_id(
ui_hierarchy=rich_hierarchy,
resource_id=target.resource_id, # type: ignore
index=target.resource_id_index,
is_rich_hierarchy=False,
)
if elt_from_id and is_element_focused(elt_from_id):
logger.debug(f"Text input is focused: {target.resource_id}")
return "resource_id"
logger.warning(f"Failed to focus using resource_id='{target.resource_id}'. Fallback...")
if target.bounds:
relative_point = target.bounds.get_center()
await tap(
ctx=ctx,
selector_request=SelectorRequestWithCoordinates(
coordinates=CoordinatesSelectorRequest(x=relative_point.x, y=relative_point.y)
),
)
logger.debug(f"Tapped on coordinates ({relative_point.x}, {relative_point.y}) to focus.")
return "coordinates"
if target.text:
text_elt = find_element_by_text(rich_hierarchy, target.text, index=target.text_index)
if text_elt:
bounds = get_bounds_for_element(text_elt)
if bounds:
relative_point = bounds.get_center()
await tap(
ctx=ctx,
selector_request=SelectorRequestWithCoordinates(
coordinates=CoordinatesSelectorRequest(
x=relative_point.x, y=relative_point.y
)
),
)
logger.debug(f"Tapped on text element '{target.text}' to focus.")
return "text"
logger.error(
"Failed to focus element. No valid locator (resource_id, coordinates, or text) succeeded."
)
return None
def validate_coordinates_bounds(
target: Target, screen_width: int, screen_height: int
) -> str | None:
"""Validate that coordinates are within screen bounds.
Returns error message if invalid, None if valid.
"""
if not target.bounds:
return None
center = target.bounds.get_center()
errors = []
if center.x < 0 or center.x >= screen_width:
errors.append(f"x={center.x} is outside screen width (0-{screen_width})")
if center.y < 0 or center.y >= screen_height:
errors.append(f"y={center.y} is outside screen height (0-{screen_height})")
return "; ".join(errors) if errors else None
def has_valid_selectors(target: Target) -> bool:
"""Check if target has at least one valid selector."""
has_coordinates = target.bounds is not None
has_resource_id = target.resource_id is not None and target.resource_id != ""
has_text = target.text is not None and target.text != ""
return has_coordinates or has_resource_id or has_text
class IdSelectorRequest(BaseModel):
"""Selector request using resource id."""
"""Target selector driven by Android resource ID."""
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
id: str
def to_dict(self) -> dict[str, str | int]:
"""Convert selector to dictionary."""
return {"id": self.id}
class TextSelectorRequest(BaseModel):
"""Selector request using element text."""
"""Target selector driven by exact or case-insensitive element label."""
model_config = ConfigDict(extra="forbid")
text: str
def to_dict(self) -> dict[str, str | int]:
"""Convert selector to dictionary."""
return {"text": self.text}
class SelectorRequestWithCoordinates(BaseModel):
"""Selector request using absolute coordinates."""
"""Target selector pointing to an absolute pixel coordinate."""
model_config = ConfigDict(extra="forbid")
coordinates: CoordinatesSelectorRequest
def to_dict(self) -> dict[str, str | int]:
"""Convert selector to dictionary."""
return {"point": self.coordinates.to_str()}
class SelectorRequestWithPercentages(BaseModel):
"""Selector request using percentage coordinates."""
"""Target selector pointing to normalized fractional [0.0, 1.0] coordinates."""
model_config = ConfigDict(extra="forbid")
percentages: PercentagesSelectorRequest
def to_dict(self) -> dict[str, str | int]:
"""Convert selector to dictionary."""
return {"point": self.percentages.to_str()}
class IdWithTextSelectorRequest(BaseModel):
"""Selector request using both resource id and text."""
"""Composite selector pairing resource ID with expected text."""
model_config = ConfigDict(extra="forbid")
id: str
text: str
def to_dict(self) -> dict[str, str | int]:
"""Convert selector to dictionary."""
return {"id": self.id, "text": self.text}
@@ -341,37 +106,30 @@ SelectorRequest = (
def _extract_resource_id_and_text_from_selector(
selector: SelectorRequest,
) -> tuple[str | None, str | None]:
"""Extract resource_id and text from a selector."""
resource_id = None
text = None
"""Unpack identifier and textual query from any supported SelectorRequest."""
if isinstance(selector, IdSelectorRequest):
resource_id = selector.id
elif isinstance(selector, TextSelectorRequest):
text = selector.text
elif isinstance(selector, IdWithTextSelectorRequest):
resource_id = selector.id
text = selector.text
return resource_id, text
return selector.id, None
if isinstance(selector, TextSelectorRequest):
return None, selector.text
if isinstance(selector, IdWithTextSelectorRequest):
return selector.id, selector.text
return None, None
async def tap(
ctx: ArtemisContext,
selector_request: SelectorRequest,
index: int | None = None,
):
"""Tap on a selector.
Index is optional and is used when you have multiple views matching the same
selector. ui_hierarchy is optional and used for ADB taps to find elements.
"""
) -> Any:
"""Dispatch an on-screen tap to an element or coordinate selector."""
controller = UnifiedMobileController(ctx)
if isinstance(selector_request, SelectorRequestWithCoordinates):
result = await controller.tap_at(
x=selector_request.coordinates.x, y=selector_request.coordinates.y
res = await controller.tap_at(
x=selector_request.coordinates.x,
y=selector_request.coordinates.y,
)
return result.error if result.error else None
return res.error if res.error else None
if isinstance(selector_request, SelectorRequestWithPercentages):
coords = selector_request.percentages.to_coords(
width=ctx.device.device_width,
@@ -379,11 +137,183 @@ async def tap(
)
return await controller.tap_at(coords.x, coords.y)
# For other selectors, we need the UI hierarchy
resource_id, text = _extract_resource_id_and_text_from_selector(selector_request)
res_id, txt = _extract_resource_id_and_text_from_selector(selector_request)
return await controller.tap_element(
resource_id=resource_id,
text=text,
resource_id=res_id,
text=txt,
index=index if index is not None else 0,
)
def find_element_by_text(
ui_hierarchy: list[dict[str, Any]], text: str, index: int | None = None
) -> dict[str, Any] | None:
"""Search for elements matching label text across nested hierarchy tree nodes."""
target_text = (text or "").strip().lower()
if not target_text:
return None
matched_nodes: list[dict[str, Any]] = []
def _traverse(nodes: list[dict[str, Any]]) -> None:
for node in nodes:
if not isinstance(node, dict):
continue
attrs = node.get("attributes", node)
val = attrs.get("text")
if isinstance(val, str) and val.strip().lower() == target_text:
matched_nodes.append(attrs if "attributes" in node else node)
children = node.get("children")
if isinstance(children, list) and children:
_traverse(children)
_traverse(ui_hierarchy)
if not matched_nodes:
return None
target_idx = max(0, index or 0)
if target_idx < len(matched_nodes):
return matched_nodes[target_idx]
return None
async def tap_bottom_right_of_element(bounds: ElementBounds, ctx: ArtemisContext) -> None:
"""Position pointer near the trailing boundary of an input element."""
corner_point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99)
await tap(
ctx=ctx,
selector_request=SelectorRequestWithCoordinates(
coordinates=CoordinatesSelectorRequest(x=corner_point.x, y=corner_point.y)
),
)
async def move_cursor_to_end_if_bounds(
ctx: ArtemisContext,
state: State,
target: Target,
elt: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Position insertion point at the end of an input element boundary."""
current_tree = state.latest_ui_hierarchy or []
if target.resource_id:
element_node = elt or find_element_by_resource_id(
ui_hierarchy=current_tree,
resource_id=target.resource_id,
index=target.resource_id_index,
)
if not element_node:
return None
box = get_bounds_for_element(element_node)
if not box:
return element_node
await tap_bottom_right_of_element(bounds=box, ctx=ctx)
return element_node
if target.bounds:
await tap_bottom_right_of_element(bounds=target.bounds, ctx=ctx)
return elt
if target.text:
matched = find_element_by_text(current_tree, target.text, index=target.text_index)
if matched:
box = get_bounds_for_element(matched)
if box:
await tap_bottom_right_of_element(bounds=box, ctx=ctx)
return matched
return None
return None
async def focus_element_if_needed(
ctx: ArtemisContext, target: Target
) -> Literal["resource_id", "coordinates", "text", "already_focused"] | None:
"""Ensure element possesses active input focus, falling back through available selectors."""
if not target.resource_id and not target.bounds and not target.text:
return "already_focused"
controller = UnifiedMobileController(ctx)
hierarchy = await controller.get_ui_elements()
element_by_id = None
if target.resource_id:
element_by_id = find_element_by_resource_id(
ui_hierarchy=hierarchy,
resource_id=target.resource_id,
index=target.resource_id_index,
is_rich_hierarchy=False,
)
if element_by_id and target.text:
observed_label = get_element_text(element_by_id)
if not observed_label or target.text.strip().lower() != observed_label.strip().lower():
logger.warning(
f"ID '{target.resource_id}' and text '{target.text}' do not align to the same node."
" Falling back to secondary selectors."
)
element_by_id = None
if element_by_id:
if not is_element_focused(element_by_id):
await tap(
ctx=ctx,
selector_request=IdSelectorRequest(id=target.resource_id), # type: ignore[arg-type]
index=target.resource_id_index,
)
hierarchy = await controller.get_ui_elements()
element_by_id = find_element_by_resource_id(
ui_hierarchy=hierarchy,
resource_id=target.resource_id, # type: ignore[arg-type]
index=target.resource_id_index,
is_rich_hierarchy=False,
)
if element_by_id and is_element_focused(element_by_id):
return "resource_id"
if target.bounds:
centroid = target.bounds.get_center()
await tap(
ctx=ctx,
selector_request=SelectorRequestWithCoordinates(
coordinates=CoordinatesSelectorRequest(x=centroid.x, y=centroid.y)
),
)
return "coordinates"
if target.text:
matched_text_node = find_element_by_text(hierarchy, target.text, index=target.text_index)
if matched_text_node:
text_box = get_bounds_for_element(matched_text_node)
if text_box:
centroid = text_box.get_center()
await tap(
ctx=ctx,
selector_request=SelectorRequestWithCoordinates(
coordinates=CoordinatesSelectorRequest(x=centroid.x, y=centroid.y)
),
)
return "text"
logger.error("Failed to focus element. No valid locator (resource_id, coordinates, or text) succeeded.")
return None
def validate_coordinates_bounds(target: Target, screen_width: int, screen_height: int) -> str | None:
"""Verify target centroid resides within active viewport boundaries."""
if not target.bounds:
return None
c = target.bounds.get_center()
errors: list[str] = []
if not (0 <= c.x < screen_width):
errors.append(f"x={c.x} is outside screen width (0-{screen_width})")
if not (0 <= c.y < screen_height):
errors.append(f"y={c.y} is outside screen height (0-{screen_height})")
return "; ".join(errors) if errors else None
def has_valid_selectors(target: Target) -> bool:
"""Verify target defines at least one valid locator criterion."""
return bool(target.bounds is not None or target.resource_id or target.text)
-3
View File
@@ -34,7 +34,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolCategory
from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
@@ -42,8 +41,6 @@ logger = get_logger(__name__)
class VideoAnalyzerArgs(BaseModel):
"""Arguments schema for video analyzer tool."""
model_config = {"ignored_types": (CyFunctionDetector,)}
time_description: str = Field(
...,
description=(
-3
View File
@@ -23,7 +23,6 @@ from artemis.data_engine.trace import trace_langchain_tool
from artemis.drivers.base import BaseDeviceDriver
from artemis.tools.base import ArtemisTool, ToolCategory
from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
@@ -31,8 +30,6 @@ logger = get_logger(__name__)
class WaitArgs(BaseModel):
"""Arguments schema for wait tool."""
model_config = {"ignored_types": (CyFunctionDetector,)}
seconds: int = Field(
...,
description=("The duration to wait/sleep in seconds. Minimum is 1, maximum is 60."),