somew1nd
67248c887e
Add execution incidents, action bursts and shared Flash history. Expose verification and Explorer settings with streamed check results. Support graceful cancellation, recording recovery and bundled UI resources. Add CI quality checks and clean up comments and documentation.
267 lines
10 KiB
Python
267 lines
10 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""In-memory index of the current screen's UI elements for the Explorer.
|
|
|
|
Built from the fused XML/OCR hierarchy so searches do not depend on the Data
|
|
Engine having persisted the frame. Supports fuzzy text search, exact matches
|
|
and coordinate hit testing.
|
|
"""
|
|
|
|
from dataclasses import dataclass, field
|
|
import difflib
|
|
import re
|
|
from typing import Any, Literal
|
|
|
|
from artemis.utils.visualization import parse_bounds
|
|
|
|
Bounds = tuple[int, int, int, int]
|
|
ElementSource = Literal["xml", "ocr"]
|
|
|
|
#: Attributes that mark an XML node as something the user can interact with.
|
|
INTERACTION_KEYS = (
|
|
"clickable",
|
|
"scrollable",
|
|
"long-clickable",
|
|
"checkable",
|
|
"focusable",
|
|
"editable",
|
|
"selected",
|
|
)
|
|
|
|
#: Minimum query length for substring matches to count as strong evidence;
|
|
#: shorter fragments ("ok", "a") match far too much.
|
|
_SUBSTRING_MIN_CHARS = 3
|
|
_SUBSTRING_SCORE = 0.85
|
|
|
|
|
|
def normalize_text(value: Any) -> str:
|
|
"""Case-folds and collapses whitespace so visually equal labels compare equal."""
|
|
return re.sub(r"\s+", " ", str(value or "")).strip().casefold()
|
|
|
|
|
|
def text_similarity(query: str, text: str) -> float:
|
|
"""Similarity in ``[0, 1]`` between a query and an element label.
|
|
|
|
Uses the Data Engine's ``SequenceMatcher`` ratio, with a minimum score
|
|
for substring matches when the query is at least three characters long.
|
|
"""
|
|
q, t = normalize_text(query), normalize_text(text)
|
|
if not q or not t:
|
|
return 0.0
|
|
ratio = difflib.SequenceMatcher(None, q, t).ratio()
|
|
if len(q) >= _SUBSTRING_MIN_CHARS and (q in t or t in q):
|
|
ratio = max(ratio, _SUBSTRING_SCORE)
|
|
return ratio
|
|
|
|
|
|
def _is_truthy_attr(value: Any) -> bool:
|
|
return value is True or str(value).strip().lower() == "true"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ScreenElement:
|
|
"""One text-bearing element of the screen with pixel bounds."""
|
|
|
|
text: str
|
|
bounds: Bounds
|
|
source: ElementSource
|
|
class_name: str | None = None
|
|
resource_id: str | None = None
|
|
interactive: bool = False
|
|
node: dict[str, Any] = field(default_factory=dict, compare=False, hash=False, repr=False)
|
|
|
|
@property
|
|
def center(self) -> tuple[int, int]:
|
|
left, top, right, bottom = self.bounds
|
|
return (left + right) // 2, (top + bottom) // 2
|
|
|
|
@property
|
|
def area(self) -> int:
|
|
left, top, right, bottom = self.bounds
|
|
return max(1, right - left) * max(1, bottom - top)
|
|
|
|
def contains(self, x: int, y: int) -> bool:
|
|
left, top, right, bottom = self.bounds
|
|
return left <= x <= right and top <= y <= bottom
|
|
|
|
def overlap_ratio(self, other: "ScreenElement") -> float:
|
|
"""Intersection over the smaller area; 1.0 when one box covers the other."""
|
|
l1, t1, r1, b1 = self.bounds
|
|
l2, t2, r2, b2 = other.bounds
|
|
inter_w = min(r1, r2) - max(l1, l2)
|
|
inter_h = min(b1, b2) - max(t1, t2)
|
|
if inter_w <= 0 or inter_h <= 0:
|
|
return 0.0
|
|
return (inter_w * inter_h) / min(self.area, other.area)
|
|
|
|
def describe(self) -> str:
|
|
"""Compact one-line description for tool text output."""
|
|
parts = [f"'{self.text}'"] if self.text else []
|
|
if self.class_name:
|
|
parts.append(f"class={self.class_name}")
|
|
if self.resource_id:
|
|
parts.append(f"id={self.resource_id}")
|
|
parts.append("source=ocr" if self.source == "ocr" else "source=ui-tree")
|
|
if self.interactive:
|
|
parts.append("interactive")
|
|
return " ".join(parts)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TextMatch:
|
|
element: ScreenElement
|
|
score: float
|
|
matched_text: str
|
|
|
|
|
|
class ScreenIndex:
|
|
"""Queryable snapshot of the visible, text-bearing elements of one screen."""
|
|
|
|
def __init__(self, elements: list[ScreenElement], width: int, height: int):
|
|
self.elements = elements
|
|
self.width = width
|
|
self.height = height
|
|
|
|
def __len__(self) -> int:
|
|
return len(self.elements)
|
|
|
|
@property
|
|
def ocr_elements(self) -> list[ScreenElement]:
|
|
return [e for e in self.elements if e.source == "ocr"]
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# Construction
|
|
# ------------------------------------------------------------------ #
|
|
|
|
@classmethod
|
|
def empty(cls, width: int, height: int) -> "ScreenIndex":
|
|
return cls([], width, height)
|
|
|
|
@classmethod
|
|
def from_hierarchy(
|
|
cls, fused_xml: list[dict[str, Any]] | None, width: int, height: int
|
|
) -> "ScreenIndex":
|
|
"""Builds the index from a fused UI hierarchy (XML nodes plus OCR text).
|
|
|
|
Off-screen and zero-area boxes are dropped, and the same label at the
|
|
same place is kept once. Nodes without any label are skipped: the
|
|
index exists to answer text and hit-test queries, and an unlabeled
|
|
wrapper answers neither usefully.
|
|
"""
|
|
elements: list[ScreenElement] = []
|
|
seen: set[tuple[str, Bounds]] = set()
|
|
|
|
def add(element: ScreenElement) -> None:
|
|
key = (normalize_text(element.text), element.bounds)
|
|
if key in seen or not cls._is_visible(element.bounds, width, height):
|
|
return
|
|
seen.add(key)
|
|
elements.append(element)
|
|
|
|
for node in fused_xml or []:
|
|
if not isinstance(node, dict):
|
|
continue
|
|
class_name = node.get("class") or node.get("className")
|
|
resource_id = node.get("resource-id") or node.get("resourceId")
|
|
interactive = any(_is_truthy_attr(node.get(key)) for key in INTERACTION_KEYS)
|
|
|
|
for ocr in node.get("ocr_elements") or []:
|
|
ocr_bounds = parse_bounds(ocr.get("bounds")) if isinstance(ocr, dict) else None
|
|
ocr_text = str(ocr.get("text") or "").strip() if isinstance(ocr, dict) else ""
|
|
if ocr_bounds and ocr_text:
|
|
add(
|
|
ScreenElement(
|
|
text=ocr_text,
|
|
bounds=ocr_bounds,
|
|
source="ocr",
|
|
class_name=str(class_name) if class_name else None,
|
|
resource_id=str(resource_id) if resource_id else None,
|
|
interactive=interactive,
|
|
node=node,
|
|
)
|
|
)
|
|
|
|
label = str(node.get("text") or node.get("content-desc") or "").strip()
|
|
bounds = parse_bounds(node.get("bounds"))
|
|
if label and bounds:
|
|
add(
|
|
ScreenElement(
|
|
text=label,
|
|
bounds=bounds,
|
|
source="xml",
|
|
class_name=str(class_name) if class_name else None,
|
|
resource_id=str(resource_id) if resource_id else None,
|
|
interactive=interactive,
|
|
node=node,
|
|
)
|
|
)
|
|
return cls(elements, width, height)
|
|
|
|
@staticmethod
|
|
def _is_visible(bounds: Bounds, width: int, height: int) -> bool:
|
|
left, top, right, bottom = bounds
|
|
if right <= left or bottom <= top:
|
|
return False
|
|
return right > 0 and bottom > 0 and left < width and top < height
|
|
|
|
# ------------------------------------------------------------------ #
|
|
# Queries
|
|
# ------------------------------------------------------------------ #
|
|
|
|
def search_text(self, query: str, threshold: float = 0.6, limit: int = 8) -> list[TextMatch]:
|
|
"""Fuzzy label search, best matches first."""
|
|
matches: list[TextMatch] = []
|
|
for element in self.elements:
|
|
score = text_similarity(query, element.text)
|
|
if score > threshold:
|
|
matches.append(TextMatch(element=element, score=score, matched_text=element.text))
|
|
matches.sort(key=lambda m: (-m.score, m.element.area))
|
|
return matches[:limit]
|
|
|
|
def exact_matches(self, query: str) -> list[ScreenElement]:
|
|
"""Elements whose label equals ``query`` after normalization.
|
|
|
|
Overlapping duplicates (the same label from the UI tree and from OCR
|
|
at the same place) collapse to one element, preferring the UI-tree
|
|
node because it carries structure the OCR box lacks.
|
|
"""
|
|
wanted = normalize_text(query)
|
|
if not wanted:
|
|
return []
|
|
hits = [e for e in self.elements if normalize_text(e.text) == wanted]
|
|
hits.sort(key=lambda e: (e.source != "xml", e.area))
|
|
unique: list[ScreenElement] = []
|
|
for element in hits:
|
|
if all(element.overlap_ratio(kept) < 0.5 for kept in unique):
|
|
unique.append(element)
|
|
return unique
|
|
|
|
def elements_at(self, x: int, y: int) -> list[ScreenElement]:
|
|
"""Elements covering pixel ``(x, y)``: UI-tree nodes innermost first, then OCR.
|
|
|
|
Mirrors the historical coordinate audit: UI-tree candidates stop at the
|
|
first interactive node (at most two are reported) so the answer names
|
|
what a tap would actually hit rather than every enclosing wrapper.
|
|
"""
|
|
covering = [e for e in self.elements if e.contains(x, y)]
|
|
xml = sorted((e for e in covering if e.source == "xml"), key=lambda e: e.area)
|
|
picked: list[ScreenElement] = []
|
|
for element in xml:
|
|
picked.append(element)
|
|
if element.interactive or len(picked) >= 2:
|
|
break
|
|
ocr = sorted((e for e in covering if e.source == "ocr"), key=lambda e: e.area)
|
|
return picked + ocr
|