feat(android): add native controls and continuous multi-touch

artemis/utils/visualization.py

@@ -13,12 +13,14 @@
 # limitations under the License.
 
 import io
+import json
 import os
 import re
 from typing import Any
 
 from artemis.utils.coordinates import parse_swipe_parameters
 from artemis.utils.input_state import input_state
+from artemis.utils.native_control_state import informative_native_control, native_control_state
 from PIL import Image, ImageDraw, ImageFont
 
 
@@ -411,7 +413,14 @@ def format_minimal_list_with_elements(
         error = str(node.get("error") or "").strip()
         if value_evidence.get("showing_hint") is True:
             text = ""
-        shown = text.strip() or hint or ("(empty input)" if value_evidence else "")
+        native_state = native_control_state(node)
+        native_control = informative_native_control(native_state)
+        shown = (
+            text.strip()
+            or hint
+            or ("(empty input)" if value_evidence else "")
+            or ("(native control)" if native_control else "")
+        )
         if value_evidence:
             # OCR describes pixels, including placeholders. It cannot replace
             # the editable node's value evidence or its hit-test bounds.
@@ -423,10 +432,24 @@ def format_minimal_list_with_elements(
                 continue
 
             norm_bounds = normalize_bounds(left, top, right, bottom)
-            kind = "Text" if text.strip() else "Hint"
+            kind = (
+                "Text"
+                if text.strip()
+                else ("Control" if native_control and not hint and not value_evidence else "Hint")
+            )
             line = f"[{idx}] {kind}: '{shown}' | Bounds: {norm_bounds}"
             if resource_id := node.get("resource-id") or node.get("resource_id"):
                 line += f" | resource_id: '{resource_id}'"
+            if native_control:
+                line += f" | class: {node.get('class', '')}"
+                if package := node.get("package") or node.get("package_name"):
+                    line += f" | package_name: {json.dumps(package, ensure_ascii=False)}"
+                if desc := node.get("content-desc"):
+                    line += f" | content_desc: {json.dumps(desc, ensure_ascii=False)}"
+                if native_state.get("range"):
+                    line += f" | range: {json.dumps(native_state['range'], ensure_ascii=False)}"
+                if native_state.get("actions"):
+                    line += f" | actions: {','.join(native_state['actions'])}"
             if error:
                 line += f" | Error: '{error}'"
             if value_evidence:
@@ -450,6 +473,11 @@ def format_minimal_list_with_elements(
             if error:
                 element["error"] = error
             element.update(value_evidence)
+            element.update(native_state)
+            if native_control:
+                element["content_desc"] = node.get("content-desc") or ""
+                element["native_text"] = node.get("text") or ""
+                element["package_name"] = node.get("package") or node.get("package_name") or ""
             elements.append(element)
             labels.append(str(idx))
             added_items.append((shown, cx, cy))