merge(memory): integrate incremental transcript compaction

artemis/utils/element_hit_test.py

@@ -0,0 +1,123 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Record-time action semantics enrichment via element hit testing.
+
+An action addressed by bare coordinates carries no element semantics
+("tap(632,1180)" says nothing about what was tapped). Before such an action is
+persisted to the DataEngine, these helpers resolve the smallest indexed element
+covering the tap point on the pre-action frame and attach best-effort
+``target_text`` / ``target_class`` / ``target_resource_id`` fields, plus a
+``target_label_source`` marker distinguishing model-named targets from
+after-the-fact inference:
+
+- ``"index"``: the model addressed the element by its perception index — the
+  semantics come straight from the indexed element (highest confidence).
+- ``"hit_test"``: inferred by hit testing the coordinates against XML-derived
+  indexed elements on the pre-action frame.
+- ``"ocr"``: inferred by hit testing against OCR-derived indexed elements
+  (used only when no XML-derived element covers the point).
+- ``"none"``: no element covers the point (or no perception data was
+  available) — the action stays a bare-coordinate action.
+
+Enrichment is strictly best-effort: missing/malformed element data degrades to
+``"none"`` and never raises.
+"""
+
+from typing import Any
+
+from artemis.utils.visualization import parse_bounds
+
+_LABEL_SOURCE_KEY = "target_label_source"
+
+
+def _element_bounds(element: dict[str, Any]) -> tuple[int, int, int, int] | None:
+    """Returns (left, top, right, bottom) pixel bounds of an indexed element.
+
+    Accepts both list bounds (``[l, t, r, b]``, the shape produced by
+    ``format_minimal_list_with_elements``) and Android bounds strings
+    (``"[l,t][r,b]"``). Returns None for missing or malformed bounds.
+    """
+    bounds = element.get("bounds")
+    if isinstance(bounds, str):
+        return parse_bounds(bounds)
+    if isinstance(bounds, (list, tuple)) and len(bounds) == 4:
+        try:
+            left, top, right, bottom = (int(v) for v in bounds)
+        except (TypeError, ValueError):
+            return None
+        return left, top, right, bottom
+    return None
+
+
+def find_element_at_point(
+    elements: list[dict[str, Any]] | None, x: int, y: int
+) -> tuple[dict[str, Any] | None, str]:
+    """Finds the smallest indexed element whose bounds contain pixel point (x, y).
+
+    XML-derived elements are preferred over OCR-derived ones (OCR boxes carry
+    text but weaker structure); within each group the smallest covering area
+    wins. Returns ``(element, source)`` where source is ``"hit_test"``,
+    ``"ocr"``, or ``"none"`` (with element None).
+    """
+    best_xml: tuple[int, dict[str, Any]] | None = None
+    best_ocr: tuple[int, dict[str, Any]] | None = None
+
+    for element in elements or []:
+        if not isinstance(element, dict):
+            continue
+        rect = _element_bounds(element)
+        if rect is None:
+            continue
+        left, top, right, bottom = rect
+        if not (left <= x <= right and top <= y <= bottom):
+            continue
+        area = max(1, (right - left)) * max(1, (bottom - top))
+        if element.get("is_ocr"):
+            if best_ocr is None or area < best_ocr[0]:
+                best_ocr = (area, element)
+        else:
+            if best_xml is None or area < best_xml[0]:
+                best_xml = (area, element)
+
+    if best_xml is not None:
+        return best_xml[1], "hit_test"
+    if best_ocr is not None:
+        return best_ocr[1], "ocr"
+    return None, "none"
+
+
+def hit_test_semantics(
+    elements: list[dict[str, Any]] | None, x: int, y: int
+) -> dict[str, Any]:
+    """Best-effort semantic fields for a bare-coordinate action at pixel (x, y).
+
+    Always returns a dict containing ``target_label_source``; the
+    ``target_text`` / ``target_class`` / ``target_resource_id`` fields are only
+    present when the covering element carries them.
+    """
+    try:
+        element, source = find_element_at_point(elements, int(x), int(y))
+    except Exception:
+        return {_LABEL_SOURCE_KEY: "none"}
+    semantics: dict[str, Any] = {_LABEL_SOURCE_KEY: source}
+    if element is None:
+        return semantics
+    if element.get("text"):
+        semantics["target_text"] = element["text"]
+    if element.get("class"):
+        semantics["target_class"] = element["class"]
+    if element.get("resource_id"):
+        semantics["target_resource_id"] = element["resource_id"]
+    return semantics

artemis/utils/image_hash.py

@@ -0,0 +1,81 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Pure-PIL perceptual hashing (dHash) for screen-similarity hints (M4).
+
+A 64-bit difference hash: grayscale, resize to 9x8, compare each pixel to its
+right neighbor. Robust to JPEG re-encoding and mild rendering noise, cheap to
+compute (one small resize), and comparable with an integer Hamming distance —
+no model call, no historical image bytes needed at compare time.
+
+Hashes are stored per step in ``extra_metadata`` at record time
+(``pre_image_dhash`` / ``post_image_dhash``, 16 hex chars) and compared
+against the current screenshot before prompt construction.
+"""
+
+import io
+
+from artemis.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+#: Grid width of the dHash (the hash carries HASH_SIZE * HASH_SIZE bits).
+HASH_SIZE = 8
+
+
+def dhash_hex(image_bytes: bytes | None, hash_size: int = HASH_SIZE) -> str | None:
+    """64-bit dHash of an encoded image, as a 16-char hex string.
+
+    Best-effort: returns ``None`` on any failure (missing/corrupt bytes,
+    Pillow unavailable) — perceptual hashing is an enhancement, never a
+    dependency.
+    """
+    if not image_bytes:
+        return None
+    try:
+        from PIL import Image
+
+        with Image.open(io.BytesIO(image_bytes)) as img:
+            # JPEG fast-path: DCT-scaled draft decode keeps the sync cost of
+            # hashing at record time to a few milliseconds. The draft is only
+            # an intermediate resolution; the final resize below fixes the
+            # grid, so hashes stay comparable across code paths.
+            try:
+                img.draft("L", (hash_size * 8, hash_size * 8))
+            except Exception:
+                pass
+            small = img.convert("L").resize(
+                (hash_size + 1, hash_size), Image.Resampling.LANCZOS
+            )
+            pixels = list(small.getdata())
+        bits = 0
+        for row in range(hash_size):
+            for col in range(hash_size):
+                left = pixels[row * (hash_size + 1) + col]
+                right = pixels[row * (hash_size + 1) + col + 1]
+                bits = (bits << 1) | (1 if left > right else 0)
+        return f"{bits:0{hash_size * hash_size // 4}x}"
+    except Exception as e:
+        logger.debug(f"dhash_hex failed: {e}")
+        return None
+
+
+def hamming_distance_hex(hash_a: str | None, hash_b: str | None) -> int | None:
+    """Bit-level Hamming distance between two hex hashes; None if incomparable."""
+    if not hash_a or not hash_b or len(hash_a) != len(hash_b):
+        return None
+    try:
+        return bin(int(hash_a, 16) ^ int(hash_b, 16)).count("1")
+    except ValueError:
+        return None

artemis/utils/task_tree.py

@@ -801,14 +801,22 @@ def build_plan_and_history(
     keep_subgoal_hashes: set | None = None,
     min_summaries: int = 5,
     last_n_detailed: int = 1,
-    detailed_subgoal_hashes: set | None = None,
     all_detailed: bool = False,
     strict_milestone_pruning: bool = False,
     recent_window_size: int = 3,
-    chronological_last_step: bool = False,
     for_failure_analyzer: bool = False,
+    chunks: list | None = None,
 ) -> str:
-    """Builds a clean plan list and separate flat chronological execution history with adaptive compression."""
+    """Builds a clean plan list and separate flat chronological execution history with adaptive compression.
+
+    ``chunks`` (optional, M4): pre-rendered history-chunk blocks — each a dict
+    with ``start_step_number``/``end_step_number``/``text``. When provided,
+    the chunk blocks are emitted right after the history header and the
+    per-step lines inside any chunked range are dropped (the chunk block
+    replaces them), except steps that must render detailed (the recent
+    window / the most-recent step stay untouched). When ``chunks`` is
+    ``None`` or empty the output is byte-identical to the pre-M4 rendering.
+    """
     # 1. Plan Checklist Section
     output_parts = []
 
@@ -900,6 +908,26 @@ def build_plan_and_history(
             # Ensure the last step is always in detailed_step_ids
             detailed_step_ids.add(full_info_step_id)
 
+        # M4: chunk blocks replace the per-step lines of already-chunked
+        # ranges. Chunks only ever cover frozen (old) turns, so they are
+        # emitted first, in chronological order; steps that must render
+        # detailed (recent window / most-recent step) are never suppressed.
+        chunked_ranges: list[tuple[int, int]] = []
+        if chunks:
+            for chunk in sorted(
+                chunks, key=lambda c: c.get("start_step_number") or 0
+            ):
+                text = chunk.get("text")
+                if not text:
+                    continue
+                try:
+                    start_n = int(chunk.get("start_step_number"))
+                    end_n = int(chunk.get("end_step_number"))
+                except (TypeError, ValueError):
+                    continue
+                chunked_ranges.append((start_n, end_n))
+                output_parts.append(text)
+
         for step in steps:
             step_id = step["step_id"]
 
@@ -907,6 +935,13 @@ def build_plan_and_history(
             if step_id not in visible_step_ids:
                 continue
 
+            if chunked_ranges and step_id not in detailed_step_ids and step_id != full_info_step_id:
+                step_number = step.get("step_number")
+                if isinstance(step_number, int) and any(
+                    start_n <= step_number <= end_n for start_n, end_n in chunked_ranges
+                ):
+                    continue  # represented by its chunk block above
+
             relative_time = step.get("relative_time") or "N/A"
             summary = step.get("summary")
             action = step.get("action_taken")