feat(memory): add incremental transcript compaction

artemis/data_engine/engine.py

@@ -14,6 +14,7 @@
 
 import asyncio
 from collections.abc import Callable
+import functools
 import hashlib
 import json
 import os
@@ -29,6 +30,7 @@ from artemis.context import ArtemisContext
 from artemis.data_engine.models import (
     BackgroundTaskRecord,
     FailedOutputRecord,
+    HistoryChunkRecord,
     ImageRecord,
     SessionMetadata,
     StepRecord,
@@ -48,6 +50,43 @@ logger = get_logger(__name__)
 
 _CURRENT_DATA_ENGINE = None
 
+# Overlay packages that never identify the app the user is working in.
+_FOREGROUND_APP_IGNORED = {"com.android.systemui"}
+
+
+def _derive_foreground_app(ui_tree: Any | None) -> str | None:
+    """Best-effort foreground package from a perception UI tree (M5).
+
+    Counts ``package``/``packageName`` attributes across the node list
+    (recursing into ``children``), ignores system-overlay packages, and
+    returns the most frequent value. Pure string work — never raises, never
+    touches the device.
+    """
+    counts: dict[str, int] = {}
+
+    def _visit(node: Any, depth: int = 0) -> None:
+        if depth > 50:
+            return
+        if isinstance(node, dict):
+            pkg = node.get("package") or node.get("packageName")
+            if isinstance(pkg, str) and pkg and pkg not in _FOREGROUND_APP_IGNORED:
+                counts[pkg] = counts.get(pkg, 0) + 1
+            children = node.get("children")
+            if isinstance(children, (list, tuple)):
+                for child in children:
+                    _visit(child, depth + 1)
+        elif isinstance(node, (list, tuple)):
+            for child in node:
+                _visit(child, depth + 1)
+
+    try:
+        _visit(ui_tree)
+    except Exception:
+        return None
+    if not counts:
+        return None
+    return max(counts.items(), key=lambda kv: kv[1])[0]
+
 
 class DataEngine:
     """Core engine for managing runtime data, storage, and streaming."""
@@ -538,6 +577,33 @@ class DataEngine:
         if pre_image_name and post_image_name and pre_image_name == post_image_name:
             post_image_name = None
 
+        # M5: persist the foreground app for the recall search surface (the
+        # parameter was historically accepted but never stored). The explicit
+        # parameter wins; otherwise it is derived best-effort from the UI
+        # tree's package attributes — a pure string scan, no device call.
+        extra_metadata = dict(extra_metadata or {})
+        try:
+            app = foreground_app or _derive_foreground_app(ui_tree)
+            if app:
+                extra_metadata.setdefault("foreground_app", app)
+        except Exception:
+            pass
+
+        # M4: stamp perceptual hashes for the local screen-similarity hint
+        # (pure PIL dHash; best-effort, computed synchronously to keep the
+        # record immutable once handed to the background writer).
+        try:
+            from artemis.utils.image_hash import dhash_hex
+
+            pre_dhash = dhash_hex(pre_screenshot_bytes)
+            if pre_dhash:
+                extra_metadata.setdefault("pre_image_dhash", pre_dhash)
+            post_dhash = dhash_hex(post_screenshot_bytes) if post_image_name else pre_dhash
+            if post_dhash:
+                extra_metadata.setdefault("post_image_dhash", post_dhash)
+        except Exception:
+            pass
+
         step = StepRecord(
             step_id=step_id,
             session_id=self.current_session_id,
@@ -550,7 +616,7 @@ class DataEngine:
             operator_raw_thinking=operator_raw_thinking,
             operator_native_thinking=operator_native_thinking,
             last_execution_result=last_execution_result,
-            extra_metadata=extra_metadata or {},
+            extra_metadata=extra_metadata,
         )
 
         def _run_background_storage():
@@ -971,8 +1037,24 @@ class DataEngine:
         path = self.get_image_path(image_name)
         return path if path.exists() else None
 
-    def update_step_summary(self, step_id: UUID | int | str, summary: str):
-        """Update the step with a concise summary in background."""
+    def update_step_summary(
+        self,
+        step_id: UUID | int | str,
+        summary: str | None,
+        *,
+        source: str | None = None,
+        version: int | None = None,
+        model: str | None = None,
+        status: str = "ready",
+    ):
+        """Update the step summary in background, with status/version metadata.
+
+        Writes ``summary_status`` / ``summary_source`` / ``summary_version`` /
+        ``summary_model`` into the step's ``extra_metadata`` alongside the
+        summary text; concurrent same-step writes are ordered by version in the
+        storage layer. ``status`` may be ``pending`` / ``ready`` / ``failed`` /
+        ``stale``; only ``ready`` writes carry summary text and publish SSE.
+        """
         target_uuid = None
         if isinstance(step_id, int):
             if self.current_session_id:
@@ -993,7 +1075,19 @@ class DataEngine:
             logger.debug(f"Could not resolve step_id for {step_id} to update summary.")
             return
 
-        self._run_in_background(self.storage.update_step_summary, target_uuid, summary)
+        self._run_in_background(
+            functools.partial(
+                self.storage.update_step_summary,
+                target_uuid,
+                summary,
+                source=source,
+                version=version,
+                model=model,
+                status=status,
+            )
+        )
+        if status != "ready" or summary is None:
+            return
         step_num = self.get_step_number(target_uuid if not isinstance(step_id, int) else step_id)
         payload = {
             "step_id": str(target_uuid),
@@ -1004,6 +1098,60 @@ class DataEngine:
             payload["step_number"] = step_num
         self._publish("step_updated", payload)
 
+    def record_history_chunk(
+        self,
+        *,
+        start_step_number: int,
+        end_step_number: int,
+        version: int,
+        status: str,
+        start_step_id: str | None = None,
+        end_step_id: str | None = None,
+        source_step_ids: list[str] | None = None,
+        subgoal_hash: str | None = None,
+        band1: dict[str, Any] | None = None,
+        band2: str | None = None,
+        band3: str | None = None,
+        rendered_text: str | None = None,
+    ) -> UUID | None:
+        """Persist one history-chunk version row in the background (append-only).
+
+        Chunk identity is the step range; callers own version numbering (a
+        newer version for the same range supersedes older ones at read time).
+        """
+        if not self.current_session_id:
+            return None
+        chunk = HistoryChunkRecord(
+            chunk_id=uuid4(),
+            session_id=self.current_session_id,
+            start_step_id=start_step_id,
+            end_step_id=end_step_id,
+            start_step_number=start_step_number,
+            end_step_number=end_step_number,
+            source_step_ids=source_step_ids or [],
+            subgoal_hash=subgoal_hash,
+            version=version,
+            status=status,
+            band1=band1 or {},
+            band2=band2,
+            band3=band3,
+            rendered_text=rendered_text,
+        )
+        self._run_in_background(self.storage.create_history_chunk, chunk)
+        return chunk.chunk_id
+
+    def get_history_chunks(self, *, all_versions: bool = False) -> list[HistoryChunkRecord]:
+        """History chunks of the active session (newest version per step range)."""
+        if not self.current_session_id:
+            return []
+        try:
+            return self.storage.get_history_chunks(
+                self.current_session_id, all_versions=all_versions
+            )
+        except Exception as e:
+            logger.error(f"Failed to fetch history chunks: {e}")
+            return []
+
     def update_step_thinking(self, step_id: UUID, operator_raw_thinking: str):
         """Update step's thinking in SQLite in background."""
         self._run_in_background(self.storage.update_step_thinking, step_id, operator_raw_thinking)

artemis/data_engine/models.py

@@ -55,6 +55,33 @@ class StepRecord(BaseModel):
     extra_metadata: dict[str, Any] = Field(default_factory=dict)
 
 
+class HistoryChunkRecord(BaseModel):
+    """One version of a compressed history segment (history redesign §3.3/§6.1).
+
+    Identity is the step *range* ``(session_id, start_step_number,
+    end_step_number)``; the milestone ``subgoal_hash`` is label metadata only.
+    Rows are append-only: a later write for the same range carries a higher
+    ``version`` (capsule readiness, post-hoc checkpoint annotations).
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    chunk_id: UUID | str
+    session_id: UUID | str
+    start_step_id: str | None = None
+    end_step_id: str | None = None
+    start_step_number: int
+    end_step_number: int
+    source_step_ids: list[str] = Field(default_factory=list)
+    subgoal_hash: str | None = None
+    version: int = 1
+    status: str = "pending"  # pending (band ③ only), ready (bands ①②③)
+    band1: dict[str, Any] = Field(default_factory=dict)  # synopsis + structured fields
+    band2: str | None = None  # interval narrative
+    band3: str | None = None  # mechanical per-step action ledger
+    rendered_text: str | None = None
+    created_at: float = Field(default_factory=time.time)
+
+
 class TraceRecord(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
     trace_id: UUID | str

artemis/data_engine/storage.py

@@ -27,6 +27,7 @@ from uuid import UUID
 from artemis.data_engine.models import (
     BackgroundTaskRecord,
     FailedOutputRecord,
+    HistoryChunkRecord,
     ImageRecord,
     SessionMetadata,
     StepRecord,
@@ -205,6 +206,26 @@ class StorageManager:
                     FOREIGN KEY(session_id) REFERENCES sessions(session_id)
                 )
             """)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS history_chunks (
+                    chunk_id TEXT PRIMARY KEY,
+                    session_id TEXT,
+                    start_step_id TEXT,
+                    end_step_id TEXT,
+                    start_step_number INTEGER,
+                    end_step_number INTEGER,
+                    source_step_ids TEXT,
+                    subgoal_hash TEXT,
+                    version INTEGER,
+                    status TEXT,
+                    band1 TEXT,
+                    band2 TEXT,
+                    band3 TEXT,
+                    rendered_text TEXT,
+                    created_at REAL,
+                    FOREIGN KEY(session_id) REFERENCES sessions(session_id)
+                )
+            """)
             conn.execute("""
                 CREATE TABLE IF NOT EXISTS video_recordings (
                     video_id TEXT PRIMARY KEY,
@@ -534,22 +555,179 @@ class StorageManager:
             )
             conn.commit()
 
-    def update_step_summary(self, step_id: UUID, summary: str):
-        """Update the summary for a step."""
+    def update_step_summary(
+        self,
+        step_id: UUID,
+        summary: str | None,
+        *,
+        source: str | None = None,
+        version: int | None = None,
+        model: str | None = None,
+        status: str = "ready",
+    ) -> bool:
+        """Versioned, status-carrying summary write (replaces the blind overwrite).
+
+        Writes ``summary_status`` / ``summary_source`` / ``summary_version`` /
+        ``summary_model`` into the step's ``extra_metadata``. Concurrent writes
+        to the same step are ordered by version: an explicit ``version`` lower
+        than the stored one is dropped; ``version=None`` auto-increments the
+        stored version. Status downgrades are refused without a newer explicit
+        version: ``pending`` never overwrites ``ready``/``failed``, and
+        ``failed`` never overwrites ``ready``. The ``summary`` column itself is
+        only touched by a ``ready`` write with a non-None summary.
+
+        Returns True when the write was applied, False when it was dropped as
+        stale (or the step does not exist).
+        """
+        with self._get_connection() as conn:
+            cursor = conn.execute(
+                "SELECT extra_metadata FROM steps WHERE step_id = ?",
+                (str(step_id),),
+            )
+            row = cursor.fetchone()
+            if row is None:
+                return False
+
+            try:
+                meta = json.loads(row["extra_metadata"]) if row["extra_metadata"] else {}
+                if not isinstance(meta, dict):
+                    meta = {}
+            except Exception:
+                meta = {}
+
+            current_version = meta.get("summary_version")
+            current_version = current_version if isinstance(current_version, int) else 0
+            current_status = meta.get("summary_status")
+
+            if version is not None:
+                if version < current_version:
+                    return False
+                new_version = version
+                is_newer = version > current_version
+            else:
+                new_version = current_version + 1
+                is_newer = False
+
+            # Status downgrades require an explicitly newer version.
+            if not is_newer:
+                if status == "pending" and current_status in ("ready", "failed"):
+                    return False
+                if status == "failed" and current_status == "ready":
+                    return False
+
+            meta["summary_status"] = status
+            meta["summary_version"] = new_version
+            if source is not None:
+                meta["summary_source"] = source
+            if model is not None:
+                meta["summary_model"] = model
+
+            if status == "ready" and summary is not None:
+                conn.execute(
+                    "UPDATE steps SET summary = ?, extra_metadata = ? WHERE step_id = ?",
+                    (summary, json.dumps(meta), str(step_id)),
+                )
+            else:
+                conn.execute(
+                    "UPDATE steps SET extra_metadata = ? WHERE step_id = ?",
+                    (json.dumps(meta), str(step_id)),
+                )
+            conn.commit()
+            return True
+
+    def create_history_chunk(self, chunk: HistoryChunkRecord):
+        """Insert one history-chunk version row (append-only; never updates)."""
         with self._get_connection() as conn:
             conn.execute(
                 """
-                UPDATE steps 
-                SET summary = ?
-                WHERE step_id = ?
+                INSERT INTO history_chunks (
+                    chunk_id, session_id, start_step_id, end_step_id,
+                    start_step_number, end_step_number, source_step_ids,
+                    subgoal_hash, version, status, band1, band2, band3,
+                    rendered_text, created_at
+                )
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                 """,
                 (
-                    summary,
-                    str(step_id),
+                    str(chunk.chunk_id),
+                    str(chunk.session_id),
+                    chunk.start_step_id,
+                    chunk.end_step_id,
+                    chunk.start_step_number,
+                    chunk.end_step_number,
+                    json.dumps(chunk.source_step_ids),
+                    chunk.subgoal_hash,
+                    chunk.version,
+                    chunk.status,
+                    json.dumps(chunk.band1, ensure_ascii=False, cls=SafeJSONEncoder),
+                    chunk.band2,
+                    chunk.band3,
+                    chunk.rendered_text,
+                    chunk.created_at,
                 ),
             )
             conn.commit()
 
+    def get_history_chunks(
+        self, session_id: UUID | str, *, all_versions: bool = False
+    ) -> list[HistoryChunkRecord]:
+        """History chunks of a session ordered by step range.
+
+        By default only the newest version per step range is returned; pass
+        ``all_versions=True`` for the full append-only trail (audit/tests).
+        """
+        with self._get_connection() as conn:
+            cursor = conn.execute(
+                """
+                SELECT * FROM history_chunks WHERE session_id = ?
+                ORDER BY start_step_number ASC, version ASC, created_at ASC
+                """,
+                (str(session_id),),
+            )
+            rows = cursor.fetchall()
+
+        records: list[HistoryChunkRecord] = []
+        for row in rows:
+            try:
+                source_ids = json.loads(row["source_step_ids"]) if row["source_step_ids"] else []
+            except Exception:
+                source_ids = []
+            try:
+                band1 = json.loads(row["band1"]) if row["band1"] else {}
+                if not isinstance(band1, dict):
+                    band1 = {}
+            except Exception:
+                band1 = {}
+            records.append(
+                HistoryChunkRecord(
+                    chunk_id=row["chunk_id"],
+                    session_id=row["session_id"],
+                    start_step_id=row["start_step_id"],
+                    end_step_id=row["end_step_id"],
+                    start_step_number=row["start_step_number"],
+                    end_step_number=row["end_step_number"],
+                    source_step_ids=source_ids,
+                    subgoal_hash=row["subgoal_hash"],
+                    version=row["version"],
+                    status=row["status"],
+                    band1=band1,
+                    band2=row["band2"],
+                    band3=row["band3"],
+                    rendered_text=row["rendered_text"],
+                    created_at=row["created_at"],
+                )
+            )
+        if all_versions:
+            return records
+
+        latest: dict[tuple[int, int], HistoryChunkRecord] = {}
+        for rec in records:
+            key = (rec.start_step_number, rec.end_step_number)
+            current = latest.get(key)
+            if current is None or rec.version >= current.version:
+                latest[key] = rec
+        return sorted(latest.values(), key=lambda r: r.start_step_number)
+
     def update_step_thinking(self, step_id: UUID, operator_raw_thinking: str):
         """Update the raw thinking process for a step."""
         with self._get_connection() as conn: