feat(memory): add incremental transcript compaction
This commit is contained in:
somew1nd committed
1 parent
cbe8f4f0c6
commit
41ae386a3a
65 files changed
+9971
-815
No files matched your search
@@ -12,27 +12,33 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
"""Pro-graph step summarizer node (M2: visual-transition lens dispatch).
|
||||
|
||||
from langchain_core.messages import (
|
||||
HumanMessage,
|
||||
SystemMessage,
|
||||
)
|
||||
Per the history-module redesign §5 (single step-level lens, confirmed
|
||||
2026-08-31) this node no longer generates a per-step text capsule with its own
|
||||
LLM call. It dispatches the step to the shared
|
||||
:class:`~artemis.memory.step_memory.StepMemoryService` visual-transition lens
|
||||
(``ctx.step_memory``): pre/post screenshots are pulled from the DataEngine by
|
||||
step id, the red action-overlay and the ``flash_summarizer.md`` neutral-wording
|
||||
contract apply unchanged, and the result lands as a versioned
|
||||
``source="visual_transition"`` summary write (M0 semantics).
|
||||
|
||||
The auxiliary agents' compiled view (``build_plan_and_history``) is untouched:
|
||||
a step whose summary is not ready yet falls back to detailed rendering
|
||||
(``task_tree.py`` semantics), so the transition is safe. The intent/strategy
|
||||
capsule moves up to the chunk-level ``StepCapsuleLens`` in M3.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import CURRENT_TRACE_ID, trace
|
||||
from artemis.data_engine.trace import trace
|
||||
from artemis.graph.state import State
|
||||
from artemis.graph.visibility import strict_state
|
||||
from artemis.services.llm import acomplete, get_llm
|
||||
from artemis.utils.coordinates import normalize_any_structure
|
||||
from artemis.memory import ensure_step_memory
|
||||
from artemis.utils.logger import get_logger
|
||||
from artemis.utils.task_tree import (
|
||||
build_plan_and_history,
|
||||
get_active_subgoal_hashes,
|
||||
)
|
||||
from artemis.utils.task_tree import format_result_clean
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -44,180 +50,90 @@ class SummarizerNode:
|
||||
@trace(type="agent", name="summarizer")
|
||||
async def __call__(self, state: State):
|
||||
state = strict_state(state, "summarizer")
|
||||
# 1. Trigger background task for summary and data engine update
|
||||
if self.ctx.data_engine and state.current_step_id:
|
||||
step_id = UUID(state.current_step_id)
|
||||
decisions = state.structured_decisions
|
||||
operator_raw_thinking = getattr(state, "operator_raw_thinking", None)
|
||||
operator_native_thinking = getattr(state, "operator_native_thinking", None)
|
||||
last_execution_result = getattr(state, "last_execution_result", None)
|
||||
|
||||
# Spawn background task
|
||||
trace_id = CURRENT_TRACE_ID.get()
|
||||
task = asyncio.create_task(
|
||||
self._generate_summary_and_update_engine(
|
||||
step_id,
|
||||
decisions,
|
||||
operator_raw_thinking,
|
||||
operator_native_thinking,
|
||||
last_execution_result,
|
||||
trace_id,
|
||||
),
|
||||
name=f"summarizer_step_{step_id}",
|
||||
)
|
||||
if hasattr(self.ctx, "background_tasks") and isinstance(
|
||||
self.ctx.background_tasks, list
|
||||
):
|
||||
self.ctx.background_tasks.append(task)
|
||||
|
||||
if not (self.ctx.data_engine and state.current_step_id):
|
||||
return {}
|
||||
try:
|
||||
self._dispatch_visual_summary(state)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to dispatch visual step summary: {e}")
|
||||
return {}
|
||||
|
||||
async def _generate_summary_and_update_engine(
|
||||
self,
|
||||
step_id,
|
||||
decisions,
|
||||
operator_raw_thinking,
|
||||
operator_native_thinking,
|
||||
last_execution_result,
|
||||
trace_id,
|
||||
):
|
||||
"""Background task to generate summary and update Data Engine with retries."""
|
||||
def _dispatch_visual_summary(self, state: State) -> None:
|
||||
engine = self.ctx.data_engine
|
||||
step_id = state.current_step_id
|
||||
|
||||
max_retries = 3
|
||||
retry_delay = 2
|
||||
step_number = None
|
||||
try:
|
||||
step_number = engine.get_step_number(step_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not resolve step number for {step_id}: {e}")
|
||||
|
||||
for attempt in range(max_retries):
|
||||
record = None
|
||||
if isinstance(step_number, int):
|
||||
try:
|
||||
logger.info(f"Generating summary for step {step_id} (attempt {attempt + 1})")
|
||||
|
||||
# Prepare prompt for summary
|
||||
llm = get_llm(ctx=self.ctx, name="summarizer")
|
||||
|
||||
prompt_path = Path(__file__).parent.joinpath("summarizer.md")
|
||||
if not prompt_path.exists():
|
||||
prompt = (
|
||||
"You are a summarizer. Summarize the latest action and"
|
||||
" its result in 1-2 concise sentences. Focus on: Action"
|
||||
" taken -> Result -> Observation."
|
||||
)
|
||||
else:
|
||||
prompt = prompt_path.read_text(encoding="utf-8")
|
||||
|
||||
# Normalize coordinates before presenting to the summarizer LLM
|
||||
width = (
|
||||
getattr(self.ctx.device, "device_width", 1080)
|
||||
if self.ctx and self.ctx.device
|
||||
else 1080
|
||||
)
|
||||
height = (
|
||||
getattr(self.ctx.device, "device_height", 2400)
|
||||
if self.ctx and self.ctx.device
|
||||
else 2400
|
||||
)
|
||||
|
||||
content = []
|
||||
|
||||
# Consistently use optimal build_plan_and_history to construct accurate context
|
||||
plan_and_history_str = ""
|
||||
if self.ctx.data_engine:
|
||||
try:
|
||||
friendly_steps = self.ctx.data_engine.get_agent_friendly_steps()
|
||||
if isinstance(friendly_steps, list):
|
||||
# 1. Trim history to only include steps up to the current step being summarized (inclusive)
|
||||
current_steps = []
|
||||
for s in friendly_steps:
|
||||
current_steps.append(s)
|
||||
if str(s.get("step_id")) == str(step_id):
|
||||
break
|
||||
|
||||
# 2. Read task plan to provide goal orientation to the summarizer
|
||||
task_plan = ""
|
||||
base_dir = getattr(self.ctx.data_engine, "base_dir", None)
|
||||
if base_dir and isinstance(base_dir, (str, Path)):
|
||||
notes_dir = Path(base_dir) / "notes"
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
if task_plan_path.exists():
|
||||
task_plan = task_plan_path.read_text(encoding="utf-8")
|
||||
|
||||
# 3. Unified rendering: concise for first N steps (with self-healing facts), detailed for current step (without redundant summary header)
|
||||
|
||||
subgoal_hash = "default"
|
||||
if task_plan:
|
||||
subgoal_hash, _ = get_active_subgoal_hashes(task_plan)
|
||||
|
||||
plan_and_history_str = build_plan_and_history(
|
||||
task_plan=task_plan,
|
||||
steps=current_steps,
|
||||
current_subgoal_hash=subgoal_hash,
|
||||
last_n_detailed=1,
|
||||
min_summaries=10,
|
||||
strict_milestone_pruning=False,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to build plan and history for Summarizer: {e}")
|
||||
|
||||
if plan_and_history_str:
|
||||
content.append(plan_and_history_str)
|
||||
else:
|
||||
# Fallback to standard raw fields if chronological trace couldn't be generated
|
||||
if decisions:
|
||||
try:
|
||||
dec_obj = json.loads(decisions)
|
||||
dec_obj = normalize_any_structure(dec_obj, width, height)
|
||||
content.append(
|
||||
f"Decisions made: {json.dumps(dec_obj, ensure_ascii=False)}"
|
||||
)
|
||||
except Exception:
|
||||
content.append(f"Decisions made: {decisions}")
|
||||
|
||||
if operator_raw_thinking:
|
||||
content.append(f"Operator explicit thoughts:\n{operator_raw_thinking}")
|
||||
|
||||
if operator_native_thinking:
|
||||
content.append(f"Operator native thoughts:\n{operator_native_thinking}")
|
||||
|
||||
messages = [
|
||||
SystemMessage(content=prompt),
|
||||
HumanMessage(content="\n\n".join(content)),
|
||||
]
|
||||
|
||||
aggregated_chunk = await acomplete(llm, messages)
|
||||
|
||||
generated_text = ""
|
||||
content = getattr(aggregated_chunk, "content", "") if aggregated_chunk else ""
|
||||
if isinstance(content, str):
|
||||
generated_text = content
|
||||
elif isinstance(content, list):
|
||||
for item in content:
|
||||
if isinstance(item, dict) and item.get("type") == "text":
|
||||
generated_text += item.get("text", "")
|
||||
|
||||
summary = generated_text.strip()
|
||||
logger.info(f"Summary generated: {summary}")
|
||||
|
||||
if aggregated_chunk and getattr(aggregated_chunk, "usage_metadata", None):
|
||||
usage = aggregated_chunk.usage_metadata
|
||||
cached_tokens = usage.get("input_token_details", {}).get("cache_read", 0)
|
||||
logger.info(
|
||||
"Summarizer LLM usage:"
|
||||
f" prompt_tokens={usage.get('input_tokens')}"
|
||||
f" (cached={cached_tokens}),"
|
||||
f" completion_tokens={usage.get('output_tokens')}"
|
||||
)
|
||||
|
||||
# Update Data Engine
|
||||
self.ctx.data_engine.update_step_summary(step_id, summary)
|
||||
logger.info(f"Data Engine updated with summary for step {step_id}")
|
||||
return # Success
|
||||
|
||||
record = engine.get_step_record(step_number)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to generate summary or update engine (attempt {attempt + 1}): {e}"
|
||||
)
|
||||
if attempt < max_retries - 1:
|
||||
await asyncio.sleep(retry_delay)
|
||||
retry_delay *= 2 # Exponential backoff
|
||||
else:
|
||||
logger.error(
|
||||
f"Max retries reached for step {step_id}. Summary generation failed."
|
||||
)
|
||||
logger.warning(f"Could not load step record {step_number}: {e}")
|
||||
|
||||
# Graceful degradation: a missing pre/post image simply dispatches with
|
||||
# None bytes — the lens renders a text-only transition account.
|
||||
pre_bytes = self._read_step_image(step_number, "pre")
|
||||
post_bytes = self._read_step_image(step_number, "post")
|
||||
|
||||
action_name, action_args = self._extract_action(record, state)
|
||||
result = getattr(state, "last_execution_result", None)
|
||||
# format_result_clean reports only errors/repairs; a clean run keeps
|
||||
# the controller's bare status word.
|
||||
exec_outcome = format_result_clean(result)
|
||||
if not exec_outcome:
|
||||
if isinstance(result, dict) and result.get("status"):
|
||||
exec_outcome = str(result["status"])
|
||||
else:
|
||||
exec_outcome = "unknown"
|
||||
|
||||
service = ensure_step_memory(self.ctx)
|
||||
service.dispatch(
|
||||
step_number=step_number if isinstance(step_number, int) else 0,
|
||||
action_name=action_name,
|
||||
action_args=action_args,
|
||||
pre_img_bytes=pre_bytes,
|
||||
post_img_bytes=post_bytes,
|
||||
exec_outcome=exec_outcome,
|
||||
data_engine_step_id=step_id,
|
||||
)
|
||||
logger.info(
|
||||
f"Dispatched visual transition summary for step {step_number} ({step_id})"
|
||||
)
|
||||
|
||||
def _read_step_image(self, step_number: int | None, which: str) -> bytes | None:
|
||||
if not isinstance(step_number, int):
|
||||
return None
|
||||
try:
|
||||
path = self.ctx.data_engine.get_step_image_path(step_number, which)
|
||||
if path is None:
|
||||
return None
|
||||
return path.read_bytes()
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not read {which} image for step {step_number}: {e}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_action(record: Any, state: State) -> tuple[str, dict]:
|
||||
actions = getattr(record, "action_taken", None) if record is not None else None
|
||||
if not actions:
|
||||
decisions = getattr(state, "structured_decisions", None)
|
||||
if decisions:
|
||||
try:
|
||||
actions = json.loads(decisions)
|
||||
except Exception:
|
||||
actions = None
|
||||
if isinstance(actions, dict):
|
||||
actions = [actions]
|
||||
if not isinstance(actions, list) or not actions or not isinstance(actions[0], dict):
|
||||
return "no_op", {}
|
||||
first = actions[0]
|
||||
action_name = first.get("action") or "unknown"
|
||||
action_args = {k: v for k, v in first.items() if k != "action"}
|
||||
if len(actions) > 1:
|
||||
action_args["additional_actions"] = actions[1:]
|
||||
return action_name, action_args
|
||||
Reference in new issue
Block a user