feat: settle mobile observations before summarizing steps

This commit is contained in:
chenmingxuan committed 2026-09-29 00:39:05 +08:00
1 parent c818a09a49
commit 4465a2e199
12 files changed
+271 -9

No files matched your search

+17
View File
@@ -227,6 +227,8 @@ class VisualStepSummarizer(StepMemoryService):
action_key: str | None = None,
data_engine_step_id: UUID | str | None = None,
focus: dict[str, str] | None = None,
post_ui_elements_text: str | None = None,
observation_settled: bool | None = None,
) -> None:
"""Dispatches an asynchronous summarization task without blocking the caller.
@@ -257,6 +259,8 @@ class VisualStepSummarizer(StepMemoryService):
"exec_outcome": exec_outcome,
"data_engine_step_id": data_engine_step_id,
"focus": focus,
"post_ui_elements_text": post_ui_elements_text,
"observation_settled": observation_settled,
}
self.submit(key, payload, aliases=aliases)
@@ -409,6 +413,10 @@ class VisualStepSummarizer(StepMemoryService):
if verbatim and not self._phrase_carries(phrase, verbatim):
lead_lines.append(f"Action arguments (verbatim): {action_name}({verbatim})")
lead_lines.append(f"Controller Outcome: {exec_outcome}")
if input_data.get("observation_settled") is False:
lead_lines.append(
"Observation status: screen still changing; page state is not confirmed."
)
focus = input_data.get("focus")
if isinstance(focus, dict) and focus:
lead_lines.extend(["", render_focus_block(focus, dual=dual)])
@@ -457,6 +465,15 @@ class VisualStepSummarizer(StepMemoryService):
}
)
post_ui = input_data.get("post_ui_elements_text")
if post_bytes and isinstance(post_ui, str) and post_ui.strip():
content_blocks.append(
{
"type": "text",
"text": "--- [3] AFTER ACTION UI ELEMENTS ---\n" + post_ui[:8000],
}
)
messages: list[BaseMessage] = [
SystemMessage(content=rendered_prompt),
HumanMessage(content=content_blocks),