feat: settle mobile observations before summarizing steps
This commit is contained in:
1 parent
c818a09a49
commit
4465a2e199
12 files changed
+271
-9
No files matched your search
@@ -227,6 +227,8 @@ class VisualStepSummarizer(StepMemoryService):
|
||||
action_key: str | None = None,
|
||||
data_engine_step_id: UUID | str | None = None,
|
||||
focus: dict[str, str] | None = None,
|
||||
post_ui_elements_text: str | None = None,
|
||||
observation_settled: bool | None = None,
|
||||
) -> None:
|
||||
"""Dispatches an asynchronous summarization task without blocking the caller.
|
||||
|
||||
@@ -257,6 +259,8 @@ class VisualStepSummarizer(StepMemoryService):
|
||||
"exec_outcome": exec_outcome,
|
||||
"data_engine_step_id": data_engine_step_id,
|
||||
"focus": focus,
|
||||
"post_ui_elements_text": post_ui_elements_text,
|
||||
"observation_settled": observation_settled,
|
||||
}
|
||||
self.submit(key, payload, aliases=aliases)
|
||||
|
||||
@@ -409,6 +413,10 @@ class VisualStepSummarizer(StepMemoryService):
|
||||
if verbatim and not self._phrase_carries(phrase, verbatim):
|
||||
lead_lines.append(f"Action arguments (verbatim): {action_name}({verbatim})")
|
||||
lead_lines.append(f"Controller Outcome: {exec_outcome}")
|
||||
if input_data.get("observation_settled") is False:
|
||||
lead_lines.append(
|
||||
"Observation status: screen still changing; page state is not confirmed."
|
||||
)
|
||||
focus = input_data.get("focus")
|
||||
if isinstance(focus, dict) and focus:
|
||||
lead_lines.extend(["", render_focus_block(focus, dual=dual)])
|
||||
@@ -457,6 +465,15 @@ class VisualStepSummarizer(StepMemoryService):
|
||||
}
|
||||
)
|
||||
|
||||
post_ui = input_data.get("post_ui_elements_text")
|
||||
if post_bytes and isinstance(post_ui, str) and post_ui.strip():
|
||||
content_blocks.append(
|
||||
{
|
||||
"type": "text",
|
||||
"text": "--- [3] AFTER ACTION UI ELEMENTS ---\n" + post_ui[:8000],
|
||||
}
|
||||
)
|
||||
|
||||
messages: list[BaseMessage] = [
|
||||
SystemMessage(content=rendered_prompt),
|
||||
HumanMessage(content=content_blocks),
|
||||
|
||||
Reference in new issue
Block a user