fix: preserve exploration demonstration source and recording scope
This commit is contained in:
1 parent
f4cb55c030
commit
973ccae679
22 files changed
+624
-55
No files matched your search
@@ -985,10 +985,12 @@ async def run_final_check(
|
||||
HumanMessage(content=content),
|
||||
]
|
||||
if getattr(ctx, "exploration_id", None):
|
||||
from artemis.exploration.demonstrations import DEMONSTRATION_SOURCE_RULE
|
||||
from artemis.exploration.runtime import conversation_message, history_tool
|
||||
|
||||
# Questions, answers and demonstrations are separate from device steps.
|
||||
# The verifier needs their actual order and the user's revised criteria.
|
||||
messages.append(SystemMessage(content=DEMONSTRATION_SOURCE_RULE))
|
||||
messages.append(conversation_message(ctx))
|
||||
tools.append(history_tool(ctx))
|
||||
with CheckerStreamCapture(ctx) as capture:
|
||||
|
||||
@@ -295,13 +295,23 @@ class PlannerNode:
|
||||
|
||||
human_message = Template(human_content).render(**render_kwargs)
|
||||
|
||||
target = (
|
||||
"电脑" if getattr(self.ctx, "execution_target", "android") == "computer" else "安卓"
|
||||
)
|
||||
human_message_content = [{"type": "text", "text": human_message}]
|
||||
if getattr(self.ctx, "execution_target", "android") == "computer":
|
||||
if target == "电脑":
|
||||
screen = await UnifiedMobileController(self.ctx).get_screen_data()
|
||||
human_message_content.append(
|
||||
{"type": "text", "text": screen.observation_text + self.ctx.plugins.prompt()}
|
||||
)
|
||||
human_message_content.append({"type": "text", "text": screen.observation_text})
|
||||
plugins = getattr(self.ctx, "plugins", None)
|
||||
if plugins:
|
||||
human_message_content.append({"type": "text", "text": plugins.prompt()})
|
||||
if screenshot_b64:
|
||||
human_message_content.append(
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"当前主设备截图({target}),仅代表此来源,不决定其他步骤的执行平台。",
|
||||
}
|
||||
)
|
||||
human_message_content.append(
|
||||
{
|
||||
"type": "image_url",
|
||||
@@ -365,23 +375,23 @@ class PlannerNode:
|
||||
refresh_after_interrupt = False
|
||||
current_messages.append(conversation_message(self.ctx))
|
||||
screen = await UnifiedMobileController(self.ctx).get_screen_data()
|
||||
observation = [
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"恢复后的主设备观察({target}),仅代表此来源;"
|
||||
"其他来源的操作须通过对应工具观察,不能以此图更换示范平台。",
|
||||
}
|
||||
]
|
||||
if screen.observation_text:
|
||||
observation.append({"type": "text", "text": screen.observation_text})
|
||||
if screen.base64:
|
||||
current_messages.append(
|
||||
HumanMessage(
|
||||
content=[
|
||||
{
|
||||
"type": "text",
|
||||
"text": "恢复后的最新画面,以此重新确认当前状态。",
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:image/png;base64,{screen.base64}"
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
observation.append(
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/png;base64,{screen.base64}"},
|
||||
}
|
||||
)
|
||||
current_messages.append(HumanMessage(content=observation))
|
||||
operation = with_fallback(
|
||||
main_call=lambda: invoke_llm_with_timeout_message(
|
||||
acomplete(llm, current_messages)
|
||||
|
||||
Reference in new issue
Block a user