feat: add plugin runtime and cross-device task orchestration

This commit is contained in:
chenmingxuan committed 2026-10-14 20:21:44 +08:00
1 parent 3ee4a5f7a8
commit 6663233ab8
82 files changed
+4147 -124

No files matched your search

+7 -2
View File
@@ -274,7 +274,7 @@ class PlannerNode:
screenshot_b64 = await controller.take_screenshot()
# Save to Data Engine if available to keep trace
if self.ctx.data_engine:
if self.ctx.data_engine and screenshot_b64:
img_bytes = base64.b64decode(screenshot_b64)
image_name = self.ctx.data_engine.get_or_create_image(img_bytes)
saved_path = str(self.ctx.data_engine.get_image_path(image_name))
@@ -284,7 +284,7 @@ class PlannerNode:
except Exception as e:
logger.error(f"Failed to take screenshot for Initial Plan: {e}")
if not screenshot_b64:
if not screenshot_b64 and getattr(self.ctx, "execution_target", "android") != "computer":
logger.error("Planner failed to obtain a screenshot.")
raise ValueError("Planner failed to obtain a screenshot.")
@@ -296,6 +296,11 @@ class PlannerNode:
human_message = Template(human_content).render(**render_kwargs)
human_message_content = [{"type": "text", "text": human_message}]
if getattr(self.ctx, "execution_target", "android") == "computer":
screen = await UnifiedMobileController(self.ctx).get_screen_data()
human_message_content.append(
{"type": "text", "text": screen.observation_text + self.ctx.plugins.prompt()}
)
if screenshot_b64:
human_message_content.append(
{