feat: add plugin runtime and cross-device task orchestration
This commit is contained in:
1 parent
3ee4a5f7a8
commit
6663233ab8
82 files changed
+4147
-124
No files matched your search
@@ -274,7 +274,7 @@ class PlannerNode:
|
||||
screenshot_b64 = await controller.take_screenshot()
|
||||
|
||||
# Save to Data Engine if available to keep trace
|
||||
if self.ctx.data_engine:
|
||||
if self.ctx.data_engine and screenshot_b64:
|
||||
img_bytes = base64.b64decode(screenshot_b64)
|
||||
image_name = self.ctx.data_engine.get_or_create_image(img_bytes)
|
||||
saved_path = str(self.ctx.data_engine.get_image_path(image_name))
|
||||
@@ -284,7 +284,7 @@ class PlannerNode:
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to take screenshot for Initial Plan: {e}")
|
||||
|
||||
if not screenshot_b64:
|
||||
if not screenshot_b64 and getattr(self.ctx, "execution_target", "android") != "computer":
|
||||
logger.error("Planner failed to obtain a screenshot.")
|
||||
raise ValueError("Planner failed to obtain a screenshot.")
|
||||
|
||||
@@ -296,6 +296,11 @@ class PlannerNode:
|
||||
human_message = Template(human_content).render(**render_kwargs)
|
||||
|
||||
human_message_content = [{"type": "text", "text": human_message}]
|
||||
if getattr(self.ctx, "execution_target", "android") == "computer":
|
||||
screen = await UnifiedMobileController(self.ctx).get_screen_data()
|
||||
human_message_content.append(
|
||||
{"type": "text", "text": screen.observation_text + self.ctx.plugins.prompt()}
|
||||
)
|
||||
if screenshot_b64:
|
||||
human_message_content.append(
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user