feat: add guided exploration and share user-started MCP backend
This commit is contained in:
1 parent
f86db1ced6
commit
ee256ad7ce
144 files changed
+8625
-2075
No files matched your search
+12
-15
@@ -125,6 +125,18 @@ async def perception_node(state: State, ctx: ArtemisContext) -> dict:
|
||||
ctx.data_engine.allocate_step_id()
|
||||
logger.info(f"Pre-allocated step ID in Perception Node: {ctx.data_engine.current_step_id}")
|
||||
|
||||
controller = UnifiedMobileController(ctx)
|
||||
|
||||
# Observe before acknowledging guidance. A failed capture must not remove
|
||||
# instructions that the Operator has never seen.
|
||||
if not _should_skip_settling(state):
|
||||
await asyncio.sleep(0.4)
|
||||
from artemis.exploration.runtime import observe_or_pause
|
||||
|
||||
device_data = await observe_or_pause(ctx, controller)
|
||||
latest_screenshot_b64 = device_data.base64
|
||||
xml_hierarchy = device_data.elements
|
||||
|
||||
# Check for injected instruction without blocking event loop
|
||||
injected_instruction = None
|
||||
user_stop_requested = False
|
||||
@@ -144,21 +156,6 @@ async def perception_node(state: State, ctx: ArtemisContext) -> dict:
|
||||
# this moment stay protected until the next instruction.
|
||||
_unprotect_current_check_lines(ctx)
|
||||
|
||||
controller = UnifiedMobileController(ctx)
|
||||
|
||||
# 1. Capture screenshot with hybrid settling strategy
|
||||
if _should_skip_settling(state):
|
||||
logger.info("Skipping settling. Retrieving screen data directly...")
|
||||
device_data = await controller.get_screen_data()
|
||||
latest_screenshot_b64 = device_data.base64
|
||||
xml_hierarchy = device_data.elements
|
||||
else:
|
||||
logger.info("Screen settling required. Delaying 0.4s and capturing screen data...")
|
||||
await asyncio.sleep(0.4)
|
||||
device_data = await controller.get_screen_data()
|
||||
latest_screenshot_b64 = device_data.base64
|
||||
xml_hierarchy = device_data.elements
|
||||
|
||||
latest_screenshot_bytes = base64.b64decode(latest_screenshot_b64)
|
||||
|
||||
# Update context device dimensions dynamically
|
||||
|
||||
Reference in new issue
Block a user