feat: add guided exploration and share user-started MCP backend

This commit is contained in:
chenmingxuan committed 2026-10-16 18:32:46 +08:00
1 parent f86db1ced6
commit ee256ad7ce
144 files changed
+8625 -2075

No files matched your search

+12 -15
View File
@@ -125,6 +125,18 @@ async def perception_node(state: State, ctx: ArtemisContext) -> dict:
ctx.data_engine.allocate_step_id()
logger.info(f"Pre-allocated step ID in Perception Node: {ctx.data_engine.current_step_id}")
controller = UnifiedMobileController(ctx)
# Observe before acknowledging guidance. A failed capture must not remove
# instructions that the Operator has never seen.
if not _should_skip_settling(state):
await asyncio.sleep(0.4)
from artemis.exploration.runtime import observe_or_pause
device_data = await observe_or_pause(ctx, controller)
latest_screenshot_b64 = device_data.base64
xml_hierarchy = device_data.elements
# Check for injected instruction without blocking event loop
injected_instruction = None
user_stop_requested = False
@@ -144,21 +156,6 @@ async def perception_node(state: State, ctx: ArtemisContext) -> dict:
# this moment stay protected until the next instruction.
_unprotect_current_check_lines(ctx)
controller = UnifiedMobileController(ctx)
# 1. Capture screenshot with hybrid settling strategy
if _should_skip_settling(state):
logger.info("Skipping settling. Retrieving screen data directly...")
device_data = await controller.get_screen_data()
latest_screenshot_b64 = device_data.base64
xml_hierarchy = device_data.elements
else:
logger.info("Screen settling required. Delaying 0.4s and capturing screen data...")
await asyncio.sleep(0.4)
device_data = await controller.get_screen_data()
latest_screenshot_b64 = device_data.base64
xml_hierarchy = device_data.elements
latest_screenshot_bytes = base64.b64decode(latest_screenshot_b64)
# Update context device dimensions dynamically