feat: add guided exploration and share user-started MCP backend

artemis/agents/checker/checker.py

@@ -865,6 +865,13 @@ async def run_final_check(
         SystemMessage(content=system_content),
         HumanMessage(content=content),
     ]
+    if getattr(ctx, "exploration_id", None):
+        from artemis.exploration.runtime import conversation_message, history_tool
+
+        # Questions, answers and demonstrations are separate from device steps.
+        # The verifier needs their actual order and the user's revised criteria.
+        messages.append(conversation_message(ctx))
+        tools.append(history_tool(ctx))
     with CheckerStreamCapture(ctx) as capture:
         try:
             report = await _run_check_loop(ctx, messages, tools, items)

artemis/agents/checker/flash.py

@@ -26,6 +26,9 @@ async def settle_flash_outcome(
     final_report_received: bool = False,
 ) -> dict:
     """Check once at exit, without operating the device or rerunning the task."""
+    stop_capture = getattr(ctx, "on_execution_review", None)
+    if stop_capture:
+        await stop_capture()
     status = str(execution_report.get("status") or "failed")
     result = None
     if status == "completed" or final_report_received:

artemis/agents/operator/operator.py

@@ -513,6 +513,9 @@ class OperatorNode:
         fallback_limit = settings.ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS
         active_llm = base_llm
         using_tool_fallback = False
+        exploration_plan_before = (
+            self._read_task_plan_text() if getattr(self.ctx, "exploration_id", None) else None
+        )
 
         for iteration in range(max_iterations):
             control = getattr(self.ctx, "task_control", None)
@@ -553,6 +556,10 @@ class OperatorNode:
             if interrupted:
                 return None, None, None, False
             assert response is not None
+            if getattr(self.ctx, "exploration_id", None):
+                from artemis.exploration.runtime import publish_response
+
+                publish_response(self.ctx, response)
             if isinstance(control, TaskControl) and has_pending_instruction(control.directory):
                 # The model answered a prompt that predates the user's new direction.
                 return None, None, None, False
@@ -687,6 +694,12 @@ class OperatorNode:
             if other_calls:
                 results = []
                 for tc in other_calls:
+                    if (
+                        isinstance(control, TaskControl)
+                        and (control.directory / "pause.request").exists()
+                    ):
+                        results.append(("已请求暂停，未执行该工具；恢复后重新观察。", "error"))
+                        continue
                     tool_name = tc["name"]
                     logger.info(f"Operator requested tool: {tool_name}")
                     if tool_name in ["ask_diagnoser", "video_analyzer"]:
@@ -862,7 +875,19 @@ class OperatorNode:
 
                         action_calls = []
 
-            if isinstance(control, TaskControl) and has_pending_instruction(control.directory):
+            if isinstance(control, TaskControl) and (
+                has_pending_instruction(control.directory)
+                or (control.directory / "pause.request").exists()
+            ):
+                current_messages.extend(tool_outputs)
+                for action in action_calls:
+                    current_messages.append(
+                        ToolMessage(
+                            tool_call_id=action["id"],
+                            content="探索已暂停或收到新指导，未执行该操作。",
+                            status="error",
+                        )
+                    )
                 return None, None, None, False
 
             if action_calls:
@@ -958,6 +983,32 @@ class OperatorNode:
             for tm in tool_outputs:
                 current_messages.append(tm)
 
+            if (
+                getattr(self.ctx, "exploration_id", None)
+                and other_calls
+                and not has_terminal_calls
+                and not other_tool_failed
+            ):
+                from artemis.exploration.runtime import needs_exploration_turn
+
+                plan_after = self._read_task_plan_text()
+                snapshot = parse_plan(plan_after or "")
+                if (
+                    plan_after != exploration_plan_before
+                    and snapshot.has_top_level
+                    and snapshot.all_top_level_done
+                    and (
+                        not snapshot.continuous_top_level
+                        or getattr(state, "user_stop_requested", False)
+                    )
+                    and not needs_exploration_turn(self.ctx)
+                ):
+                    # Completing the plan is useful progress. Return to the
+                    # convergence gate and final review instead of demanding
+                    # another device action just to end the operator turn.
+                    logger.info("Exploration plan completed; returning for final review.")
+                    break
+
             if action_calls and not validation_errors:
                 break
 
@@ -1103,6 +1154,10 @@ class OperatorNode:
             from langchain_core.messages import SystemMessage
 
             messages.insert(0, SystemMessage(content=self.ctx.plugins.prompt()))
+        if getattr(self.ctx, "exploration_id", None):
+            from artemis.exploration.runtime import exploration_messages
+
+            messages[0:0] = exploration_messages(self.ctx)
 
         # 8. Invoke LLM Loop
         new_subagent_calls = []

artemis/agents/planner/planner.py

@@ -333,6 +333,17 @@ class PlannerNode:
             llm_fallback = llm_fallback.bind_tools(all_tools)
 
         current_messages = messages
+        exploration_control = None
+        if getattr(self.ctx, "exploration_id", None):
+            from artemis.exploration.runtime import (
+                exploration_messages,
+                conversation_message,
+                publish_response,
+            )
+
+            exploration_control = getattr(self.ctx, "task_control", None)
+            current_messages[0:0] = exploration_messages(self.ctx, planning=True)
+        refresh_after_interrupt = False
         max_validation_attempts = 3
         max_iterations = 10
 
@@ -340,7 +351,36 @@ class PlannerNode:
             tool_called = False
 
             for _ in range(max_iterations):
-                response = await with_fallback(
+                if exploration_control is not None:
+                    from artemis.runtime.task_control import consume_instructions
+
+                    resumed = await exploration_control.checkpoint(
+                        on_pause=getattr(self.ctx, "on_task_pause", None),
+                        on_resume=getattr(self.ctx, "on_task_resume", None),
+                    )
+                    guidance = consume_instructions(exploration_control.directory)
+                    if resumed or guidance or refresh_after_interrupt:
+                        refresh_after_interrupt = False
+                        current_messages.append(conversation_message(self.ctx))
+                        screen = await UnifiedMobileController(self.ctx).get_screen_data()
+                        if screen.base64:
+                            current_messages.append(
+                                HumanMessage(
+                                    content=[
+                                        {
+                                            "type": "text",
+                                            "text": "恢复后的最新画面，以此重新确认当前状态。",
+                                        },
+                                        {
+                                            "type": "image_url",
+                                            "image_url": {
+                                                "url": f"data:image/png;base64,{screen.base64}"
+                                            },
+                                        },
+                                    ]
+                                )
+                            )
+                operation = with_fallback(
                     main_call=lambda: invoke_llm_with_timeout_message(
                         acomplete(llm, current_messages)
                     ),
@@ -348,13 +388,38 @@ class PlannerNode:
                         acomplete(llm_fallback, current_messages)
                     ),
                 )
+                if exploration_control is not None:
+                    interrupted, response = await exploration_control.run_interruptibly(
+                        operation,
+                        on_pause=getattr(self.ctx, "on_task_pause", None),
+                        on_resume=getattr(self.ctx, "on_task_resume", None),
+                    )
+                    if interrupted:
+                        refresh_after_interrupt = True
+                        continue
+                else:
+                    response = await operation
 
+                if exploration_control is not None:
+                    publish_response(self.ctx, response)
                 if response is None or not response.tool_calls:
                     break
 
                 current_messages.append(response)
 
                 for tc in response.tool_calls:
+                    if (
+                        exploration_control is not None
+                        and (exploration_control.directory / "pause.request").exists()
+                    ):
+                        current_messages.append(
+                            ToolMessage(
+                                tool_call_id=tc["id"],
+                                content="已请求暂停，未执行该工具；恢复后重新观察。",
+                                status="error",
+                            )
+                        )
+                        continue
                     tool_name = tc["name"]
                     logger.info(f"Planner called tool: {tool_name} with args: {tc['args']}")
                     if ":" in tool_name:

artemis/agents/video_analyzer/audio_runner.py

@@ -65,7 +65,7 @@ async def exec_analyze_audio_only(
     current_end = round(end_time, 1) if isinstance(end_time, (int, float)) else end_time
 
     if current_end is None:
-        controller = _va.get_controller(analyzer.ctx)
+        controller = getattr(analyzer, "media_source", None) or _va.get_controller(analyzer.ctx)
         async with _va.TRANSCODE_SEMAPHORE:
             metadata = await controller.extract_segment_metadata(current_start, None)
         duration = getattr(metadata, "duration_seconds", None)
@@ -225,7 +225,7 @@ async def _prepare_audio_media(
     analyzer, media: _AudioMedia, current_start: float, current_end: float
 ):
     """Extracts the segment and its audio track, tracking artifacts for cleanup."""
-    controller = _va.get_controller(analyzer.ctx)
+    controller = getattr(analyzer, "media_source", None) or _va.get_controller(analyzer.ctx)
     async with _va.TRANSCODE_SEMAPHORE:
         result = await controller.extract_segment_metadata(current_start, current_end)
     if not result.success or not result.video_path:

artemis/agents/video_analyzer/chunk_runner.py

@@ -53,8 +53,8 @@ async def exec_single_chunk(
 ) -> str:
     """Analyzes one closed chunk interval, with claim/lease and retry handling."""
     max_retries = analyzer.max_retries
-    current_start = round(start_time, 1) if isinstance(start_time, (int, float)) else start_time
-    current_end = round(end_time, 1) if isinstance(end_time, (int, float)) else end_time
+    current_start = round(start_time, 3) if isinstance(start_time, (int, float)) else start_time
+    current_end = round(end_time, 3) if isinstance(end_time, (int, float)) else end_time
 
     if not isinstance(current_start, (int, float)) or not isinstance(current_end, (int, float)):
         raise ValueError("Child video analysis requires a closed numeric interval")
@@ -241,7 +241,7 @@ async def _prepare_chunk_media(
     analyzer, media: _ChunkMedia, current_start: float, current_end: float
 ):
     """Extracts and compresses the chunk segment, tracking artifacts for cleanup."""
-    controller = _va.get_controller(analyzer.ctx)
+    controller = getattr(analyzer, "media_source", None) or _va.get_controller(analyzer.ctx)
     async with _va.TRANSCODE_SEMAPHORE:
         result = await controller.extract_segment_metadata(current_start, current_end)
     if not result.success or not result.video_path:

artemis/agents/video_analyzer/universal_engine.py

@@ -117,12 +117,22 @@ def _gather_universal_frames(
     """Extracts uniform keyframes plus dense action-window frames, merged by timestamp."""
     # Universal providers consume stills rather than the uploaded clip. Use
     # the raw extracted segment so slowdown/transcode PTS never skew labels.
-    keyframes = _va.extract_keyframes_from_video(
-        raw_path,
-        fps=1.0,
-        max_frames=30,
-        max_dimension=1080,
-    )
+    if getattr(analyzer, "media_source", None) is not None and end_time is not None:
+        # Saved demonstrations have no automatic action timestamps. Cover the
+        # entire clip, including its tail, instead of taking only its first 30s.
+        last = max(0.0, end_time - actual_start - 0.1)
+        interval = max(1.0, last / 29)
+        offsets = sorted({min(last, i * interval) for i in range(30)})
+        keyframes = _va.extract_frames_at_timestamps(
+            raw_path, offsets, max_frames=30, max_dimension=1080
+        )
+    else:
+        keyframes = _va.extract_keyframes_from_video(
+            raw_path,
+            fps=1.0,
+            max_frames=30,
+            max_dimension=1080,
+        )
     dense_offsets = analyzer._dense_action_offsets(
         start_time,
         end_time if end_time is not None else start_time,

artemis/agents/video_analyzer/video_analyzer.py

@@ -133,9 +133,10 @@ async def _invoke_with_retry(operation, label: str, max_attempts: int = 3):
 
 
 class VideoAnalyzer:
-    def __init__(self, ctx: ArtemisContext):
+    def __init__(self, ctx: ArtemisContext, *, media_source=None, blackboard=None):
         self.ctx = ctx
-        self.blackboard = get_video_blackboard(ctx)
+        self.media_source = media_source
+        self.blackboard = blackboard if blackboard is not None else get_video_blackboard(ctx)
         self.blackboard_entries = self.blackboard.list_ledger_entries()
         agent_config = getattr(ctx, "agent_config", None)
         self.video_config = getattr(agent_config, "video_analyzer", None)
@@ -304,6 +305,9 @@ class VideoAnalyzer:
     def _action_timestamps(self, start: float, end: float) -> list[float]:
         """Return test-relative timestamps for recorded actions in an interval."""
 
+        if self.media_source is not None:
+            return []
+
         engine = self.ctx.data_engine
         session_id = getattr(engine, "current_session_id", None) if engine else None
         storage = getattr(engine, "storage", None) if engine else None
@@ -441,7 +445,7 @@ class VideoAnalyzer:
     async def exec_extract_segment_metadata(
         self, start_time: float, end_time: float | None = None
     ) -> str:
-        controller = get_controller(self.ctx)
+        controller = self.media_source or get_controller(self.ctx)
         async with TRANSCODE_SEMAPHORE:
             result = await controller.extract_segment_metadata(start_time, end_time)
         if result.success and result.video_path:
@@ -497,7 +501,7 @@ class VideoAnalyzer:
         self, requested_start: float
     ) -> tuple[float, float | None, str | None]:
         """Resolve an open-ended request to a stable numeric interval (or an error)."""
-        controller = get_controller(self.ctx)
+        controller = self.media_source or get_controller(self.ctx)
         async with TRANSCODE_SEMAPHORE:
             metadata_result = await controller.extract_segment_metadata(requested_start, None)
         if not metadata_result.success: