feat: add plugin runtime and cross-device task orchestration

artemis/agents/checker/checker.py

@@ -317,7 +317,7 @@ def build_checker_tools(
         *get_history_tools(ctx),
         get_read_note_tool_pure(ctx),
     ]
-    if probes_enabled(ctx):
+    if getattr(ctx, "execution_target", "android") != "computer" and probes_enabled(ctx):
         tools.append(get_probe_tool(ctx))
     return tools
 
@@ -758,6 +758,8 @@ async def _capture_final_screen(ctx: ArtemisContext) -> tuple[str | None, str]:
 
         controller = UnifiedMobileController(ctx)
         device_data = await controller.get_screen_data()
+        if getattr(device_data, "platform", "android") == "computer":
+            return device_data.base64 or None, device_data.observation_text
         screenshot_b64 = device_data.base64
         xml_hierarchy = device_data.elements
         width = device_data.width

artemis/agents/flash/flash_runner.md

@@ -7,7 +7,7 @@
 
 
 # ROLE & OBJECTIVE
-You are an autonomous and highly efficient Android Device Execution Agent. Your goal is to accomplish the user's objective on the device.
+You are an autonomous UI execution agent. Accomplish the user's objective using the actual available device and plugin tools. Only Android action tools use normalized 0–1000 coordinates; plugin tools use their own documented coordinates and element indices.
 
 **Objective: {{ goal }}**
 
@@ -26,7 +26,7 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
 {% if "wait_for_delay" in available_tools %}- **Timed Waiting & Transitions**: When facing mid-transition animations, loading spinners, required watch durations, or ad countdown timers, use the precise `wait_for_delay` command to allow time to pass or the UI state to stabilize. Some transition states are subtle and may even resemble a phone bug or system freeze; it is always advisable to wait first, and only consider retrying or choosing an alternative path if the state remains broken.{% endif %}
 - **Time-Sensitive Tasks & Transient UI**: Because of the turn latency above, a target element that you see on the current screenshot (e.g., auto-dismissing overlays, temporary message bars, menus or control bars with short timeouts) may have already vanished by the time your physical click command reaches the device! This is one of the most common reasons why the UI does not change as expected after a single step. We call these **Time-Sensitive Tasks**.
   - *Mental Model*: When facing these tasks, any attempt to interact step-by-step (such as tapping once to wake up a menu/bar and waiting for the next turn to tap the target button) WILL fail because the element will auto-dismiss during your decision delay. You MUST combine the summon action on the trigger element (`Trigger`) and the target action (`Target`) into a single atomic sequence.
-  - *Chained Execution*: Use `click_sequence(sequence=[[trigger_x, trigger_y], [target_x, target_y]])` in `0-1000` normalized scale to execute both taps consecutively with a 50ms delay, catching the transient element before the timeout closes it.
+{% if "click_sequence" in available_tools %}  - *Chained Execution*: Use `click_sequence(sequence=[[trigger_x, trigger_y], [target_x, target_y]])` in `0-1000` normalized scale to execute both taps consecutively with a 50ms delay, catching the transient element before the timeout closes it.{% endif %}
 
 ---
 

artemis/agents/flash/runner.py

@@ -217,6 +217,15 @@ class FlashRunner:
 
             tools.append(VIDEO_ANALYZER_TOOL)
         tools.append(REPORT_TASK_STATUS_TOOL)
+        if getattr(self.ctx, "execution_target", None) == "computer":
+            from artemis.mcp.action_manifest import DEVICE_ACTIONS
+
+            tools = [
+                t
+                for t in tools
+                if t.name not in DEVICE_ACTIONS
+                and t.name not in {"ask_explorer", "video_analyzer", "click_sequence"}
+            ]
         # With an actuator installed, drop declarations for device actions the
         # backend does not implement (and append its extension tools); without
         # one, the full historical declaration set is kept.
@@ -225,6 +234,8 @@ class FlashRunner:
             from artemis.mcp.action_manifest import filter_declarations
 
             tools = filter_declarations(tools, actuator, "flash")
+        if getattr(self.ctx, "plugins", None):
+            tools.extend(self.ctx.plugins.declarations())
         return tools
 
     def _prune_intermediate_screenshots(self, messages: list[BaseMessage]) -> None:
@@ -305,7 +316,10 @@ class FlashRunner:
         prompt_path = Path(__file__).parent / "flash_runner.md"
         prompt_template = prompt_path.read_text(encoding="utf-8")
         available_tools = frozenset(t.name for t in tools_declaration)
-        return Template(prompt_template).render(goal=self.goal, available_tools=available_tools)
+        prompt = Template(prompt_template).render(goal=self.goal, available_tools=available_tools)
+        if getattr(self.ctx, "plugins", None):
+            prompt += self.ctx.plugins.prompt()
+        return prompt
 
     # ------------------------------------------------------------------
     # Per-turn helpers (observe / think)
@@ -612,6 +626,11 @@ class FlashRunner:
         elements. Nothing is captured for non-action tools.
         """
         post_img_bytes = exec_result.screenshot_bytes
+        if (
+            exec_result.metadata.get("plugin_id")
+            and getattr(self.ctx, "execution_target", "android") != "computer"
+        ):
+            return post_img_bytes
         if post_img_bytes or name not in action_names:
             return post_img_bytes
         if state is not None:
@@ -705,6 +724,15 @@ class FlashRunner:
             action_dict["normalized_start_coordinates"] = norm_start
             action_dict["normalized_end_coordinates"] = norm_end
         action_dict.update(metadata.get("target_semantics") or {})
+        if metadata.get("plugin_id"):
+            action_dict.update(
+                {
+                    key: metadata[key]
+                    for key in ("plugin_id", "plugin_version", "resource_ids", "plugin_status")
+                }
+            )
+            action_dict.pop("coordinates", None)
+            action_dict.pop(COORDINATE_SPACE_KEY, None)
         return action_dict
 
     def _record_action_step(
@@ -800,10 +828,39 @@ class FlashRunner:
         # Dynamic dispatch set: manifest device actions plus any backend
         # extension tools, so extension steps are recorded like actions.
         action_names = self.executor.action_tool_names
+        plugin_action = False
         try:
-            exec_result = await self.executor.execute(
+            operation = self.executor.execute(
                 name, args, tc_id, state, index_elements=index_elements
             )
+            from artemis.plugins.runtime import get_plugin_runtime
+
+            plugins = get_plugin_runtime(self.ctx)
+            if plugins and name in plugins.bindings:
+                plugin_action = True
+                # A phone observation cannot serve as the before-image for a desktop action.
+                pre_screenshot_bytes = plugins.last_images.get(plugins.bindings[name][0])
+                xml_list = ""
+            control = getattr(self.ctx, "task_control", None)
+            if plugins and name in plugins.tool_names and isinstance(control, TaskControl):
+                interrupted, exec_result = await control.run_interruptibly(
+                    operation,
+                    on_pause=getattr(self.ctx, "on_task_pause", None),
+                    on_resume=getattr(self.ctx, "on_task_resume", None),
+                )
+                if interrupted:
+                    messages.extend(
+                        tool_result_messages(
+                            tc_id,
+                            "操作被暂停或指导打断，结果不确定。先重新观察，不能重放旧动作。",
+                            name=name,
+                            status="error",
+                            llm=self._llm,
+                        )
+                    )
+                    return None, "操作已打断，必须重新观察。", action_sequence
+            else:
+                exec_result = await operation
 
             post_img_bytes = await self._capture_post_screenshot(
                 exec_result, name, action_names, state
@@ -869,7 +926,11 @@ class FlashRunner:
             # device actions always report text — their post screenshot is the
             # next observation tail, never a tool-message image. The image
             # carrier follows the model's provider (tool_result_messages).
-            raw_blocks = exec_result.raw_result if name not in action_names else None
+            raw_blocks = (
+                exec_result.raw_result
+                if name not in action_names or exec_result.metadata.get("plugin_id")
+                else None
+            )
             content = (
                 raw_blocks
                 if isinstance(raw_blocks, list) and raw_blocks
@@ -896,6 +957,15 @@ class FlashRunner:
                     status="error",
                 )
             )
+        if plugin_action and getattr(self.ctx, "execution_target", "android") != "computer":
+            # The loop's observation always belongs to Android in a mixed task.
+            # Plugin images travel in their labelled tool result and step record;
+            # never pair a desktop bitmap with the phone's element indexes.
+            shot_path, observation_image, observation_text = await capture_screenshot_and_parse_ui(
+                self.ctx, state, self.controller
+            )
+            state.latest_screenshot = shot_path
+            return observation_image, observation_text, action_sequence
         return pre_screenshot_bytes, xml_list, action_sequence
 
     async def _invoke_model(self, llm, current_tools: list, messages: list[BaseMessage]):
@@ -1156,7 +1226,7 @@ class FlashRunner:
             # S + F + A + tail: chunk compression and the scrub edge advance here.
             messages = ledger.render([tail])
             turn_base = len(messages) - 1
-            current_tools = report_only_tools if is_final else tools_declaration
+            current_tools = report_only_tools if is_final else self._get_tools()
 
             if isinstance(control, TaskControl):
                 interrupted, response = await control.run_interruptibly(

artemis/agents/operator/operator.py

@@ -183,6 +183,8 @@ class OperatorNode:
             available_device_actions,
         )
 
+        if getattr(self.ctx, "execution_target", None) == "computer":
+            return frozenset()
         actuator = getattr(self.ctx, "actuator", None)
         if actuator is not None and callable(getattr(actuator, "capabilities", None)):
             try:
@@ -535,6 +537,15 @@ class OperatorNode:
                     )
                 )
 
+            plugins = getattr(self.ctx, "plugins", None)
+            if plugins:
+                traced_tools[:] = [
+                    t
+                    for t in traced_tools
+                    if not t.name.startswith("plug_")
+                    and t.name not in {"plugin_load", "plugin_read_file"}
+                ]
+                traced_tools.extend(plugins.langchain_tools())
             bound_llm = active_llm.bind_tools(tools=traced_tools)
             interrupted, response = await self._await_or_pause(
                 invoke_llm_with_timeout_message(acomplete(bound_llm, current_messages))
@@ -650,6 +661,8 @@ class OperatorNode:
             # Manifest-driven, so a backend's extension actions are classified as
             # turn-ending device actions without touching this file.
             action_tool_names = sorted(self._available_device_actions())
+            if plugins:
+                action_tool_names.extend(plugins.bindings)
 
             def normalize_name(name: str) -> str:
                 return name.split(":")[-1] if ":" in name else name
@@ -707,7 +720,9 @@ class OperatorNode:
                                     tool_call_id=tc["id"],
                                     state=state,
                                 )
-                                if normalize_name(tool_name) in INTERRUPTIBLE_HELPERS:
+                                if normalize_name(tool_name) in INTERRUPTIBLE_HELPERS or (
+                                    plugins and tool_name in plugins.tool_names
+                                ):
                                     interrupted, result_obj = await self._await_or_pause(tool_call)
                                     if interrupted:
                                         span.result = "任务被暂停或指导打断；重新观察屏幕后继续"
@@ -1002,7 +1017,10 @@ class OperatorNode:
         fused_xml = state.latest_ui_hierarchy if hasattr(state, "latest_ui_hierarchy") else []
         state.latest_screenshot if hasattr(state, "latest_screenshot") else None
 
-        if not latest_screenshot_b64 or xml_hierarchy is None:
+        if (
+            not latest_screenshot_b64
+            and getattr(self.ctx, "execution_target", "android") != "computer"
+        ) or xml_hierarchy is None:
             logger.error(
                 "Operator requires perception data in state (populated by Perception node)"
             )
@@ -1034,6 +1052,8 @@ class OperatorNode:
             except (AttributeError, TypeError):
                 pass
         minimal_list, elements, labels = format_minimal_list_with_elements(fused_xml, width, height)
+        if getattr(self.ctx, "execution_target", "android") == "computer":
+            minimal_list = operator_raw_data.get("observation_text", "尚未取得电脑观察结果")
 
         # Save transient indexed_points and indexed_elements on the state instance
         state.indexed_points = [el["center"] for el in elements]
@@ -1079,6 +1099,10 @@ class OperatorNode:
             active_background_tasks=active_background_tasks,
             newly_finished_tasks=newly_finished_tasks,
         )
+        if getattr(self.ctx, "plugins", None):
+            from langchain_core.messages import SystemMessage
+
+            messages.insert(0, SystemMessage(content=self.ctx.plugins.prompt()))
 
         # 8. Invoke LLM Loop
         new_subagent_calls = []
@@ -1226,6 +1250,12 @@ class OperatorNode:
         tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
         args = tc["args"]
 
+        from artemis.plugins.runtime import get_plugin_runtime
+
+        plugins = get_plugin_runtime(self.ctx)
+        if plugins and tool_name in plugins.bindings:
+            return [{"action": tool_name, "args": args}], None
+
         # Get device resolution fallbacks
         width = 1080
         height = 2400

artemis/agents/operator/prompts.py

@@ -380,13 +380,18 @@ class ObservationPromptComponent(PromptComponent):
         latest_screenshot_b64 = kwargs.get("latest_screenshot_b64")
         minimal_list = kwargs.get("minimal_list")
 
-        builder.add_human_content("--- Current Screenshot ---")
-        builder.add_human_content(
-            {
-                "type": "image_url",
-                "image_url": {"url": f"data:image/jpeg;base64,{latest_screenshot_b64}"},
-            }
-        )
+        if latest_screenshot_b64:
+            builder.add_human_content("--- Current Screenshot ---")
+            builder.add_human_content(
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/jpeg;base64,{latest_screenshot_b64}"},
+                }
+            )
+        else:
+            builder.add_human_content(
+                "--- No screenshot available: use the text observation and request a target window state before coordinate actions. ---"
+            )
         builder.add_human_content(f"--- Visible UI Elements ---\n{minimal_list}")
 
 

artemis/agents/planner/planner.py

@@ -274,7 +274,7 @@ class PlannerNode:
                 screenshot_b64 = await controller.take_screenshot()
 
                 # Save to Data Engine if available to keep trace
-                if self.ctx.data_engine:
+                if self.ctx.data_engine and screenshot_b64:
                     img_bytes = base64.b64decode(screenshot_b64)
                     image_name = self.ctx.data_engine.get_or_create_image(img_bytes)
                     saved_path = str(self.ctx.data_engine.get_image_path(image_name))
@@ -284,7 +284,7 @@ class PlannerNode:
             except Exception as e:
                 logger.error(f"Failed to take screenshot for Initial Plan: {e}")
 
-        if not screenshot_b64:
+        if not screenshot_b64 and getattr(self.ctx, "execution_target", "android") != "computer":
             logger.error("Planner failed to obtain a screenshot.")
             raise ValueError("Planner failed to obtain a screenshot.")
 
@@ -296,6 +296,11 @@ class PlannerNode:
         human_message = Template(human_content).render(**render_kwargs)
 
         human_message_content = [{"type": "text", "text": human_message}]
+        if getattr(self.ctx, "execution_target", "android") == "computer":
+            screen = await UnifiedMobileController(self.ctx).get_screen_data()
+            human_message_content.append(
+                {"type": "text", "text": screen.observation_text + self.ctx.plugins.prompt()}
+            )
         if screenshot_b64:
             human_message_content.append(
                 {

artemis/agents/summarizer/summarizer.py

@@ -84,6 +84,16 @@ class SummarizerNode:
 
         action_name, action_args = self._extract_action(record, state, ctx=self.ctx)
         result = getattr(state, "last_execution_result", None)
+        if getattr(self.ctx, "execution_target", "android") == "mixed" and isinstance(result, dict):
+            if any(
+                action.get("plugin_id")
+                for action in result.get("execution", [])
+                if isinstance(action, dict)
+            ):
+                # The step began with a mobile observation. Never describe a phone-to-desktop
+                # image pair as a transition on one screen.
+                pre_bytes = None
+                post_bytes = None
         # format_result_clean reports only errors/repairs; a clean run keeps
         # the controller's bare status word.
         exec_outcome = format_result_clean(result)
@@ -181,7 +191,12 @@ class SummarizerNode:
         device = getattr(ctx, "device", None) if ctx is not None else None
         width = getattr(device, "device_width", None) or 1080
         height = getattr(device, "device_height", None) or 2400
-        actions = normalize_any_structure(actions, width, height)
+        actions = [
+            action
+            if str(action.get("action", "")).startswith("plug_")
+            else normalize_any_structure(action, width, height)
+            for action in actions
+        ]
         first = actions[0]
         action_name = first.get("action") or "unknown"
         action_args = {k: v for k, v in first.items() if k != "action"}

artemis/agents/validator/execution_loop.py

@@ -181,6 +181,33 @@ async def _process_action(
     outcome = _ActionOutcome(action_item=action_item)
     attempts_log = []
 
+    from artemis.plugins.runtime import get_plugin_runtime
+    from artemis.runtime.task_control import TaskControl
+
+    plugins = get_plugin_runtime(node.ctx)
+    if plugins and action_name in plugins.bindings:
+        operation = plugins.execute(action_name, action_item.get("args", {}), str(uuid.uuid4()))
+        control = getattr(node.ctx, "task_control", None)
+        if isinstance(control, TaskControl):
+            interrupted, result = await control.run_interruptibly(
+                operation,
+                on_pause=getattr(node.ctx, "on_task_pause", None),
+                on_resume=getattr(node.ctx, "on_task_resume", None),
+            )
+            if interrupted:
+                outcome.success = False
+                outcome.error_msg = "插件操作已打断，结果不确定；继续前必须重新观察。"
+                return outcome
+        else:
+            result = await operation
+        outcome.success = result.status == "success"
+        outcome.error_msg = "" if outcome.success else result.text_summary
+        outcome.action_item.update(result.metadata)
+        outcome.action_item["result"] = result.text_summary
+        if result.screenshot_bytes:
+            outcome.post_screenshot_b64 = base64.b64encode(result.screenshot_bytes).decode()
+        return outcome
+
     validation_passed = True
     if not burst:
         # 1. Pre-execution validation (Safety Net)
@@ -338,6 +365,8 @@ def _model_facing_action(ctx, action_item: dict) -> dict:
     The live item carries the controller's pixels; every phrase rendered for
     a model is normalized first, and the space marker makes that idempotent.
     """
+    if action_item.get("plugin_id"):
+        return action_item
     width = getattr(getattr(ctx, "device", None), "device_width", None) or 1080
     height = getattr(getattr(ctx, "device", None), "device_height", None) or 2400
     return normalize_action_dict(action_item, width, height)
@@ -419,11 +448,19 @@ async def run_validation_loop(node, state: State) -> dict:
             execution.append(outcome.action_item)
 
             post_image_name = None
-            if outcome.post_screenshot_b64:
+            plugin_action = bool(outcome.action_item.get("plugin_id"))
+            mixed_plugin_action = (
+                plugin_action and getattr(ctx, "execution_target", "android") == "mixed"
+            )
+            if outcome.post_screenshot_b64 and not mixed_plugin_action:
                 post_image_name = _absorb_post_screenshot(ctx, state, outcome.post_screenshot_b64)
                 last_screenshot_b64 = outcome.post_screenshot_b64
                 if post_image_name:
                     last_screenshot_name = post_image_name
+            elif mixed_plugin_action:
+                # Keep the Android step observation on its own resource. Plugin
+                # images remain attached to the tool/action trace and its exports.
+                post_image_name = outcome.action_item.get("post_image_name")
 
             _record_action_end(
                 ctx,

artemis/agents/validator/validator.py

@@ -86,12 +86,21 @@ class ValidatorNode:
         which on Windows was the *only* path that ever ran -- is gone: the server now
         lives in-process, so there is no spawn to fail and exactly one execution path.
         """
+        if getattr(self.ctx, "execution_target", None) == "computer":
+            from artemis.plugins.desktop import PluginReadSession
+
+            return PluginReadSession(self.ctx)
         return await get_action_session(self.ctx)
 
     def _parse_decisions(self, structured_decisions: str) -> tuple[list[dict] | None, str | None]:
         return action_execution.parse_decisions(structured_decisions)
 
     async def _get_initial_screenshot(self, session, state: State) -> tuple[str, str | None]:
+        if (
+            getattr(self.ctx, "execution_target", None) == "computer"
+            and not state.latest_screenshot
+        ):
+            return "", None
         return await action_execution.read_initial_screenshot(state)
 
     async def _execute_validation_loop(self, state: State):