feat: add plugin runtime and cross-device task orchestration

artemis/agents/flash/flash_runner.md

@@ -7,7 +7,7 @@
 
 
 # ROLE & OBJECTIVE
-You are an autonomous and highly efficient Android Device Execution Agent. Your goal is to accomplish the user's objective on the device.
+You are an autonomous UI execution agent. Accomplish the user's objective using the actual available device and plugin tools. Only Android action tools use normalized 0–1000 coordinates; plugin tools use their own documented coordinates and element indices.
 
 **Objective: {{ goal }}**
 
@@ -26,7 +26,7 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
 {% if "wait_for_delay" in available_tools %}- **Timed Waiting & Transitions**: When facing mid-transition animations, loading spinners, required watch durations, or ad countdown timers, use the precise `wait_for_delay` command to allow time to pass or the UI state to stabilize. Some transition states are subtle and may even resemble a phone bug or system freeze; it is always advisable to wait first, and only consider retrying or choosing an alternative path if the state remains broken.{% endif %}
 - **Time-Sensitive Tasks & Transient UI**: Because of the turn latency above, a target element that you see on the current screenshot (e.g., auto-dismissing overlays, temporary message bars, menus or control bars with short timeouts) may have already vanished by the time your physical click command reaches the device! This is one of the most common reasons why the UI does not change as expected after a single step. We call these **Time-Sensitive Tasks**.
   - *Mental Model*: When facing these tasks, any attempt to interact step-by-step (such as tapping once to wake up a menu/bar and waiting for the next turn to tap the target button) WILL fail because the element will auto-dismiss during your decision delay. You MUST combine the summon action on the trigger element (`Trigger`) and the target action (`Target`) into a single atomic sequence.
-  - *Chained Execution*: Use `click_sequence(sequence=[[trigger_x, trigger_y], [target_x, target_y]])` in `0-1000` normalized scale to execute both taps consecutively with a 50ms delay, catching the transient element before the timeout closes it.
+{% if "click_sequence" in available_tools %}  - *Chained Execution*: Use `click_sequence(sequence=[[trigger_x, trigger_y], [target_x, target_y]])` in `0-1000` normalized scale to execute both taps consecutively with a 50ms delay, catching the transient element before the timeout closes it.{% endif %}
 
 ---
 

artemis/agents/flash/runner.py

@@ -217,6 +217,15 @@ class FlashRunner:
 
             tools.append(VIDEO_ANALYZER_TOOL)
         tools.append(REPORT_TASK_STATUS_TOOL)
+        if getattr(self.ctx, "execution_target", None) == "computer":
+            from artemis.mcp.action_manifest import DEVICE_ACTIONS
+
+            tools = [
+                t
+                for t in tools
+                if t.name not in DEVICE_ACTIONS
+                and t.name not in {"ask_explorer", "video_analyzer", "click_sequence"}
+            ]
         # With an actuator installed, drop declarations for device actions the
         # backend does not implement (and append its extension tools); without
         # one, the full historical declaration set is kept.
@@ -225,6 +234,8 @@ class FlashRunner:
             from artemis.mcp.action_manifest import filter_declarations
 
             tools = filter_declarations(tools, actuator, "flash")
+        if getattr(self.ctx, "plugins", None):
+            tools.extend(self.ctx.plugins.declarations())
         return tools
 
     def _prune_intermediate_screenshots(self, messages: list[BaseMessage]) -> None:
@@ -305,7 +316,10 @@ class FlashRunner:
         prompt_path = Path(__file__).parent / "flash_runner.md"
         prompt_template = prompt_path.read_text(encoding="utf-8")
         available_tools = frozenset(t.name for t in tools_declaration)
-        return Template(prompt_template).render(goal=self.goal, available_tools=available_tools)
+        prompt = Template(prompt_template).render(goal=self.goal, available_tools=available_tools)
+        if getattr(self.ctx, "plugins", None):
+            prompt += self.ctx.plugins.prompt()
+        return prompt
 
     # ------------------------------------------------------------------
     # Per-turn helpers (observe / think)
@@ -612,6 +626,11 @@ class FlashRunner:
         elements. Nothing is captured for non-action tools.
         """
         post_img_bytes = exec_result.screenshot_bytes
+        if (
+            exec_result.metadata.get("plugin_id")
+            and getattr(self.ctx, "execution_target", "android") != "computer"
+        ):
+            return post_img_bytes
         if post_img_bytes or name not in action_names:
             return post_img_bytes
         if state is not None:
@@ -705,6 +724,15 @@ class FlashRunner:
             action_dict["normalized_start_coordinates"] = norm_start
             action_dict["normalized_end_coordinates"] = norm_end
         action_dict.update(metadata.get("target_semantics") or {})
+        if metadata.get("plugin_id"):
+            action_dict.update(
+                {
+                    key: metadata[key]
+                    for key in ("plugin_id", "plugin_version", "resource_ids", "plugin_status")
+                }
+            )
+            action_dict.pop("coordinates", None)
+            action_dict.pop(COORDINATE_SPACE_KEY, None)
         return action_dict
 
     def _record_action_step(
@@ -800,10 +828,39 @@ class FlashRunner:
         # Dynamic dispatch set: manifest device actions plus any backend
         # extension tools, so extension steps are recorded like actions.
         action_names = self.executor.action_tool_names
+        plugin_action = False
         try:
-            exec_result = await self.executor.execute(
+            operation = self.executor.execute(
                 name, args, tc_id, state, index_elements=index_elements
             )
+            from artemis.plugins.runtime import get_plugin_runtime
+
+            plugins = get_plugin_runtime(self.ctx)
+            if plugins and name in plugins.bindings:
+                plugin_action = True
+                # A phone observation cannot serve as the before-image for a desktop action.
+                pre_screenshot_bytes = plugins.last_images.get(plugins.bindings[name][0])
+                xml_list = ""
+            control = getattr(self.ctx, "task_control", None)
+            if plugins and name in plugins.tool_names and isinstance(control, TaskControl):
+                interrupted, exec_result = await control.run_interruptibly(
+                    operation,
+                    on_pause=getattr(self.ctx, "on_task_pause", None),
+                    on_resume=getattr(self.ctx, "on_task_resume", None),
+                )
+                if interrupted:
+                    messages.extend(
+                        tool_result_messages(
+                            tc_id,
+                            "操作被暂停或指导打断，结果不确定。先重新观察，不能重放旧动作。",
+                            name=name,
+                            status="error",
+                            llm=self._llm,
+                        )
+                    )
+                    return None, "操作已打断，必须重新观察。", action_sequence
+            else:
+                exec_result = await operation
 
             post_img_bytes = await self._capture_post_screenshot(
                 exec_result, name, action_names, state
@@ -869,7 +926,11 @@ class FlashRunner:
             # device actions always report text — their post screenshot is the
             # next observation tail, never a tool-message image. The image
             # carrier follows the model's provider (tool_result_messages).
-            raw_blocks = exec_result.raw_result if name not in action_names else None
+            raw_blocks = (
+                exec_result.raw_result
+                if name not in action_names or exec_result.metadata.get("plugin_id")
+                else None
+            )
             content = (
                 raw_blocks
                 if isinstance(raw_blocks, list) and raw_blocks
@@ -896,6 +957,15 @@ class FlashRunner:
                     status="error",
                 )
             )
+        if plugin_action and getattr(self.ctx, "execution_target", "android") != "computer":
+            # The loop's observation always belongs to Android in a mixed task.
+            # Plugin images travel in their labelled tool result and step record;
+            # never pair a desktop bitmap with the phone's element indexes.
+            shot_path, observation_image, observation_text = await capture_screenshot_and_parse_ui(
+                self.ctx, state, self.controller
+            )
+            state.latest_screenshot = shot_path
+            return observation_image, observation_text, action_sequence
         return pre_screenshot_bytes, xml_list, action_sequence
 
     async def _invoke_model(self, llm, current_tools: list, messages: list[BaseMessage]):
@@ -1156,7 +1226,7 @@ class FlashRunner:
             # S + F + A + tail: chunk compression and the scrub edge advance here.
             messages = ledger.render([tail])
             turn_base = len(messages) - 1
-            current_tools = report_only_tools if is_final else tools_declaration
+            current_tools = report_only_tools if is_final else self._get_tools()
 
             if isinstance(control, TaskControl):
                 interrupted, response = await control.run_interruptibly(