feat: improve task execution, verification and packaging

Add execution incidents, action bursts and shared Flash history.

Expose verification and Explorer settings with streamed check results.

Support graceful cancellation, recording recovery and bundled UI resources.

Add CI quality checks and clean up comments and documentation.

artemis/agents/explorer/universal_runner.py

@@ -16,15 +16,15 @@
 
 Split out of ``artemis.agents.explorer.explorer``: the ``_run_universal``
 reasoning loop, packaged as a mixin consumed by ``Explorer``.  Patched
-collaborators (``get_llm``, ``is_ocr_configured``, ``logger``,
-``UNIVERSAL_EXPLORER_TOOLS``) are resolved through the facade module at call
-time; see ``artemis.agents.explorer._facade``.
+collaborators (``get_llm``, ``logger``, ``UNIVERSAL_EXPLORER_TOOLS``) are
+resolved through the facade module at call time; see
+``artemis.agents.explorer._facade``.
 """
 
 import asyncio
 import base64
 import json
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from langchain_core.messages import (
     BaseMessage,
@@ -34,46 +34,74 @@ from langchain_core.messages import (
 )
 
 from artemis.agents.explorer._facade import facade
+from artemis.agents.explorer.geometry import is_valid_norm_point
+from artemis.agents.explorer.tiers import SUBMIT_TOOL
 from artemis.graph.state import State
 
 
 class UniversalRunnerMixin:
     """Universal-engine reasoning loop of :class:`Explorer`."""
 
+    if TYPE_CHECKING:
+        from artemis.context import ArtemisContext
+
+        ctx: ArtemisContext
+        denylisted_tools: set[str]
+
+        def _hidden_tool_names(self) -> set[str]: ...
+
+        def _enrich_candidates(self, candidates: list[Any]) -> list[Any]: ...
+
+        async def exec_ask_perception_tool(
+            self, search_query: str, nx: int, ny: int, detect_queries: list[str]
+        ) -> dict[str, Any]: ...
+
+        async def exec_detect_objects(
+            self, queries: list[str], target_image_id: str = "img_0"
+        ) -> dict[str, Any]: ...
+
+        async def exec_ask_image_processor(
+            self, instruction: str, target_image_id: str = "img_0"
+        ) -> dict[str, Any]: ...
+
+        async def exec_get_ocr_list(self) -> dict[str, Any]: ...
+
+        async def exec_inspect_region(
+            self, x_min: int, y_min: int, x_max: int, y_max: int, zoom_factor: float = 2.0
+        ) -> dict[str, Any]: ...
+
+    def _universal_exposed_tools(self) -> list[dict[str, Any]]:
+        """OpenAI-style tool schemas the model may see (tier-hidden ones excluded)."""
+        _ex = facade()
+        hidden = self._hidden_tool_names()
+        return [t for t in _ex.UNIVERSAL_EXPLORER_TOOLS if t["function"]["name"] not in hidden]
+
     async def _run_universal(
         self,
+        *,
         query: str,
         context_feedback: str,
         screenshot_path: str,
         state: State,
         minimal_list: str,
-        version: str,
         prompt_template: str,
-        max_iterations: int,
+        max_turns: int,
     ) -> str:
-        """Executes Explorer reasoning loop via Universal LangChain ChatModel."""
+        """Executes the Explorer reasoning loop via a universal LangChain ChatModel."""
         _ex = facade()
         llm = _ex.get_llm(self.ctx, name="explorer")
 
-        # Filter universal tools based on denylist and OCR configuration
-        denylisted = set(self.denylisted_tools)
-        if not _ex.is_ocr_configured():
-            denylisted.add("get_ocr_list")
-        exposed_tools = [
-            t for t in _ex.UNIVERSAL_EXPLORER_TOOLS if t["function"]["name"] not in denylisted
-        ]
+        exposed_tools = self._universal_exposed_tools()
         bound_llm = llm.bind_tools(exposed_tools)
 
         messages = self._build_universal_messages(
             query, context_feedback, screenshot_path, minimal_list, prompt_template
         )
 
-        iterations = 0
-        agent_outcome = ""
-
-        while iterations < max_iterations:
-            iterations += 1
-            is_final_turn = iterations == max_iterations
+        turn = 0
+        while turn < max_turns:
+            turn += 1
+            is_final_turn = turn == max_turns
 
             if is_final_turn:
                 messages.append(
@@ -85,7 +113,7 @@ class UniversalRunnerMixin:
                     )
                 )
                 submit_only_tools = [
-                    t for t in exposed_tools if t["function"]["name"] == "submit_answer"
+                    t for t in exposed_tools if t["function"]["name"] == SUBMIT_TOOL
                 ]
                 current_llm = llm.bind_tools(submit_only_tools) if submit_only_tools else bound_llm
             else:
@@ -96,9 +124,7 @@ class UniversalRunnerMixin:
 
             tool_calls = getattr(response, "tool_calls", None) or []
             if not tool_calls:
-                _ex.logger.warning(
-                    f"Universal Explorer iteration {iterations}: No tool calls generated."
-                )
+                _ex.logger.warning(f"Universal Explorer turn {turn}: No tool calls generated.")
                 if is_final_turn:
                     return json.dumps(
                         {
@@ -109,27 +135,19 @@ class UniversalRunnerMixin:
                     )
                 continue
 
-            # Check for submit_answer
             submit_outcome = self._universal_submit_outcome(tool_calls)
             if submit_outcome is not None:
-                agent_outcome = submit_outcome
-                return agent_outcome
+                return submit_outcome
 
-            # Execute non-submit tools
-            await self._universal_dispatch_tools(tool_calls, messages, iterations)
+            await self._universal_dispatch_tools(tool_calls, messages, turn)
 
-        if not agent_outcome:
-            agent_outcome = json.dumps(
-                {
-                    "candidates": [],
-                    "fallback_message": (
-                        "Explorer reached max iterations without finding candidates."
-                    ),
-                },
-                ensure_ascii=False,
-            )
-
-        return agent_outcome
+        return json.dumps(
+            {
+                "candidates": [],
+                "fallback_message": "Explorer reached max iterations without finding candidates.",
+            },
+            ensure_ascii=False,
+        )
 
     def _build_universal_messages(
         self,
@@ -167,7 +185,7 @@ class UniversalRunnerMixin:
 
     def _universal_submit_outcome(self, tool_calls: list) -> str | None:
         """Returns the validated submit_answer outcome JSON, or None if absent."""
-        submit_call = next((tc for tc in tool_calls if tc.get("name") == "submit_answer"), None)
+        submit_call = next((tc for tc in tool_calls if tc.get("name") == SUBMIT_TOOL), None)
         if not submit_call:
             return None
 
@@ -175,36 +193,102 @@ class UniversalRunnerMixin:
         candidates = args.get("candidates", [])
         fallback_message = args.get("fallback_message", "")
 
-        # Validate coordinates
-        valid_candidates = []
-        for cand in candidates:
-            if isinstance(cand, dict) and "coords" in cand:
-                coords = cand["coords"]
-                if isinstance(coords, list) and len(coords) == 2:
-                    try:
-                        nx, ny = int(coords[0]), int(coords[1])
-                        if 0 <= nx <= 1000 and 0 <= ny <= 1000:
-                            valid_candidates.append(cand)
-                    except (ValueError, TypeError):
-                        pass
+        valid_candidates = self._enrich_candidates(
+            [
+                cand
+                for cand in candidates
+                if isinstance(cand, dict) and is_valid_norm_point(cand.get("coords"))
+            ]
+        )
 
         return json.dumps(
-            {
-                "candidates": valid_candidates,
-                "fallback_message": fallback_message,
-            },
+            {"candidates": valid_candidates, "fallback_message": fallback_message},
             ensure_ascii=False,
         )
 
-    async def _universal_dispatch_tools(
-        self, tool_calls: list, messages: list[BaseMessage], iterations: int
-    ) -> None:
-        """Executes non-submit tool calls and appends their ToolMessages."""
+    async def _universal_execute_tool(self, name: str, args: dict) -> dict[str, Any]:
+        """Runs one perception tool by name; unknown names yield an error result."""
+        if name == "ask_perception_tool":
+            return await self.exec_ask_perception_tool(
+                search_query=args.get("search_query"),
+                nx=args.get("nx"),
+                ny=args.get("ny"),
+                detect_queries=args.get("detect_queries"),
+            )
+        if name == "detect_objects":
+            return await self.exec_detect_objects(
+                queries=args.get("queries"),
+                target_image_id=args.get("target_image_id", "img_0"),
+            )
+        if name == "get_ocr_list":
+            return await self.exec_get_ocr_list()
+        if name == "ask_image_processor":
+            return await self.exec_ask_image_processor(
+                instruction=args.get("instruction"),
+                target_image_id=args.get("target_image_id", "img_0"),
+            )
+        if name == "inspect_region":
+            return await self.exec_inspect_region(
+                x_min=args.get("x_min"),
+                y_min=args.get("y_min"),
+                x_max=args.get("x_max"),
+                y_max=args.get("y_max"),
+                zoom_factor=args.get("zoom_factor", 2.0),
+            )
+        return {"text": f"Error: Tool '{name}' is not recognized."}
+
+    @staticmethod
+    def _tool_image_paths(res: Any) -> list[str]:
+        """Collects the annotated image paths a tool result may carry."""
+        if not isinstance(res, dict):
+            return []
+        paths: list[str] = []
+        single = res.get("image_path")
+        if single:
+            paths.append(str(single))
+        paths.extend(str(p) for p in (res.get("image_paths") or []) if p)
+        return paths
+
+    def _build_tool_images_message(self, images: list[tuple[str, str]]) -> HumanMessage | None:
+        """One HumanMessage carrying every annotated image produced this turn.
+
+        LangChain ``ToolMessage`` content is text-only for most providers, so
+        the images ride on a follow-up human turn instead; unreadable files
+        are skipped so a missing annotation never aborts the loop.
+        """
         _ex = facade()
+        blocks: list[dict[str, Any]] = []
+        tool_names: list[str] = []
+        for tool_name, path in images:
+            try:
+                with open(path, "rb") as f:
+                    img_b64 = base64.b64encode(f.read()).decode("utf-8")
+            except OSError as read_err:
+                _ex.logger.warning(f"Skipping unreadable tool image {path}: {read_err}")
+                continue
+            if tool_name not in tool_names:
+                tool_names.append(tool_name)
+            blocks.append(
+                {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"}}
+            )
+        if not blocks:
+            return None
+        header = {
+            "type": "text",
+            "text": f"[Annotated image(s) returned by: {', '.join(tool_names)}]",
+        }
+        return HumanMessage(content=[header, *blocks])
+
+    async def _universal_dispatch_tools(
+        self, tool_calls: list, messages: list[BaseMessage], turn: int
+    ) -> None:
+        """Executes non-submit tool calls, appending ToolMessages and their images."""
+        _ex = facade()
+        images: list[tuple[str, str]] = []
         for tc in tool_calls:
             name = tc.get("name")
             args = tc.get("args", {})
-            call_id = tc.get("id", f"call_{iterations}_{name}")
+            call_id = tc.get("id", f"call_{turn}_{name}")
 
             if name in self.denylisted_tools:
                 messages.append(
@@ -217,39 +301,10 @@ class UniversalRunnerMixin:
                 continue
 
             try:
-                if name == "ask_perception_tool":
-                    res = await self.exec_ask_perception_tool(
-                        search_query=args.get("search_query"),
-                        nx=args.get("nx"),
-                        ny=args.get("ny"),
-                        detect_queries=args.get("detect_queries"),
-                    )
-                elif name == "detect_objects":
-                    res = await self.exec_detect_objects(
-                        queries=args.get("queries"),
-                        target_image_id=args.get("target_image_id", "img_0"),
-                    )
-                elif name == "get_ocr_list":
-                    res = await self.exec_get_ocr_list()
-                elif name == "ask_image_processor":
-                    res = await self.exec_ask_image_processor(
-                        instruction=args.get("instruction"),
-                        target_image_id=args.get("target_image_id", "img_0"),
-                    )
-                elif name == "inspect_region":
-                    res = await self.exec_inspect_region(
-                        x_min=args.get("x_min"),
-                        y_min=args.get("y_min"),
-                        x_max=args.get("x_max"),
-                        y_max=args.get("y_max"),
-                        zoom_factor=args.get("zoom_factor", 2.0),
-                    )
-                else:
-                    res = {"text": f"Error: Tool '{name}' is not recognized."}
-
+                res = await self._universal_execute_tool(name, args)
                 res_text = res.get("text") or res.get("result") or str(res)
                 messages.append(ToolMessage(tool_call_id=call_id, name=name, content=res_text))
-
+                images.extend((name, path) for path in self._tool_image_paths(res))
             except Exception as tool_err:
                 _ex.logger.error(f"Error executing tool '{name}': {tool_err}")
                 messages.append(
@@ -259,3 +314,7 @@ class UniversalRunnerMixin:
                         content=f"Error executing tool '{name}': {tool_err}",
                     )
                 )
+
+        image_message = self._build_tool_images_message(images) if images else None
+        if image_message is not None:
+            messages.append(image_message)