refactor(explorer): split explorer into facade and phase modules

artemis/agents/explorer/universal_runner.py

@@ -0,0 +1,261 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Universal (LangChain ChatModel) execution loop for the Explorer agent.
+
+Split out of ``artemis.agents.explorer.explorer``: the ``_run_universal``
+reasoning loop, packaged as a mixin consumed by ``Explorer``.  Patched
+collaborators (``get_llm``, ``is_ocr_configured``, ``logger``,
+``UNIVERSAL_EXPLORER_TOOLS``) are resolved through the facade module at call
+time; see ``artemis.agents.explorer._facade``.
+"""
+
+import asyncio
+import base64
+import json
+from typing import Any
+
+from langchain_core.messages import (
+    BaseMessage,
+    HumanMessage,
+    SystemMessage,
+    ToolMessage,
+)
+
+from artemis.agents.explorer._facade import facade
+from artemis.graph.state import State
+
+
+class UniversalRunnerMixin:
+    """Universal-engine reasoning loop of :class:`Explorer`."""
+
+    async def _run_universal(
+        self,
+        query: str,
+        context_feedback: str,
+        screenshot_path: str,
+        state: State,
+        minimal_list: str,
+        version: str,
+        prompt_template: str,
+        max_iterations: int,
+    ) -> str:
+        """Executes Explorer reasoning loop via Universal LangChain ChatModel."""
+        _ex = facade()
+        llm = _ex.get_llm(self.ctx, name="explorer")
+
+        # Filter universal tools based on denylist and OCR configuration
+        denylisted = set(self.denylisted_tools)
+        if not _ex.is_ocr_configured():
+            denylisted.add("get_ocr_list")
+        exposed_tools = [
+            t for t in _ex.UNIVERSAL_EXPLORER_TOOLS if t["function"]["name"] not in denylisted
+        ]
+        bound_llm = llm.bind_tools(exposed_tools)
+
+        messages = self._build_universal_messages(
+            query, context_feedback, screenshot_path, minimal_list, prompt_template
+        )
+
+        iterations = 0
+        agent_outcome = ""
+
+        while iterations < max_iterations:
+            iterations += 1
+            is_final_turn = iterations == max_iterations
+
+            if is_final_turn:
+                messages.append(
+                    HumanMessage(
+                        content=(
+                            "[WARNING] This is your final iteration. You MUST"
+                            " call 'submit_answer' to submit your final result."
+                        )
+                    )
+                )
+                submit_only_tools = [
+                    t for t in exposed_tools if t["function"]["name"] == "submit_answer"
+                ]
+                current_llm = llm.bind_tools(submit_only_tools) if submit_only_tools else bound_llm
+            else:
+                current_llm = bound_llm
+
+            response = await asyncio.wait_for(current_llm.ainvoke(messages), timeout=180)
+            messages.append(response)
+
+            tool_calls = getattr(response, "tool_calls", None) or []
+            if not tool_calls:
+                _ex.logger.warning(
+                    f"Universal Explorer iteration {iterations}: No tool calls generated."
+                )
+                if is_final_turn:
+                    return json.dumps(
+                        {
+                            "candidates": [],
+                            "fallback_message": str(response.content) or "No candidates found.",
+                        },
+                        ensure_ascii=False,
+                    )
+                continue
+
+            # Check for submit_answer
+            submit_outcome = self._universal_submit_outcome(tool_calls)
+            if submit_outcome is not None:
+                agent_outcome = submit_outcome
+                return agent_outcome
+
+            # Execute non-submit tools
+            await self._universal_dispatch_tools(tool_calls, messages, iterations)
+
+        if not agent_outcome:
+            agent_outcome = json.dumps(
+                {
+                    "candidates": [],
+                    "fallback_message": (
+                        "Explorer reached max iterations without finding candidates."
+                    ),
+                },
+                ensure_ascii=False,
+            )
+
+        return agent_outcome
+
+    def _build_universal_messages(
+        self,
+        query: str,
+        context_feedback: str,
+        screenshot_path: str,
+        minimal_list: str,
+        prompt_template: str,
+    ) -> list[BaseMessage]:
+        """Builds the initial system/user message pair for the universal loop."""
+        with open(screenshot_path, "rb") as f:
+            img_b64 = base64.b64encode(f.read()).decode("utf-8")
+
+        user_content: list[dict[str, Any]] = [
+            {
+                "type": "text",
+                "text": (
+                    f"Operator Request:\n- Query: {query}\n"
+                    f"- Context Feedback: {context_feedback}\n\n"
+                    "Initial marked UI elements list:\n"
+                    f"{minimal_list}\n"
+                ),
+            },
+            {
+                "type": "image_url",
+                "image_url": {"url": f"data:image/jpeg;base64,{img_b64}"},
+            },
+        ]
+
+        messages: list[BaseMessage] = [
+            SystemMessage(content=prompt_template),
+            HumanMessage(content=user_content),
+        ]
+        return messages
+
+    def _universal_submit_outcome(self, tool_calls: list) -> str | None:
+        """Returns the validated submit_answer outcome JSON, or None if absent."""
+        submit_call = next((tc for tc in tool_calls if tc.get("name") == "submit_answer"), None)
+        if not submit_call:
+            return None
+
+        args = submit_call.get("args", {})
+        candidates = args.get("candidates", [])
+        fallback_message = args.get("fallback_message", "")
+
+        # Validate coordinates
+        valid_candidates = []
+        for cand in candidates:
+            if isinstance(cand, dict) and "coords" in cand:
+                coords = cand["coords"]
+                if isinstance(coords, list) and len(coords) == 2:
+                    try:
+                        nx, ny = int(coords[0]), int(coords[1])
+                        if 0 <= nx <= 1000 and 0 <= ny <= 1000:
+                            valid_candidates.append(cand)
+                    except (ValueError, TypeError):
+                        pass
+
+        return json.dumps(
+            {
+                "candidates": valid_candidates,
+                "fallback_message": fallback_message,
+            },
+            ensure_ascii=False,
+        )
+
+    async def _universal_dispatch_tools(
+        self, tool_calls: list, messages: list[BaseMessage], iterations: int
+    ) -> None:
+        """Executes non-submit tool calls and appends their ToolMessages."""
+        _ex = facade()
+        for tc in tool_calls:
+            name = tc.get("name")
+            args = tc.get("args", {})
+            call_id = tc.get("id", f"call_{iterations}_{name}")
+
+            if name in self.denylisted_tools:
+                messages.append(
+                    ToolMessage(
+                        tool_call_id=call_id,
+                        name=name,
+                        content=f"Tool '{name}' is denylisted and unavailable.",
+                    )
+                )
+                continue
+
+            try:
+                if name == "ask_perception_tool":
+                    res = await self.exec_ask_perception_tool(
+                        search_query=args.get("search_query"),
+                        nx=args.get("nx"),
+                        ny=args.get("ny"),
+                        detect_queries=args.get("detect_queries"),
+                    )
+                elif name == "detect_objects":
+                    res = await self.exec_detect_objects(
+                        queries=args.get("queries"),
+                        target_image_id=args.get("target_image_id", "img_0"),
+                    )
+                elif name == "get_ocr_list":
+                    res = await self.exec_get_ocr_list()
+                elif name == "ask_image_processor":
+                    res = await self.exec_ask_image_processor(
+                        instruction=args.get("instruction"),
+                        target_image_id=args.get("target_image_id", "img_0"),
+                    )
+                elif name == "inspect_region":
+                    res = await self.exec_inspect_region(
+                        x_min=args.get("x_min"),
+                        y_min=args.get("y_min"),
+                        x_max=args.get("x_max"),
+                        y_max=args.get("y_max"),
+                        zoom_factor=args.get("zoom_factor", 2.0),
+                    )
+                else:
+                    res = {"text": f"Error: Tool '{name}' is not recognized."}
+
+                res_text = res.get("text") or res.get("result") or str(res)
+                messages.append(ToolMessage(tool_call_id=call_id, name=name, content=res_text))
+
+            except Exception as tool_err:
+                _ex.logger.error(f"Error executing tool '{name}': {tool_err}")
+                messages.append(
+                    ToolMessage(
+                        tool_call_id=call_id,
+                        name=name,
+                        content=f"Error executing tool '{name}': {tool_err}",
+                    )
+                )