refactor(explorer): split explorer into facade and phase modules
This commit is contained in:
somew1nd committed
1 parent
9cad5b690f
commit
972201bb4d
7 files changed
+2589
-2051
No files matched your search
@@ -0,0 +1,796 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Native Gemini execution loop of ``Explorer.run``.
|
||||
|
||||
Split out of ``artemis.agents.explorer.explorer``: the native-SDK reasoning
|
||||
loop and its named phases (model invocation, submit_answer interception and
|
||||
validation, tool dispatching, failure mapping, resource cleanup), packaged as
|
||||
a mixin consumed by ``Explorer``. Patched collaborators (``settings``,
|
||||
``logger``, ``_generate_content_with_reliability``) are resolved through the
|
||||
facade module at call time; see ``artemis.agents.explorer._facade``.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
from google.genai import types
|
||||
|
||||
from artemis.agents.explorer._facade import facade
|
||||
from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
|
||||
from artemis.data_engine.trace import TraceSpan
|
||||
|
||||
|
||||
class NativeRunnerMixin:
|
||||
"""Native Gemini SDK reasoning loop of :class:`Explorer`."""
|
||||
|
||||
def _make_image_part_getter(self, client, use_file_api: bool, uploaded_files: list):
|
||||
"""Builds the image-part loader used for screenshots and tool images."""
|
||||
|
||||
def get_image_part(file_path: str) -> types.Part:
|
||||
if use_file_api:
|
||||
file_ref = client.files.upload(file=file_path)
|
||||
uploaded_files.append(file_ref)
|
||||
return types.Part(
|
||||
file_data=types.FileData(
|
||||
file_uri=file_ref.uri,
|
||||
mime_type=file_ref.mime_type or "image/jpeg",
|
||||
)
|
||||
)
|
||||
else:
|
||||
with open(file_path, "rb") as f:
|
||||
img_bytes = f.read()
|
||||
return types.Part.from_bytes(data=img_bytes, mime_type="image/jpeg")
|
||||
|
||||
return get_image_part
|
||||
|
||||
def _build_initial_contents(
|
||||
self, query: str, context_feedback: str, minimal_list: str, initial_part: types.Part
|
||||
) -> list[types.Content]:
|
||||
"""Builds the initial user content with the operator request and screenshot."""
|
||||
return [
|
||||
types.Content(
|
||||
role="user",
|
||||
parts=[
|
||||
types.Part.from_text(
|
||||
text=f"""Operator Request:
|
||||
- Query: {query}
|
||||
- Context Feedback: {context_feedback}
|
||||
|
||||
Initial marked UI elements list (corresponding to numbers ①, ②... in the image):
|
||||
{minimal_list}
|
||||
"""
|
||||
),
|
||||
initial_part,
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
def _resolve_caching_flag(self, enable_caching):
|
||||
"""Resolves the effective caching flag from env or settings when unset."""
|
||||
_ex = facade()
|
||||
if enable_caching is None:
|
||||
env_cache = os.getenv("ARTEMIS_EXPLORER_CACHING", "").lower()
|
||||
if env_cache in ["true", "false"]:
|
||||
enable_caching = env_cache == "true"
|
||||
else:
|
||||
enable_caching = getattr(_ex.settings, "EXPLORER_CACHING", True)
|
||||
return enable_caching
|
||||
|
||||
def _create_cache_resource(
|
||||
self, client, model_name: str, prompt_template: str, contents: list
|
||||
):
|
||||
"""Creates the explicit cache resource; returns None on failure."""
|
||||
_ex = facade()
|
||||
cached_content = None
|
||||
try:
|
||||
_ex.logger.info("Creating cache resource for Explorer...")
|
||||
cached_content = client.caches.create(
|
||||
model=model_name,
|
||||
config=types.CreateCachedContentConfig(
|
||||
contents=[contents[0]],
|
||||
system_instruction=prompt_template,
|
||||
tools=[types.Tool(function_declarations=self.get_exposed_tools())],
|
||||
tool_config=types.ToolConfig(
|
||||
function_calling_config=types.FunctionCallingConfig(
|
||||
mode=types.FunctionCallingConfigMode.ANY
|
||||
),
|
||||
include_server_side_tool_invocations=True,
|
||||
),
|
||||
ttl="300s",
|
||||
),
|
||||
)
|
||||
_ex.logger.info(f"Cache resource created successfully: {cached_content.name}")
|
||||
except Exception as cache_err:
|
||||
_ex.logger.error(f"Failed to create cache resource: {cache_err}")
|
||||
cached_content = None
|
||||
return cached_content
|
||||
|
||||
async def _run_native(
|
||||
self,
|
||||
*,
|
||||
client,
|
||||
query: str,
|
||||
context_feedback: str,
|
||||
minimal_list: str,
|
||||
img_to_read: str,
|
||||
enable_caching,
|
||||
prompt_template: str,
|
||||
model_name: str,
|
||||
temperature,
|
||||
thinking_level,
|
||||
max_iterations: int,
|
||||
) -> str:
|
||||
"""Runs the native Gemini path: cache setup, reasoning loop, cleanup."""
|
||||
use_file_api = os.getenv("ARTEMIS_USE_FILE_API", "false").lower() == "true"
|
||||
uploaded_files = []
|
||||
get_image_part = self._make_image_part_getter(client, use_file_api, uploaded_files)
|
||||
|
||||
cached_content = None
|
||||
try:
|
||||
# Upload initial screenshot
|
||||
initial_part = get_image_part(img_to_read)
|
||||
contents = self._build_initial_contents(
|
||||
query, context_feedback, minimal_list, initial_part
|
||||
)
|
||||
|
||||
# Caching initialization
|
||||
enable_caching = self._resolve_caching_flag(enable_caching)
|
||||
|
||||
cached_content = None
|
||||
if enable_caching:
|
||||
cached_content = self._create_cache_resource(
|
||||
client, model_name, prompt_template, contents
|
||||
)
|
||||
|
||||
# 6. Execution Loop (Native SDK Tools Dispatching)
|
||||
agent_outcome = await self._native_loop(
|
||||
client=client,
|
||||
contents=contents,
|
||||
get_image_part=get_image_part,
|
||||
cached_content=cached_content,
|
||||
prompt_template=prompt_template,
|
||||
model_name=model_name,
|
||||
temperature=temperature,
|
||||
thinking_level=thinking_level,
|
||||
max_iterations=max_iterations,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
agent_outcome = self._format_native_failure(e, model_name)
|
||||
finally:
|
||||
await self._cleanup_native_resources(client, cached_content, uploaded_files)
|
||||
|
||||
return agent_outcome
|
||||
|
||||
async def _native_loop(
|
||||
self,
|
||||
*,
|
||||
client,
|
||||
contents: list,
|
||||
get_image_part,
|
||||
cached_content,
|
||||
prompt_template: str,
|
||||
model_name: str,
|
||||
temperature,
|
||||
thinking_level,
|
||||
max_iterations: int,
|
||||
) -> str:
|
||||
"""Main native reasoning loop: one model turn per iteration."""
|
||||
_ex = facade()
|
||||
iterations = 0
|
||||
agent_outcome = ""
|
||||
self.turn_latencies = []
|
||||
self.turn_cached_tokens = []
|
||||
self.trace_history = []
|
||||
|
||||
while iterations < max_iterations:
|
||||
iterations += 1
|
||||
_ex.logger.info(f"Iteration {iterations}: Invoking Native Gemini SDK for Explorer...")
|
||||
|
||||
self._prune_historical_images(contents, keep_last=1)
|
||||
|
||||
is_final_turn = iterations == max_iterations
|
||||
if is_final_turn:
|
||||
warning_msg = (
|
||||
"\n[WARNING] This is your final iteration. You MUST"
|
||||
" call 'submit_answer' to submit your final result. No"
|
||||
" other tools are available."
|
||||
)
|
||||
contents.append(
|
||||
types.Content(
|
||||
role="user",
|
||||
parts=[types.Part.from_text(text=warning_msg)],
|
||||
)
|
||||
)
|
||||
|
||||
response, thinking_parts, text_parts = await self._invoke_native_model(
|
||||
client=client,
|
||||
contents=contents,
|
||||
cached_content=cached_content,
|
||||
is_final_turn=is_final_turn,
|
||||
prompt_template=prompt_template,
|
||||
model_name=model_name,
|
||||
temperature=temperature,
|
||||
thinking_level=thinking_level,
|
||||
)
|
||||
|
||||
turn_record = {
|
||||
"iteration": iterations,
|
||||
"thoughts": ("\n".join(thinking_parts) if thinking_parts else ""),
|
||||
"tool_calls": [],
|
||||
}
|
||||
|
||||
function_calls = response.function_calls
|
||||
if not function_calls:
|
||||
self._handle_missing_tool_calls(
|
||||
contents, response, turn_record, text_parts, max_iterations, iterations
|
||||
)
|
||||
continue
|
||||
|
||||
# Record model's function call request in history
|
||||
if response.candidates and response.candidates[0].content:
|
||||
contents.append(response.candidates[0].content)
|
||||
else:
|
||||
contents.append(
|
||||
types.Content(
|
||||
role="model",
|
||||
parts=[types.Part(function_call=fc) for fc in function_calls],
|
||||
)
|
||||
)
|
||||
|
||||
# Execute function calls
|
||||
tool_response_parts = []
|
||||
|
||||
# 6.1 Intercept and validate submit_answer to manage task lifecycle
|
||||
submit_outcome = self._process_submit_answer(
|
||||
function_calls, turn_record, tool_response_parts
|
||||
)
|
||||
if submit_outcome is not None:
|
||||
agent_outcome = submit_outcome
|
||||
break
|
||||
|
||||
await self._dispatch_tool_calls(
|
||||
function_calls, get_image_part, turn_record, tool_response_parts
|
||||
)
|
||||
|
||||
if tool_response_parts:
|
||||
# Native Gemini SDK allows 'tool' role to return mixed parts
|
||||
contents.append(types.Content(role="user", parts=tool_response_parts))
|
||||
|
||||
self.trace_history.append(turn_record)
|
||||
|
||||
if iterations >= max_iterations and not agent_outcome:
|
||||
agent_outcome = (
|
||||
"Error: Explorer reached maximum iterations without a conclusive answer."
|
||||
)
|
||||
|
||||
return agent_outcome
|
||||
|
||||
def _build_generate_config(
|
||||
self,
|
||||
*,
|
||||
cached_content,
|
||||
is_final_turn: bool,
|
||||
prompt_template: str,
|
||||
temperature,
|
||||
thinking_level,
|
||||
) -> types.GenerateContentConfig:
|
||||
"""Builds the per-turn generation config (cached vs full tool config)."""
|
||||
thinking_kwargs = (
|
||||
{"thinking_config": types.ThinkingConfig(thinking_level=thinking_level)}
|
||||
if thinking_level
|
||||
else {}
|
||||
)
|
||||
if cached_content and not is_final_turn:
|
||||
return types.GenerateContentConfig(
|
||||
cached_content=cached_content.name,
|
||||
temperature=temperature,
|
||||
safety_settings=SAFETY_SETTINGS_BLOCK_NONE,
|
||||
**thinking_kwargs,
|
||||
)
|
||||
else:
|
||||
return types.GenerateContentConfig(
|
||||
system_instruction=prompt_template,
|
||||
temperature=temperature,
|
||||
safety_settings=SAFETY_SETTINGS_BLOCK_NONE,
|
||||
tools=[
|
||||
types.Tool(
|
||||
function_declarations=self.get_exposed_tools(only_submit=is_final_turn)
|
||||
)
|
||||
],
|
||||
tool_config=types.ToolConfig(
|
||||
function_calling_config=types.FunctionCallingConfig(
|
||||
mode=types.FunctionCallingConfigMode.ANY
|
||||
),
|
||||
include_server_side_tool_invocations=True,
|
||||
),
|
||||
**thinking_kwargs,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_response_texts(response) -> tuple[list[str], list[str]]:
|
||||
"""Extracts model thoughts and plain text parts from a response."""
|
||||
thinking_parts = []
|
||||
text_parts = []
|
||||
candidates = getattr(response, "candidates", None)
|
||||
if not candidates and isinstance(response, dict):
|
||||
candidates = response.get("candidates")
|
||||
|
||||
if candidates and len(candidates) > 0:
|
||||
candidate = candidates[0]
|
||||
content = getattr(candidate, "content", None)
|
||||
if not content and isinstance(candidate, dict):
|
||||
content = candidate.get("content")
|
||||
|
||||
parts = getattr(content, "parts", None)
|
||||
if not parts and isinstance(content, dict):
|
||||
parts = content.get("parts")
|
||||
|
||||
if parts:
|
||||
for part in parts:
|
||||
is_thought = False
|
||||
part_text = None
|
||||
if isinstance(part, dict):
|
||||
is_thought = part.get("thought", False)
|
||||
part_text = part.get("text", None)
|
||||
else:
|
||||
is_thought = getattr(part, "thought", False)
|
||||
part_text = getattr(part, "text", None)
|
||||
|
||||
if is_thought and part_text:
|
||||
thinking_parts.append(part_text)
|
||||
elif part_text:
|
||||
text_parts.append(part_text)
|
||||
return thinking_parts, text_parts
|
||||
|
||||
async def _invoke_native_model(
|
||||
self,
|
||||
*,
|
||||
client,
|
||||
contents: list,
|
||||
cached_content,
|
||||
is_final_turn: bool,
|
||||
prompt_template: str,
|
||||
model_name: str,
|
||||
temperature,
|
||||
thinking_level,
|
||||
):
|
||||
"""Invokes the model for one turn under a trace span; returns response and texts."""
|
||||
_ex = facade()
|
||||
ctx = self.ctx
|
||||
start_turn = time.perf_counter()
|
||||
with TraceSpan(name="gemini_explorer_call", ctx=ctx) as span:
|
||||
generate_config = self._build_generate_config(
|
||||
cached_content=cached_content,
|
||||
is_final_turn=is_final_turn,
|
||||
prompt_template=prompt_template,
|
||||
temperature=temperature,
|
||||
thinking_level=thinking_level,
|
||||
)
|
||||
|
||||
response = await _ex._generate_content_with_reliability(
|
||||
lambda: asyncio.wait_for(
|
||||
client.aio.models.generate_content(
|
||||
model=model_name,
|
||||
contents=contents,
|
||||
config=generate_config,
|
||||
),
|
||||
timeout=180,
|
||||
),
|
||||
)
|
||||
end_turn = time.perf_counter()
|
||||
turn_dur = end_turn - start_turn
|
||||
self.turn_latencies.append(turn_dur)
|
||||
|
||||
cached_tokens = 0
|
||||
if getattr(response, "usage_metadata", None) and getattr(
|
||||
response.usage_metadata,
|
||||
"cached_content_token_count",
|
||||
None,
|
||||
):
|
||||
cached_tokens = response.usage_metadata.cached_content_token_count
|
||||
self.turn_cached_tokens.append(cached_tokens)
|
||||
span.result = (
|
||||
f"Function calls: {len(response.function_calls)}"
|
||||
if response.function_calls
|
||||
else "Final answer"
|
||||
)
|
||||
# Extract and log model thoughts and text if present
|
||||
thinking_parts, text_parts = self._extract_response_texts(response)
|
||||
if thinking_parts:
|
||||
span.payload["explorer_thought"] = "\n".join(thinking_parts)
|
||||
if text_parts:
|
||||
span.payload["explorer_text"] = "\n".join(text_parts)
|
||||
|
||||
return response, thinking_parts, text_parts
|
||||
|
||||
def _handle_missing_tool_calls(
|
||||
self,
|
||||
contents: list,
|
||||
response,
|
||||
turn_record: dict,
|
||||
text_parts: list[str],
|
||||
max_iterations: int,
|
||||
iterations: int,
|
||||
) -> None:
|
||||
"""Handles a turn where the model produced plain text instead of tool calls."""
|
||||
_ex = facade()
|
||||
_ex.logger.warning(
|
||||
f"Explorer iteration {iterations}: Model hallucinated"
|
||||
" plain text. Forcing retry."
|
||||
)
|
||||
turn_record["tool_calls"].append(
|
||||
{
|
||||
"name": "hallucinated_plain_text",
|
||||
"args": {"text": "\n".join(text_parts) if text_parts else ""},
|
||||
"response": {
|
||||
"error": (
|
||||
"Forced retry because model failed to call"
|
||||
" submit_answer or any other tool"
|
||||
)
|
||||
},
|
||||
}
|
||||
)
|
||||
self.trace_history.append(turn_record)
|
||||
contents.append(response.candidates[0].content)
|
||||
contents.append(
|
||||
types.Content(
|
||||
role="user",
|
||||
parts=[
|
||||
types.Part.from_text(
|
||||
text=(
|
||||
"You have not called 'submit_answer'"
|
||||
" to submit your final answer yet. If"
|
||||
" you are not certain about the final"
|
||||
" answer, you can continue exploring"
|
||||
f" {max_iterations - iterations} more"
|
||||
" time(s)."
|
||||
)
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_submit_args(function_calls: list, args: dict, candidates: list) -> list[str]:
|
||||
"""Validates a submit_answer call; returns the list of validation errors."""
|
||||
errors = []
|
||||
|
||||
if len(function_calls) > 1:
|
||||
errors.append(
|
||||
"The tool 'submit_answer' MUST be called"
|
||||
" individually in your final turn without any other"
|
||||
" parallel tool calls. Please remove other tool"
|
||||
" calls and re-submit."
|
||||
)
|
||||
|
||||
fallback_message = args.get("fallback_message", "")
|
||||
if not candidates and not fallback_message:
|
||||
errors.append(
|
||||
"The candidates list is empty. You must provide at"
|
||||
" least one candidate matching the query, or"
|
||||
" explain observations via fallback_message."
|
||||
)
|
||||
|
||||
for i, cand in enumerate(candidates):
|
||||
label = cand.get("label")
|
||||
coords = cand.get("coords")
|
||||
if not label:
|
||||
errors.append(f"Candidate at index {i} is missing a 'label'.")
|
||||
if not coords or not isinstance(coords, list) or len(coords) != 2:
|
||||
errors.append(
|
||||
f"Candidate '{label or i}' must have 'coords'"
|
||||
" as an array of exactly 2 integers `[nx,"
|
||||
" ny]`."
|
||||
)
|
||||
else:
|
||||
try:
|
||||
nx, ny = int(coords[0]), int(coords[1])
|
||||
if not (0 <= nx <= 1000) or not (0 <= ny <= 1000):
|
||||
errors.append(
|
||||
f"Candidate '{label or i}' coordinates"
|
||||
f" `[{nx}, {ny}]` must strictly be in"
|
||||
" the `[0-1000]` normalized scale"
|
||||
" range inclusive."
|
||||
)
|
||||
except (ValueError, TypeError, IndexError):
|
||||
errors.append(
|
||||
f"Candidate '{label or i}' coordinates are invalid integers."
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
def _process_submit_answer(
|
||||
self, function_calls: list, turn_record: dict, tool_response_parts: list
|
||||
) -> str | None:
|
||||
"""Intercepts submit_answer; returns the final outcome JSON when valid.
|
||||
|
||||
On validation failure the error is fed back as a tool response and
|
||||
None is returned so the ReAct loop can continue and self-correct.
|
||||
"""
|
||||
_ex = facade()
|
||||
submit_call = next(
|
||||
(
|
||||
fc
|
||||
for fc in function_calls
|
||||
if (fc.name.split(":")[-1] if ":" in fc.name else fc.name)
|
||||
== "submit_answer"
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not submit_call:
|
||||
return None
|
||||
|
||||
args = submit_call.args or {}
|
||||
candidates = args.get("candidates", [])
|
||||
errors = self._validate_submit_args(function_calls, args, candidates)
|
||||
|
||||
if errors:
|
||||
_ex.logger.warning(f"Explorer submit_answer validation failed: {errors}")
|
||||
turn_record["tool_calls"].append(
|
||||
{
|
||||
"name": "submit_answer",
|
||||
"args": args,
|
||||
"response": {"error": ("Validation Failed:\n" + "\n".join(errors))},
|
||||
}
|
||||
)
|
||||
tool_response_parts.append(
|
||||
types.Part.from_function_response(
|
||||
name="submit_answer",
|
||||
response={
|
||||
"error": (
|
||||
"Validation Failed:\n"
|
||||
+ "\n".join(errors)
|
||||
+ "\nPlease correct these formatting"
|
||||
" errors and re-submit using"
|
||||
" submit_answer."
|
||||
)
|
||||
},
|
||||
)
|
||||
)
|
||||
# Allow ReAct loop to continue so LLM can self-correct and re-submit
|
||||
return None
|
||||
else:
|
||||
_ex.logger.info("Explorer submit_answer validation passed successfully!")
|
||||
agent_outcome = json.dumps(args, ensure_ascii=False)
|
||||
turn_record["tool_calls"].append(
|
||||
{
|
||||
"name": "submit_answer",
|
||||
"args": args,
|
||||
"response": {"result": "success"},
|
||||
}
|
||||
)
|
||||
self.trace_history.append(turn_record)
|
||||
return agent_outcome
|
||||
|
||||
@staticmethod
|
||||
def _append_tool_response_with_image(
|
||||
name: str, res: dict, get_image_part, tool_response_parts: list
|
||||
) -> None:
|
||||
"""Appends a tool result part plus its annotated image when present."""
|
||||
if res.get("image_path"):
|
||||
tool_response_parts.append(
|
||||
types.Part.from_function_response(
|
||||
name=name,
|
||||
response={
|
||||
"result": res.get("text"),
|
||||
},
|
||||
)
|
||||
)
|
||||
tool_response_parts.append(get_image_part(res["image_path"]))
|
||||
else:
|
||||
tool_response_parts.append(
|
||||
types.Part.from_function_response(
|
||||
name=name,
|
||||
response={"result": res.get("text")},
|
||||
)
|
||||
)
|
||||
|
||||
async def _execute_native_tool(
|
||||
self,
|
||||
name: str,
|
||||
args: dict,
|
||||
get_image_part,
|
||||
tool_call_trace: dict,
|
||||
tool_response_parts: list,
|
||||
) -> None:
|
||||
"""Executes one non-submit tool call and appends its response parts."""
|
||||
_ex = facade()
|
||||
if name == "ask_perception_tool":
|
||||
res = await self.exec_ask_perception_tool(
|
||||
search_query=args.get("search_query"),
|
||||
nx=args.get("nx"),
|
||||
ny=args.get("ny"),
|
||||
detect_queries=args.get("detect_queries"),
|
||||
)
|
||||
tool_call_trace["response"] = res
|
||||
tool_response_parts.append(
|
||||
types.Part.from_function_response(
|
||||
name=name,
|
||||
response={
|
||||
"text": res.get("text"),
|
||||
},
|
||||
)
|
||||
)
|
||||
if res.get("image_paths"):
|
||||
for img_p in res["image_paths"]:
|
||||
try:
|
||||
tool_response_parts.append(get_image_part(img_p))
|
||||
except Exception as e:
|
||||
_ex.logger.warning(
|
||||
f"Failed to load image response part for {img_p}: {e}"
|
||||
)
|
||||
|
||||
elif name == "detect_objects":
|
||||
res = await self.exec_detect_objects(
|
||||
queries=args.get("queries"),
|
||||
target_image_id=args.get("target_image_id", "img_0"),
|
||||
)
|
||||
tool_call_trace["response"] = res
|
||||
self._append_tool_response_with_image(name, res, get_image_part, tool_response_parts)
|
||||
|
||||
elif name == "ask_image_processor":
|
||||
res = await self.exec_ask_image_processor(
|
||||
instruction=args.get("instruction"),
|
||||
target_image_id=args.get("target_image_id", "img_0"),
|
||||
)
|
||||
tool_call_trace["response"] = res
|
||||
|
||||
tool_response_parts.append(
|
||||
types.Part.from_function_response(
|
||||
name=name,
|
||||
response={"result": res.get("text")},
|
||||
)
|
||||
)
|
||||
if res.get("image_paths"):
|
||||
for img_p in res["image_paths"]:
|
||||
try:
|
||||
tool_response_parts.append(get_image_part(img_p))
|
||||
except Exception as e:
|
||||
_ex.logger.warning(
|
||||
"Failed to upload image path"
|
||||
f" {img_p} for ask_image_processor"
|
||||
f" tool response: {e}"
|
||||
)
|
||||
|
||||
elif name == "get_ocr_list":
|
||||
res = await self.exec_get_ocr_list()
|
||||
tool_call_trace["response"] = res
|
||||
self._append_tool_response_with_image(name, res, get_image_part, tool_response_parts)
|
||||
|
||||
elif name == "inspect_region":
|
||||
res = await self.exec_inspect_region(
|
||||
x_min=args.get("x_min"),
|
||||
y_min=args.get("y_min"),
|
||||
x_max=args.get("x_max"),
|
||||
y_max=args.get("y_max"),
|
||||
zoom_factor=args.get("zoom_factor"),
|
||||
)
|
||||
tool_call_trace["response"] = res
|
||||
self._append_tool_response_with_image(name, res, get_image_part, tool_response_parts)
|
||||
else:
|
||||
tool_call_trace["response"] = {"error": f"Tool {name} not found"}
|
||||
tool_response_parts.append(
|
||||
types.Part.from_function_response(
|
||||
name=name,
|
||||
response={"error": f"Tool {name} not found"},
|
||||
)
|
||||
)
|
||||
|
||||
async def _dispatch_tool_calls(
|
||||
self,
|
||||
function_calls: list,
|
||||
get_image_part,
|
||||
turn_record: dict,
|
||||
tool_response_parts: list,
|
||||
) -> None:
|
||||
"""Dispatches all non-submit tool calls sequentially with tracing."""
|
||||
_ex = facade()
|
||||
for fc in function_calls:
|
||||
name = fc.name.split(":")[-1] if ":" in fc.name else fc.name
|
||||
if name == "submit_answer":
|
||||
continue
|
||||
args = fc.args or {}
|
||||
tool_call_trace = {
|
||||
"name": name,
|
||||
"args": args,
|
||||
}
|
||||
if name in self.denylisted_tools:
|
||||
_ex.logger.warning(
|
||||
"Explorer attempted to call denylisted tool"
|
||||
f" '{name}'. Blocking execution."
|
||||
)
|
||||
tool_call_trace["response"] = {
|
||||
"error": (f"Tool '{name}' is denylisted and unavailable.")
|
||||
}
|
||||
turn_record["tool_calls"].append(tool_call_trace)
|
||||
tool_response_parts.append(
|
||||
types.Part.from_function_response(
|
||||
name=fc.name,
|
||||
response={
|
||||
"error": (f"Tool '{name}' is denylisted and unavailable.")
|
||||
},
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
_ex.logger.info(f"Explorer executing tool '{name}' sequentially...")
|
||||
|
||||
try:
|
||||
await self._execute_native_tool(
|
||||
name, args, get_image_part, tool_call_trace, tool_response_parts
|
||||
)
|
||||
except Exception as e:
|
||||
_ex.logger.error(f"Explorer tool {name} execution failed: {e}")
|
||||
tool_call_trace["response"] = {"error": str(e)}
|
||||
tool_response_parts.append(
|
||||
types.Part.from_function_response(name=name, response={"error": str(e)})
|
||||
)
|
||||
finally:
|
||||
turn_record["tool_calls"].append(tool_call_trace)
|
||||
|
||||
def _format_native_failure(self, e: Exception, model_name: str) -> str:
|
||||
"""Maps a loop failure to the user-facing fallback outcome JSON."""
|
||||
_ex = facade()
|
||||
_ex.logger.error(f"Explorer execution loop failed: {e}")
|
||||
|
||||
err_msg = str(e)
|
||||
if "preempted" in err_msg.lower():
|
||||
clean_msg = (
|
||||
"Model request was preempted by the server due to high"
|
||||
" demand. Please try again later."
|
||||
)
|
||||
elif "overloaded" in err_msg.lower() or "503" in err_msg:
|
||||
clean_msg = "Model service is currently overloaded. Please try again later."
|
||||
elif "quota" in err_msg.lower() or "429" in err_msg:
|
||||
clean_msg = "Model API quota exceeded or rate limited."
|
||||
elif "thinkingconfig" in err_msg.lower():
|
||||
clean_msg = (
|
||||
"Model configuration error: ThinkingConfig is not"
|
||||
f" supported for model {model_name}."
|
||||
)
|
||||
else:
|
||||
clean_msg = f"Explorer agent execution failed: {err_msg}"
|
||||
|
||||
return json.dumps(
|
||||
{"candidates": [], "fallback_message": clean_msg},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
async def _cleanup_native_resources(
|
||||
self, client, cached_content, uploaded_files: list
|
||||
) -> None:
|
||||
"""Closes the HTTP client and deletes cache/file resources."""
|
||||
_ex = facade()
|
||||
if self.http_client:
|
||||
try:
|
||||
await self.http_client.aclose()
|
||||
_ex.logger.info("Closed Explorer HTTP client.")
|
||||
except Exception as close_err:
|
||||
_ex.logger.warning(f"Failed to close Explorer HTTP client: {close_err}")
|
||||
if cached_content:
|
||||
try:
|
||||
_ex.logger.info(f"Deleting cache resource: {cached_content.name}")
|
||||
client.caches.delete(name=cached_content.name)
|
||||
except Exception as cleanup_cache_err:
|
||||
_ex.logger.warning(f"Failed to delete cache resource: {cleanup_cache_err}")
|
||||
for file_ref in uploaded_files:
|
||||
try:
|
||||
client.files.delete(name=file_ref.name)
|
||||
except Exception as cleanup_err:
|
||||
_ex.logger.warning(f"Failed to delete uploaded file {file_ref.name}: {cleanup_err}")
|
||||
_ex.logger.info(f"Explorer turn latencies: {self.turn_latencies}")
|
||||
Reference in new issue
Block a user