chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

fix: correct task scheduling and simplify agent internals

Browse Source 
Fix queue cancellation, concurrency limits and worker status updates.

Use direct Explorer imports, split the Diagnoser loop into helpers and remove unused tool registration and prompt-builder code.

Correct model caching, endpoint selection and retry handling.
This commit is contained in:
somew1nd committed 2026-09-02 22:58:00 -07:00
1 parent 67248c887e
commit 12d7902a82
92 files changed
+1085 -1763
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

artemis/agents/explorer/_facade.py
-34
View File
@@ -1,34 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Late-bound access to the ``artemis.agents.explorer.explorer`` facade module.
The Explorer implementation is split across several modules, but external
callers (and ``unittest.mock.patch`` targets in the test suite) address
collaborators such as ``settings``, ``StorageManager``, ``draw_dots``,
``get_llm`` etc. through the historical module path
``artemis.agents.explorer.explorer``. Split-out code therefore resolves those
names at call time via :func:`facade` instead of importing them directly, so
patches applied to the facade module keep affecting the split-out code.
The import happens inside the function (not at module top level) to stay safe
regardless of which module of the package is imported first.
"""
def facade():
"""Return the ``artemis.agents.explorer.explorer`` module object."""
import artemis.agents.explorer.explorer as explorer_module
return explorer_module
artemis/agents/explorer/constants.py
+3
View File
@@ -22,6 +22,9 @@ Explorer tier the user configured. Tier behavior lives in
ASK_EXPLORER_TOOL_NAME = "ask_explorer"
#: Model used when the LLM configuration carries no Explorer entry.
DEFAULT_EXPLORER_MODEL = "gemini-3.8-flash"
ASK_EXPLORER_DESCRIPTION = (
"[EXPLORER] Ask the UI Explorer to locate elements on the current screen"
" that are missing from the indexed element list or whose listed"
artemis/agents/explorer/explorer.py
+7 -121
View File
@@ -12,26 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Explorer agent facade.
"""Explorer agent entry point.
This module is the stable entry point (and ``mock.patch`` target namespace)
for the Explorer agent. The implementation is split across sibling modules:
The :class:`Explorer` class combines the following modules:
- ``tiers``: the flash / pro / ultra tier table (single source of truth).
- ``tiers``: engine, tools and budgets for flash / pro / ultra.
- ``tool_declarations``: static tool schemas (universal + native).
- ``perception_tools``: the ``exec_*`` perception/vision tool method group.
- ``universal_runner``: the LangChain-based ``_run_universal`` loop.
- ``run_setup``: flash mode and the pre-loop setup phases of ``run``.
- ``native_runner``: the native Gemini reasoning loop of ``run``.
All historical module-level names (``settings``, ``StorageManager``,
``genai``, ``_run_object_detection``, ``draw_dots``, ``perform_ocr``,
``is_ocr_configured``, ``get_llm``, ``UNIVERSAL_EXPLORER_TOOLS``, ...) remain
importable and patchable here; split-out code resolves them through this
module at call time (see ``artemis.agents.explorer._facade``).
"""
import asyncio
import json
import os
from typing import Any, Literal
@@ -39,6 +31,7 @@ from typing import Any, Literal
from google import genai
from google.genai import types
from artemis.agents.explorer.constants import DEFAULT_EXPLORER_MODEL
from artemis.agents.explorer.geometry import (
FALLBACK_SCREEN_SIZE,
is_valid_norm_point,
@@ -50,51 +43,18 @@ from artemis.agents.explorer.perception_tools import PerceptionToolsMixin
from artemis.agents.explorer.run_setup import RunSetupMixin
from artemis.agents.explorer.screen_index import normalize_text, ScreenElement, ScreenIndex
from artemis.agents.explorer.tiers import ExplorerTier, get_tier
from artemis.agents.explorer.tool_declarations import (
NATIVE_EXPLORER_TOOL_DECLARATIONS,
UNIVERSAL_EXPLORER_TOOLS,
)
from artemis.agents.explorer.tool_declarations import NATIVE_EXPLORER_TOOL_DECLARATIONS
from artemis.agents.explorer.universal_runner import UniversalRunnerMixin
from artemis.agents.object_detector.object_detector import _run_object_detection
from artemis.config import settings
from artemis.context import ArtemisContext
from artemis.data_engine.storage import StorageManager
from artemis.data_engine.trace import trace
from artemis.graph.state import State
from artemis.llm.reliability import (
LLMExhaustedError,
LLMPermanentError,
classify_failure,
retry_policy_for,
)
from artemis.services.llm import _record_llm_event, _record_llm_retry, get_llm
from artemis.utils.logger import get_logger
from artemis.utils.ocr_api import is_ocr_configured, perform_ocr
from artemis.utils.visualization import draw_dots
from artemis.utils.ocr_api import is_ocr_configured
logger = get_logger(__name__)
#: Public surface plus the late-bound collaborators the split-out modules
#: resolve through this facade (``mock.patch`` targets for the test suite).
__all__ = [
"DEFAULT_EXPLORER_MODEL",
"Explorer",
"NATIVE_EXPLORER_TOOL_DECLARATIONS",
"StorageManager",
"UNIVERSAL_EXPLORER_TOOLS",
"_generate_content_with_reliability",
"_run_object_detection",
"draw_dots",
"genai",
"get_llm",
"is_ocr_configured",
"logger",
"perform_ocr",
"settings",
]
#: Model used when the LLM configuration carries no Explorer entry.
DEFAULT_EXPLORER_MODEL = "gemini-3.8-flash"
__all__ = ["Explorer"]
#: Environment override for Gemini explicit context caching ("true" / "false").
CACHING_ENV_VAR = "ARTEMIS_EXPLORER_CACHING"
@@ -104,80 +64,6 @@ CACHING_ENV_VAR = "ARTEMIS_EXPLORER_CACHING"
LABEL_TOLERANCE_RATIO = 0.02
async def _generate_content_with_reliability(operation, *, label: str = "Explorer model call"):
"""Run one native google-genai model call under the shared reliability layer.
Retry decisions are owned by ``classify_failure``/``retry_policy_for``
(artemis.llm.reliability) instead of a blanket exponential-backoff loop:
non-retryable categories (auth, bad request) are raised immediately as
``LLMPermanentError`` and exhausted retryable categories surface as
``LLMExhaustedError``. Only the retry/error-classification policy is
centralized here; the google-genai transport itself stays untouched.
``operation`` must be a zero-argument callable returning a fresh awaitable
per attempt.
"""
attempt = 0
while True:
try:
return await operation()
except Exception as call_err:
attempt += 1
failure = classify_failure(call_err)
if not failure.retryable:
logger.error(f"{label} permanently failed [{failure.category.value}]: {call_err}")
_record_llm_event(
"llm_gave_up",
{
"source": "explorer",
"error": str(call_err)[:1000],
"category": failure.category.value,
"retryable": False,
},
status="failed",
)
raise LLMPermanentError(
f"{label} failed [{failure.category.value}]: {call_err}",
failure=failure,
cause=call_err,
) from call_err
policy = retry_policy_for(failure.category)
if attempt >= policy.max_attempts:
logger.error(
f"{label} exhausted {attempt} attempt(s) [{failure.category.value}]: {call_err}"
)
_record_llm_event(
"llm_gave_up",
{
"source": "explorer",
"error": str(call_err)[:1000],
"category": failure.category.value,
"retryable": True,
"attempts": attempt,
},
status="failed",
)
raise LLMExhaustedError(
f"{label} exhausted {attempt} attempt(s)"
f" [{failure.category.value}]: {call_err}",
failure=failure,
cause=call_err,
) from call_err
delay = policy.delay_for(attempt)
logger.warning(
f"{label} failed [{failure.category.value}] on attempt"
f" {attempt}/{policy.max_attempts}: {call_err}."
f" Retrying in {delay:.2f}s..."
)
_record_llm_retry(
str(call_err),
delay,
attempt=attempt,
max_retries=policy.max_attempts,
source="explorer",
)
await asyncio.sleep(delay)
class Explorer(PerceptionToolsMixin, UniversalRunnerMixin, RunSetupMixin, NativeRunnerMixin):
"""Locates UI elements on a screenshot for the ``ask_explorer`` tool.
artemis/agents/explorer/native_runner.py
+99 -30
View File
@@ -19,9 +19,9 @@ loop and its named phases (model invocation, submit_answer interception and
validation, tool dispatching, failure mapping, resource cleanup), packaged as
a mixin consumed by ``Explorer``. Every google-genai call goes through the
SDK's ``client.aio`` surface so uploads, cache management and generation
never block the event loop. Patched collaborators (``logger``,
``_generate_content_with_reliability``) are resolved through the facade
module at call time; see ``artemis.agents.explorer._facade``.
never block the event loop. This module also owns
:func:`_generate_content_with_reliability`, the shared reliability wrapper
for native google-genai model calls.
"""
import asyncio
@@ -33,16 +33,94 @@ from typing import TYPE_CHECKING, Any
from google.genai import types
from artemis.agents.explorer._facade import facade
from artemis.agents.explorer.geometry import is_valid_norm_point
from artemis.agents.explorer.tiers import SUBMIT_TOOL
from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
from artemis.data_engine.trace import TraceSpan
from artemis.llm.reliability import (
LLMExhaustedError,
LLMPermanentError,
classify_failure,
retry_policy_for,
)
from artemis.services.llm import _record_llm_event, _record_llm_retry
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
#: Loads a local image as a ``Part`` (inline bytes or a File API reference).
ImagePartGetter = Callable[[str], Awaitable[types.Part]]
async def _generate_content_with_reliability(operation, *, label: str = "Explorer model call"):
"""Run one native google-genai model call under the shared reliability layer.
``operation`` must return a fresh awaitable on each call. Non-retryable
failures raise ``LLMPermanentError``; exhausted retries raise
``LLMExhaustedError`` according to the shared category policy.
"""
attempt = 0
while True:
try:
return await operation()
except Exception as call_err:
attempt += 1
failure = classify_failure(call_err)
if not failure.retryable:
logger.error(f"{label} permanently failed [{failure.category.value}]: {call_err}")
_record_llm_event(
"llm_gave_up",
{
"source": "explorer",
"error": str(call_err)[:1000],
"category": failure.category.value,
"retryable": False,
},
status="failed",
)
raise LLMPermanentError(
f"{label} failed [{failure.category.value}]: {call_err}",
failure=failure,
cause=call_err,
) from call_err
policy = retry_policy_for(failure.category)
if attempt >= policy.max_attempts:
logger.error(
f"{label} exhausted {attempt} attempt(s) [{failure.category.value}]: {call_err}"
)
_record_llm_event(
"llm_gave_up",
{
"source": "explorer",
"error": str(call_err)[:1000],
"category": failure.category.value,
"retryable": True,
"attempts": attempt,
},
status="failed",
)
raise LLMExhaustedError(
f"{label} exhausted {attempt} attempt(s)"
f" [{failure.category.value}]: {call_err}",
failure=failure,
cause=call_err,
) from call_err
delay = policy.delay_for(attempt)
logger.warning(
f"{label} failed [{failure.category.value}] on attempt"
f" {attempt}/{policy.max_attempts}: {call_err}."
f" Retrying in {delay:.2f}s..."
)
_record_llm_retry(
str(call_err),
delay,
attempt=attempt,
max_retries=policy.max_attempts,
source="explorer",
)
await asyncio.sleep(delay)
def _bare_tool_name(name: str) -> str:
"""Strips a namespace prefix (``ns:tool``) some models add to tool names."""
return name.split(":")[-1] if ":" in name else name
@@ -145,9 +223,8 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
self, client, model_name: str, prompt_template: str, contents: list
):
"""Creates the explicit cache resource; returns None on failure."""
_ex = facade()
try:
_ex.logger.info("Creating cache resource for Explorer...")
logger.info("Creating cache resource for Explorer...")
cached_content = await client.aio.caches.create(
model=model_name,
config=types.CreateCachedContentConfig(
@@ -163,10 +240,10 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
ttl="300s",
),
)
_ex.logger.info(f"Cache resource created successfully: {cached_content.name}")
logger.info(f"Cache resource created successfully: {cached_content.name}")
return cached_content
except Exception as cache_err:
_ex.logger.error(f"Failed to create cache resource: {cache_err}")
logger.error(f"Failed to create cache resource: {cache_err}")
return None
async def _run_native(
@@ -234,7 +311,6 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
max_turns: int,
) -> str:
"""Main native reasoning loop: one model turn per iteration."""
_ex = facade()
turn = 0
agent_outcome = ""
self.turn_latencies = []
@@ -243,7 +319,7 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
while turn < max_turns:
turn += 1
_ex.logger.info(f"Turn {turn}/{max_turns}: Invoking Native Gemini SDK for Explorer...")
logger.info(f"Turn {turn}/{max_turns}: Invoking Native Gemini SDK for Explorer...")
self._prune_historical_images(contents, keep_last=1)
@@ -413,7 +489,6 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
thinking_level,
):
"""Invokes the model for one turn under a trace span; returns response and texts."""
_ex = facade()
ctx = self.ctx
start_turn = time.perf_counter()
with TraceSpan(name="gemini_explorer_call", ctx=ctx) as span:
@@ -425,7 +500,7 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
thinking_level=thinking_level,
)
response = await _ex._generate_content_with_reliability(
response = await _generate_content_with_reliability(
lambda: asyncio.wait_for(
client.aio.models.generate_content(
model=model_name,
@@ -476,8 +551,7 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
recorded as a synthetic model text part so the conversation history
stays well-formed for the retry.
"""
_ex = facade()
_ex.logger.warning(f"Explorer turn {turn}: Model hallucinated plain text. Forcing retry.")
logger.warning(f"Explorer turn {turn}: Model hallucinated plain text. Forcing retry.")
model_text = "\n".join(text_parts) if text_parts else ""
turn_record["tool_calls"].append(
{
@@ -573,7 +647,6 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
On validation failure the error is fed back as a tool response and
None is returned so the ReAct loop can continue and self-correct.
"""
_ex = facade()
submit_call = next(
(fc for fc in function_calls if _bare_tool_name(fc.name) == SUBMIT_TOOL),
None,
@@ -586,7 +659,7 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
errors = self._validate_submit_args(function_calls, args, candidates)
if errors:
_ex.logger.warning(f"Explorer submit_answer validation failed: {errors}")
logger.warning(f"Explorer submit_answer validation failed: {errors}")
turn_record["tool_calls"].append(
{
"name": SUBMIT_TOOL,
@@ -611,7 +684,7 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
# Allow the ReAct loop to continue so the model can self-correct
return None
_ex.logger.info("Explorer submit_answer validation passed successfully!")
logger.info("Explorer submit_answer validation passed successfully!")
# Candidates submitted by a registered label inherit that element's
# bounds; the rest of the model's arguments are passed through as is.
outcome_args = dict(args)
@@ -637,7 +710,6 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
Image loading failures are logged and skipped: the textual result is
still delivered, so one unreadable annotation cannot sink the turn.
"""
_ex = facade()
tool_response_parts.append(
types.Part.from_function_response(name=name, response={result_key: res.get("text")})
)
@@ -646,7 +718,7 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
try:
tool_response_parts.append(await get_image_part(img_p))
except Exception as e:
_ex.logger.warning(f"Failed to load image response part for {img_p}: {e}")
logger.warning(f"Failed to load image response part for {img_p}: {e}")
async def _execute_native_tool(
self,
@@ -730,7 +802,6 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
absent from the declarations, because models occasionally call tools
they were never offered.
"""
_ex = facade()
for fc in function_calls:
name = _bare_tool_name(fc.name)
if name == SUBMIT_TOOL:
@@ -738,7 +809,7 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
args = fc.args or {}
tool_call_trace = {"name": name, "args": args}
if name in self.denylisted_tools:
_ex.logger.warning(
logger.warning(
f"Explorer attempted to call denylisted tool '{name}'. Blocking execution."
)
error = f"Tool '{name}' is denylisted and unavailable."
@@ -749,14 +820,14 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
)
continue
_ex.logger.info(f"Explorer executing tool '{name}' sequentially...")
logger.info(f"Explorer executing tool '{name}' sequentially...")
try:
await self._execute_native_tool(
name, args, get_image_part, tool_call_trace, tool_response_parts
)
except Exception as e:
_ex.logger.error(f"Explorer tool {name} execution failed: {e}")
logger.error(f"Explorer tool {name} execution failed: {e}")
tool_call_trace["response"] = {"error": str(e)}
tool_response_parts.append(
types.Part.from_function_response(name=name, response={"error": str(e)})
@@ -766,8 +837,7 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
def _format_native_failure(self, e: Exception, model_name: str) -> str:
"""Maps a loop failure to the user-facing fallback outcome JSON."""
_ex = facade()
_ex.logger.error(f"Explorer execution loop failed: {e}")
logger.error(f"Explorer execution loop failed: {e}")
err_msg = str(e)
if "preempted" in err_msg.lower():
@@ -794,16 +864,15 @@ Initial marked UI elements list (corresponding to numbers ①, ②... in the ima
async def _cleanup_native_resources(self, client, cached_content, uploaded_files: list) -> None:
"""Deletes the cache resource and File API uploads created for this run."""
_ex = facade()
if cached_content:
try:
_ex.logger.info(f"Deleting cache resource: {cached_content.name}")
logger.info(f"Deleting cache resource: {cached_content.name}")
await client.aio.caches.delete(name=cached_content.name)
except Exception as cleanup_cache_err:
_ex.logger.warning(f"Failed to delete cache resource: {cleanup_cache_err}")
logger.warning(f"Failed to delete cache resource: {cleanup_cache_err}")
for file_ref in uploaded_files:
try:
await client.aio.files.delete(name=file_ref.name)
except Exception as cleanup_err:
_ex.logger.warning(f"Failed to delete uploaded file {file_ref.name}: {cleanup_err}")
_ex.logger.info(f"Explorer turn latencies: {self.turn_latencies}")
logger.warning(f"Failed to delete uploaded file {file_ref.name}: {cleanup_err}")
logger.info(f"Explorer turn latencies: {self.turn_latencies}")
artemis/agents/explorer/perception_tools.py
+16 -18
View File
@@ -19,9 +19,7 @@ group plus its search helpers, packaged as a mixin consumed by ``Explorer``.
Text search, the coordinate audit and the OCR inventory all answer from the
run's in-memory :class:`~artemis.agents.explorer.screen_index.ScreenIndex`;
no Data Engine round trip is involved, so an unsynced frame never degrades
to "no UI data". Patched collaborators (``settings``, ``draw_dots``,
``_run_object_detection``, ``logger``) are resolved through the facade module
at call time; see ``artemis.agents.explorer._facade``.
to "no UI data".
"""
import asyncio
@@ -33,11 +31,16 @@ from typing import TYPE_CHECKING, Any
import cv2
from artemis.agents.explorer._facade import facade
from artemis.agents.explorer.geometry import norm_to_pixel, pixel_to_norm
from artemis.agents.explorer.screen_index import ScreenElement, ScreenIndex, normalize_text
from artemis.agents.image_processor.image_processor import ImageProcessor
from artemis.agents.object_detector.object_detector import _run_object_detection
from artemis.config import settings
from artemis.data_engine.trace import trace
from artemis.utils.logger import get_logger
from artemis.utils.visualization import draw_dots
logger = get_logger(__name__)
#: Text-search answer when the screen could not be indexed at all.
NO_UI_TREE_MESSAGE = "No UI-tree data is available for this screen; rely on visual detection."
@@ -49,7 +52,6 @@ def load_detector_templates() -> tuple[list[str], float]:
Shared by ``exec_detect_objects`` and the flash-mode ``run`` path; the two
call sites historically carried an identical inline copy of this block.
"""
_ex = facade()
detector_prompt_path = Path(__file__).parent.parent / "object_detector" / "object_detector.json"
global_timeout = 30.0
try:
@@ -61,7 +63,7 @@ def load_detector_templates() -> tuple[list[str], float]:
if isinstance(raw_timeout, (int, float)) and raw_timeout > 0:
global_timeout = float(raw_timeout)
except Exception as e:
_ex.logger.warning(f"Failed to load detector prompt config: {e}")
logger.warning(f"Failed to load detector prompt config: {e}")
detector_templates = ["Point to the following objects in the provided image: {labels_str}."]
detector_instructions = ""
@@ -127,8 +129,7 @@ class PerceptionToolsMixin:
def _next_output_path(self, subdir: str) -> Path:
"""Next free ``images/<subdir>/<image>_<n>.jpg`` under the traces directory."""
_ex = facade()
out_dir = Path(_ex.settings.TRACES_PATH) / "images" / subdir
out_dir = Path(settings.TRACES_PATH) / "images" / subdir
out_dir.mkdir(parents=True, exist_ok=True)
stem = self.image_name or "temp_image"
max_seq = 0
@@ -147,15 +148,14 @@ class PerceptionToolsMixin:
textual result is still delivered, so one bad annotation never sinks
the tool call.
"""
_ex = facade()
if not self.screenshot_path:
return None
try:
output_path = self._next_output_path(subdir)
_ex.draw_dots(self.screenshot_path, points, labels, str(output_path), color=color)
draw_dots(self.screenshot_path, points, labels, str(output_path), color=color)
return str(output_path)
except Exception as e:
_ex.logger.warning(f"Failed to draw {subdir} dots: {e}")
logger.warning(f"Failed to draw {subdir} dots: {e}")
return None
# ------------------------------------------------------------------ #
@@ -237,7 +237,6 @@ class PerceptionToolsMixin:
prefix: str = "D",
color: str = "red",
) -> dict:
_ex = facade()
try:
if not self.screenshot_path:
return {
@@ -257,7 +256,7 @@ class PerceptionToolsMixin:
# Load object detector templates
templates, global_timeout = load_detector_templates()
result = await _ex._run_object_detection(
result = await _run_object_detection(
self.ctx,
target_path,
queries,
@@ -280,7 +279,7 @@ class PerceptionToolsMixin:
"image_path": str(target_path),
}
base_dir = _ex.settings.TRACES_PATH
base_dir = settings.TRACES_PATH
images_dir = base_dir / "images"
object_detect_dir = images_dir / "object_detect"
object_detect_dir.mkdir(parents=True, exist_ok=True)
@@ -341,7 +340,7 @@ class PerceptionToolsMixin:
f'[{label_id}] "{item.get("label")}" at [{x_orig_norm},{y_orig_norm}]'
)
_ex.draw_dots(
draw_dots(
self.screenshot_path,
points,
d_labels,
@@ -361,7 +360,7 @@ class PerceptionToolsMixin:
}
except Exception as e:
_ex.logger.warning(f"Failed to process detection output for annotation: {e}")
logger.warning(f"Failed to process detection output for annotation: {e}")
return {"text": str(result), "image_path": None}
except Exception as e:
return {
@@ -569,7 +568,6 @@ class PerceptionToolsMixin:
y_max: int,
zoom_factor: float = 2.0,
) -> dict:
_ex = facade()
try:
if not self.screenshot_path:
return {
@@ -613,7 +611,7 @@ class PerceptionToolsMixin:
if new_w > 0 and new_h > 0:
cropped = cv2.resize(cropped, (new_w, new_h))
base_dir = _ex.settings.TRACES_PATH
base_dir = settings.TRACES_PATH
images_dir = base_dir / "images"
inspect_region_dir = images_dir / "inspect_region"
inspect_region_dir.mkdir(parents=True, exist_ok=True)
artemis/agents/explorer/run_setup.py
+32 -38
View File
@@ -19,11 +19,7 @@ detection path, the per-run screen loading shared by every tier (image record
resolution, screen dimensions, fused hierarchy, in-memory screen index) and
the named preparation phases executed before a reasoning loop (image pool,
model parameters, tier-aware prompt construction, initial screenshot
annotation), packaged as a mixin consumed by ``Explorer``. Patched
collaborators (``settings``, ``StorageManager``, ``is_ocr_configured``,
``perform_ocr``, ``draw_dots``, ``_run_object_detection``, ``logger``) are
resolved through the facade module at call time; see
``artemis.agents.explorer._facade``.
annotation), packaged as a mixin consumed by ``Explorer``.
"""
import base64
@@ -37,15 +33,24 @@ from typing import TYPE_CHECKING, Any
import httpx
from artemis.agents.explorer._facade import facade
from artemis.agents.explorer.constants import EXECUTION_CONSTRAINT_TEMPLATE
from artemis.agents.explorer.constants import (
DEFAULT_EXPLORER_MODEL,
EXECUTION_CONSTRAINT_TEMPLATE,
)
from artemis.agents.explorer.geometry import resolve_screen_size
from artemis.agents.explorer.perception_tools import load_detector_templates
from artemis.agents.explorer.screen_index import ScreenElement, ScreenIndex
from artemis.agents.explorer.tiers import SUBMIT_TOOL, ExplorerTier
from artemis.agents.object_detector.object_detector import _run_object_detection
from artemis.config import settings
from artemis.data_engine.storage import StorageManager
from artemis.graph.state import State
from artemis.utils.logger import get_logger
from artemis.utils.ocr_api import is_ocr_configured, perform_ocr
from artemis.utils.ocr_xml_fusion import fuse_ocr_with_xml
from artemis.utils.visualization import format_minimal_list_with_elements
from artemis.utils.visualization import draw_dots, format_minimal_list_with_elements
logger = get_logger(__name__)
#: Prompt bullets may be plain strings or ``{"text": ..., "requires": [...]}``
#: objects; the latter are emitted only when every listed tool is exposed.
@@ -109,13 +114,12 @@ class RunSetupMixin:
Several targets may be listed in ``query`` separated by ``|``; each
detection becomes a ``D<n>`` candidate described by its detected label.
"""
_ex = facade()
templates, global_timeout = load_detector_templates()
queries = [q.strip() for q in query.split("|") if q.strip()]
try:
result = await _ex._run_object_detection(
result = await _run_object_detection(
self.ctx,
screenshot_path,
queries,
@@ -140,12 +144,11 @@ class RunSetupMixin:
ensure_ascii=False,
)
except Exception as e:
_ex.logger.error(f"Flash mode object detection failed: {e}")
logger.error(f"Flash mode object detection failed: {e}")
return self._failure_outcome(f"Flash mode detection error: {e}")
def _resolve_image_record(self, screenshot_path: str) -> tuple:
"""Computes the screenshot hash and looks it up in the Data Engine DB."""
_ex = facade()
image_name = None
record = None
try:
@@ -154,24 +157,24 @@ class RunSetupMixin:
for byte_block in iter(lambda: f.read(4096), b""):
sha256_hash.update(byte_block)
computed_hash = sha256_hash.hexdigest()
_ex.logger.info(f"Computed screenshot hash: {computed_hash}")
logger.info(f"Computed screenshot hash: {computed_hash}")
# Check if it exists in DB
db_path = _ex.settings.DATA_ENGINE_DB_PATH
base_dir = _ex.settings.TRACES_PATH
storage = _ex.StorageManager(db_path, base_dir)
db_path = settings.DATA_ENGINE_DB_PATH
base_dir = settings.TRACES_PATH
storage = StorageManager(db_path, base_dir)
record = storage.get_image(computed_hash)
if record:
image_name = computed_hash
else:
_ex.logger.warning(
logger.warning(
f"Image hash {computed_hash} not found in Data Engine DB."
" Data Engine might not be synced yet."
)
except Exception as e:
_ex.logger.warning(f"Failed to compute hash or check DB: {e}")
logger.warning(f"Failed to compute hash or check DB: {e}")
return image_name, record
@@ -216,7 +219,6 @@ class RunSetupMixin:
def _resolve_model_params(self) -> tuple:
"""Resolves model name, temperature, thinking level, and fallback model."""
_ex = facade()
llm_config = getattr(self.ctx, "llm_config", None)
llm_cfg = getattr(llm_config, "explorer", None) if llm_config else None
temperature = 0.1
@@ -235,7 +237,7 @@ class RunSetupMixin:
if "/" in fallback_model:
fallback_model = fallback_model.split("/")[-1]
else:
model_name = _ex.DEFAULT_EXPLORER_MODEL
model_name = DEFAULT_EXPLORER_MODEL
return model_name, temperature, thinking_level, fallback_model
@@ -303,32 +305,27 @@ class RunSetupMixin:
HTTP client backing on-the-fly OCR is created here on first need and
released by ``Explorer.run`` regardless of the engine used.
"""
_ex = facade()
fused_xml: Any = []
if record and getattr(record, "ui_tree", None):
ui_tree = record.ui_tree
ocr_results = getattr(record, "ocr_result", None)
if ocr_results is None:
if allow_ocr and _ex.is_ocr_configured():
if allow_ocr and is_ocr_configured():
try:
_ex.logger.info(
"Previous screenshot OCR is missing. Running OCR on-the-fly..."
)
logger.info("Previous screenshot OCR is missing. Running OCR on-the-fly...")
with open(screenshot_path, "rb") as img_file:
img_b64 = base64.b64encode(img_file.read()).decode("utf-8")
if self.http_client is None:
self.http_client = httpx.AsyncClient()
ocr_results = await _ex.perform_ocr(img_b64, client=self.http_client)
ocr_results = await perform_ocr(img_b64, client=self.http_client)
except Exception as ocr_err:
_ex.logger.error(
f"On-the-fly OCR failed for previous screenshot: {ocr_err}"
)
logger.error(f"On-the-fly OCR failed for previous screenshot: {ocr_err}")
ocr_results = []
else:
ocr_results = []
fused_xml = fuse_ocr_with_xml(ui_tree, ocr_results or [])
_ex.logger.info("Successfully loaded and fused UI hierarchy for previous screenshot.")
logger.info("Successfully loaded and fused UI hierarchy for previous screenshot.")
elif screenshot_path == getattr(state, "latest_screenshot", None):
fused_xml = getattr(state, "latest_ui_hierarchy", None)
return list(fused_xml) if isinstance(fused_xml, (list, tuple)) else []
@@ -341,13 +338,10 @@ class RunSetupMixin:
Every numbered label is registered against its element so a
candidate the model later submits by number inherits real bounds.
"""
_ex = facade()
ctx = self.ctx
marked_path = None
try:
_ex.logger.info(
"Explorer self-annotating initial screenshot using latest_ui_hierarchy..."
)
logger.info("Explorer self-annotating initial screenshot using latest_ui_hierarchy...")
formatted_list, elements, labels = format_minimal_list_with_elements(
fused_xml, self.width, self.height
)
@@ -363,7 +357,7 @@ class RunSetupMixin:
else None
)
if not base_dir:
base_dir = _ex.settings.TRACES_PATH
base_dir = settings.TRACES_PATH
images_dir = base_dir / "images"
initial_marked_dir = images_dir / "initial_marked"
initial_marked_dir.mkdir(parents=True, exist_ok=True)
@@ -380,12 +374,12 @@ class RunSetupMixin:
marked_path = initial_marked_dir / f"{image_name or 'temp_image'}_{seq}.jpg"
# Draw dots on the raw screenshot
_ex.draw_dots(screenshot_path, points, labels, str(marked_path))
_ex.logger.info(
draw_dots(screenshot_path, points, labels, str(marked_path))
logger.info(
f"Successfully drew {len(points)} dots and saved marked image to {marked_path}"
)
except Exception as e:
_ex.logger.error(f"Failed to self-annotate initial screenshot: {e}")
logger.error(f"Failed to self-annotate initial screenshot: {e}")
return minimal_list, marked_path
def _prepare_initial_annotation(
artemis/agents/explorer/tool_declarations.py
+1 -2
View File
@@ -17,8 +17,7 @@
Split out of ``artemis.agents.explorer.explorer`` as pure data:
``UNIVERSAL_EXPLORER_TOOLS`` (OpenAI-style dict schemas for the universal
LangChain path) and ``NATIVE_EXPLORER_TOOL_DECLARATIONS`` (google-genai
``FunctionDeclaration`` objects for the native Gemini path). Both are
re-exported by the ``explorer`` facade module.
``FunctionDeclaration`` objects for the native Gemini path).
"""
from google.genai import types
artemis/agents/explorer/universal_runner.py
+11 -14
View File
@@ -15,10 +15,7 @@
"""Universal (LangChain ChatModel) execution loop for the Explorer agent.
Split out of ``artemis.agents.explorer.explorer``: the ``_run_universal``
reasoning loop, packaged as a mixin consumed by ``Explorer``. Patched
collaborators (``get_llm``, ``logger``, ``UNIVERSAL_EXPLORER_TOOLS``) are
resolved through the facade module at call time; see
``artemis.agents.explorer._facade``.
reasoning loop, packaged as a mixin consumed by ``Explorer``.
"""
import asyncio
@@ -33,10 +30,14 @@ from langchain_core.messages import (
ToolMessage,
)
from artemis.agents.explorer._facade import facade
from artemis.agents.explorer.geometry import is_valid_norm_point
from artemis.agents.explorer.tiers import SUBMIT_TOOL
from artemis.agents.explorer.tool_declarations import UNIVERSAL_EXPLORER_TOOLS
from artemis.graph.state import State
from artemis.services.llm import get_llm
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
class UniversalRunnerMixin:
@@ -72,9 +73,8 @@ class UniversalRunnerMixin:
def _universal_exposed_tools(self) -> list[dict[str, Any]]:
"""OpenAI-style tool schemas the model may see (tier-hidden ones excluded)."""
_ex = facade()
hidden = self._hidden_tool_names()
return [t for t in _ex.UNIVERSAL_EXPLORER_TOOLS if t["function"]["name"] not in hidden]
return [t for t in UNIVERSAL_EXPLORER_TOOLS if t["function"]["name"] not in hidden]
async def _run_universal(
self,
@@ -88,8 +88,7 @@ class UniversalRunnerMixin:
max_turns: int,
) -> str:
"""Executes the Explorer reasoning loop via a universal LangChain ChatModel."""
_ex = facade()
llm = _ex.get_llm(self.ctx, name="explorer")
llm = get_llm(self.ctx, name="explorer")
exposed_tools = self._universal_exposed_tools()
bound_llm = llm.bind_tools(exposed_tools)
@@ -124,7 +123,7 @@ class UniversalRunnerMixin:
tool_calls = getattr(response, "tool_calls", None) or []
if not tool_calls:
_ex.logger.warning(f"Universal Explorer turn {turn}: No tool calls generated.")
logger.warning(f"Universal Explorer turn {turn}: No tool calls generated.")
if is_final_turn:
return json.dumps(
{
@@ -256,7 +255,6 @@ class UniversalRunnerMixin:
the images ride on a follow-up human turn instead; unreadable files
are skipped so a missing annotation never aborts the loop.
"""
_ex = facade()
blocks: list[dict[str, Any]] = []
tool_names: list[str] = []
for tool_name, path in images:
@@ -264,7 +262,7 @@ class UniversalRunnerMixin:
with open(path, "rb") as f:
img_b64 = base64.b64encode(f.read()).decode("utf-8")
except OSError as read_err:
_ex.logger.warning(f"Skipping unreadable tool image {path}: {read_err}")
logger.warning(f"Skipping unreadable tool image {path}: {read_err}")
continue
if tool_name not in tool_names:
tool_names.append(tool_name)
@@ -283,7 +281,6 @@ class UniversalRunnerMixin:
self, tool_calls: list, messages: list[BaseMessage], turn: int
) -> None:
"""Executes non-submit tool calls, appending ToolMessages and their images."""
_ex = facade()
images: list[tuple[str, str]] = []
for tc in tool_calls:
name = tc.get("name")
@@ -306,7 +303,7 @@ class UniversalRunnerMixin:
messages.append(ToolMessage(tool_call_id=call_id, name=name, content=res_text))
images.extend((name, path) for path in self._tool_image_paths(res))
except Exception as tool_err:
_ex.logger.error(f"Error executing tool '{name}': {tool_err}")
logger.error(f"Error executing tool '{name}': {tool_err}")
messages.append(
ToolMessage(
tool_call_id=call_id,