feat: add Chinese UI, Responses models, and mirrored Windows setup
Localize the web console and persist custom model and capability settings. Use OpenAI Responses exclusively and move runtime limits into environment configuration. Bootstrap uv and the virtual environment on Windows, support optional China mirrors, and package source distributions without environments, build output, or credentials. Update regression tests and repository quality checks for the new behavior.
This commit is contained in:
chenmingxuan committed
1 parent
371aa6df56
commit
2ddec53954
183 files changed
+3427
-11857
No files matched your search
@@ -14,14 +14,9 @@
|
||||
|
||||
"""Static tool declarations for the Explorer agent.
|
||||
|
||||
Split out of ``artemis.agents.explorer.explorer`` as pure data:
|
||||
``UNIVERSAL_EXPLORER_TOOLS`` (OpenAI-style dict schemas for the universal
|
||||
LangChain path) and ``NATIVE_EXPLORER_TOOL_DECLARATIONS`` (google-genai
|
||||
``FunctionDeclaration`` objects for the native Gemini path).
|
||||
OpenAI function schemas consumed by the Responses model adapter.
|
||||
"""
|
||||
|
||||
from google.genai import types
|
||||
|
||||
UNIVERSAL_EXPLORER_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
@@ -183,212 +178,3 @@ UNIVERSAL_EXPLORER_TOOLS = [
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
NATIVE_EXPLORER_TOOL_DECLARATIONS = [
|
||||
types.FunctionDeclaration(
|
||||
name="ask_perception_tool",
|
||||
description=(
|
||||
"[Perception] Concurrently executes and awaits three sub-tasks"
|
||||
" in parallel, including UI-tree text search, coordinate audit,"
|
||||
" and visual object detection on the screen.\n\nOutput"
|
||||
" Format:\nReturns:'text' (str): A single string consolidating"
|
||||
" results with 0-1000 normalized coordinates, including UI"
|
||||
" search matches labeled [X],"
|
||||
" coordinate search matches labeled [X], and visually detected"
|
||||
" object targets labeled [D].\n(Annotated screenshots are also"
|
||||
" added to the context containing corresponding colored label"
|
||||
" dots)."
|
||||
),
|
||||
parameters=types.Schema(
|
||||
type=types.Type.OBJECT,
|
||||
properties={
|
||||
"search_query": types.Schema(
|
||||
type=types.Type.STRING,
|
||||
description=("Required. Text query to search for in UI elements."),
|
||||
),
|
||||
"nx": types.Schema(
|
||||
type=types.Type.INTEGER,
|
||||
description=(
|
||||
"Required. Normalized X coordinate (0-1000 range)"
|
||||
" to audit for overlapping elements."
|
||||
),
|
||||
),
|
||||
"ny": types.Schema(
|
||||
type=types.Type.INTEGER,
|
||||
description=(
|
||||
"Required. Normalized Y coordinate (0-1000 range)"
|
||||
" to audit for overlapping elements."
|
||||
),
|
||||
),
|
||||
"detect_queries": types.Schema(
|
||||
type=types.Type.ARRAY,
|
||||
items=types.Schema(type=types.Type.STRING),
|
||||
description=(
|
||||
"Required. List of query strings to visually detect on the screen."
|
||||
),
|
||||
),
|
||||
},
|
||||
required=["search_query", "nx", "ny", "detect_queries"],
|
||||
),
|
||||
),
|
||||
types.FunctionDeclaration(
|
||||
name="detect_objects",
|
||||
description=(
|
||||
"[Perception] Locates elements, shapes and more using a"
|
||||
" VLM. Returns their normalized coordinates (in a [0, 1000]"
|
||||
" scale) labeled with [D1], [D2] etc. in the text output, along"
|
||||
" with an annotated image containing corresponding visual label"
|
||||
" dots."
|
||||
),
|
||||
parameters=types.Schema(
|
||||
type=types.Type.OBJECT,
|
||||
properties={
|
||||
"target_image_id": types.Schema(
|
||||
type=types.Type.STRING,
|
||||
description="The ID of the image to process.",
|
||||
),
|
||||
"queries": types.Schema(
|
||||
type=types.Type.ARRAY,
|
||||
items=types.Schema(type=types.Type.STRING),
|
||||
description=("A list of single query strings to detect on the screen."),
|
||||
),
|
||||
},
|
||||
required=["queries", "target_image_id"],
|
||||
),
|
||||
),
|
||||
types.FunctionDeclaration(
|
||||
name="get_ocr_list",
|
||||
description=(
|
||||
"[Perception] Retrieves all text elements detected on the"
|
||||
" original, unprocessed screen via OCR. Returns their"
|
||||
" normalized coordinates (in a [0, 1000] scale) labeled with"
|
||||
" [O1], [O2] etc. in the text output, along with an annotated"
|
||||
" image containing corresponding visual label dots."
|
||||
),
|
||||
parameters=types.Schema(
|
||||
type=types.Type.OBJECT,
|
||||
properties={},
|
||||
),
|
||||
),
|
||||
types.FunctionDeclaration(
|
||||
name="ask_image_processor",
|
||||
description=(
|
||||
"[Transformation] A scripting tool for pixel-level image"
|
||||
" processing. CAUTION: This tool is slow and expensive. Returns"
|
||||
" a summary of the operations performed and new image IDs,"
|
||||
" along with new labels and screen coordinates."
|
||||
),
|
||||
parameters=types.Schema(
|
||||
type=types.Type.OBJECT,
|
||||
properties={
|
||||
"target_image_id": types.Schema(
|
||||
type=types.Type.STRING,
|
||||
description=(
|
||||
"The ID of the image in the Image Pool to start"
|
||||
" from (e.g. 'img_0', 'img_1')."
|
||||
),
|
||||
),
|
||||
"instruction": types.Schema(
|
||||
type=types.Type.STRING,
|
||||
description=(
|
||||
"1-2 precise commands describing a simple task."
|
||||
" Include all necessary coordinates and other"
|
||||
" parameters.\n"
|
||||
),
|
||||
),
|
||||
},
|
||||
required=["instruction", "target_image_id"],
|
||||
),
|
||||
),
|
||||
types.FunctionDeclaration(
|
||||
name="submit_answer",
|
||||
description=(
|
||||
"[Finalization] Submits the final list of ranked candidate UI"
|
||||
" elements matching the search query, or a fallback message if"
|
||||
" no elements were found, to complete the search task. Fails"
|
||||
" validation if: (1) candidates is empty and no"
|
||||
" fallback_message is provided, (2) coordinates are not exactly"
|
||||
" 2 integers [nx, ny], or (3) coordinates lie outside"
|
||||
" normalized [0, 1000] range."
|
||||
),
|
||||
parameters=types.Schema(
|
||||
type=types.Type.OBJECT,
|
||||
properties={
|
||||
"candidates": types.Schema(
|
||||
type=types.Type.ARRAY,
|
||||
items=types.Schema(
|
||||
type=types.Type.OBJECT,
|
||||
properties={
|
||||
"label": types.Schema(
|
||||
type=types.Type.STRING,
|
||||
description=(
|
||||
"Reference label (e.g. X1, O2, D3, or a number like 3)"
|
||||
),
|
||||
),
|
||||
"coords": types.Schema(
|
||||
type=types.Type.ARRAY,
|
||||
items=types.Schema(type=types.Type.INTEGER),
|
||||
description=("[nx, ny] coordinates in normalized 0-1000 scale"),
|
||||
),
|
||||
"description": types.Schema(
|
||||
type=types.Type.STRING,
|
||||
description=("Brief description of this candidate element"),
|
||||
),
|
||||
},
|
||||
required=["label", "coords"],
|
||||
),
|
||||
description=(
|
||||
"A ranked list of candidate UI elements that match"
|
||||
" the query, sorted from highest to lowest"
|
||||
" confidence (maximum 10 candidates). Leave empty"
|
||||
" if no matching elements were found on the screen."
|
||||
),
|
||||
),
|
||||
"fallback_message": types.Schema(
|
||||
type=types.Type.STRING,
|
||||
description=(
|
||||
"Optional fallback explanation if target elements"
|
||||
" were not found or if there are special"
|
||||
" observations."
|
||||
),
|
||||
),
|
||||
},
|
||||
required=["candidates"],
|
||||
),
|
||||
),
|
||||
types.FunctionDeclaration(
|
||||
name="inspect_region",
|
||||
description=(
|
||||
"[Perception] Crops and resizes a bounding box region of the"
|
||||
" original screenshot. Returns the cropped and resized image"
|
||||
" directly, without any image ID or coordinate transform."
|
||||
),
|
||||
parameters=types.Schema(
|
||||
type=types.Type.OBJECT,
|
||||
properties={
|
||||
"x_min": types.Schema(
|
||||
type=types.Type.INTEGER,
|
||||
description="Left X coordinate (0-1000).",
|
||||
),
|
||||
"y_min": types.Schema(
|
||||
type=types.Type.INTEGER,
|
||||
description="Top Y coordinate (0-1000).",
|
||||
),
|
||||
"x_max": types.Schema(
|
||||
type=types.Type.INTEGER,
|
||||
description="Right X coordinate (0-1000).",
|
||||
),
|
||||
"y_max": types.Schema(
|
||||
type=types.Type.INTEGER,
|
||||
description="Bottom Y coordinate (0-1000).",
|
||||
),
|
||||
"zoom_factor": types.Schema(
|
||||
type=types.Type.NUMBER,
|
||||
description=("Zoom factor to upscale the cropped region (1.0 to 4.0)."),
|
||||
),
|
||||
},
|
||||
required=["x_min", "y_min", "x_max", "y_max", "zoom_factor"],
|
||||
),
|
||||
),
|
||||
]
|
||||
Reference in new issue
Block a user