feat: add Chinese UI, Responses models, and mirrored Windows setup
Localize the web console and persist custom model and capability settings. Use OpenAI Responses exclusively and move runtime limits into environment configuration. Bootstrap uv and the virtual environment on Windows, support optional China mirrors, and package source distributions without environments, build output, or credentials. Update regression tests and repository quality checks for the new behavior.
This commit is contained in:
chenmingxuan committed
1 parent
371aa6df56
commit
2ddec53954
183 files changed
+3427
-11857
No files matched your search
@@ -86,14 +86,14 @@ class ReplayManager:
|
||||
It reconstructs historical execution states (SQLite database records, trace
|
||||
histories, and screenshots) and replays a single agent step in an isolated,
|
||||
deterministic sandbox environment while interacting live with a physical
|
||||
Android device and Gemini.
|
||||
Android device and OpenAI Responses.
|
||||
|
||||
Key Responsibilities:
|
||||
- Reconstructs and preloads historical SQLite states up to a specific
|
||||
step.
|
||||
- Establishes sandboxed `ArtemisContext` and `DataEngine` instances.
|
||||
- Integrates with physical devices via ADB and UIAutomator.
|
||||
- Hooks and spies on the Gemini API to capture thoughts and raw outputs.
|
||||
- Records Responses model calls to capture thoughts and raw outputs.
|
||||
- Reconciles live and preloaded traces into unified chronological trees.
|
||||
|
||||
Attributes:
|
||||
@@ -1552,8 +1552,8 @@ class ReplayManager:
|
||||
# Update values
|
||||
conn_temp.execute(
|
||||
"""
|
||||
UPDATE video_recordings
|
||||
SET local_video_path = ?, start_time = ?, end_time = ?
|
||||
UPDATE video_recordings
|
||||
SET local_video_path = ?, start_time = ?, end_time = ?
|
||||
WHERE session_id = ?
|
||||
""",
|
||||
(
|
||||
@@ -1583,8 +1583,8 @@ class ReplayManager:
|
||||
|
||||
conn_temp.execute(
|
||||
"""
|
||||
UPDATE video_recordings
|
||||
SET local_video_path = ?, start_time = ?, end_time = ?
|
||||
UPDATE video_recordings
|
||||
SET local_video_path = ?, start_time = ?, end_time = ?
|
||||
WHERE session_id = ?
|
||||
""",
|
||||
(
|
||||
@@ -1885,121 +1885,34 @@ class ReplayManager:
|
||||
|
||||
data_engine = ctx.data_engine
|
||||
|
||||
print("Executing using Live Gemini Client...")
|
||||
print("Executing using OpenAI Responses...")
|
||||
|
||||
from google.genai.models import AsyncModels
|
||||
from langchain_openai import ChatOpenAI
|
||||
from unittest.mock import patch
|
||||
|
||||
original_generate_content = AsyncModels.generate_content
|
||||
original_generate = ChatOpenAI._agenerate
|
||||
|
||||
async def wrapped_generate_content(self, *, model, contents, config=None, **kwargs):
|
||||
system_instruction = None
|
||||
if config and config.system_instruction:
|
||||
if isinstance(config.system_instruction, str):
|
||||
system_instruction = config.system_instruction
|
||||
elif hasattr(config.system_instruction, "parts"):
|
||||
parts = []
|
||||
for p in config.system_instruction.parts:
|
||||
if hasattr(p, "text") and p.text:
|
||||
parts.append(p.text)
|
||||
system_instruction = " ".join(parts)
|
||||
else:
|
||||
system_instruction = str(config.system_instruction)
|
||||
|
||||
prompt_data = {
|
||||
"model": model,
|
||||
"system_instruction": system_instruction,
|
||||
"contents": [],
|
||||
async def wrapped_generate(model, messages, *args, **kwargs):
|
||||
record = {
|
||||
"prompt": {
|
||||
"model": model.model_name,
|
||||
"contents": [
|
||||
{"role": message.type, "parts": message.content} for message in messages
|
||||
],
|
||||
},
|
||||
"response": None,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
contents_list = contents if isinstance(contents, list) else [contents]
|
||||
for content in contents_list:
|
||||
parts_desc = []
|
||||
if hasattr(content, "parts") and content.parts:
|
||||
for part in content.parts:
|
||||
if hasattr(part, "text") and part.text:
|
||||
parts_desc.append({"type": "text", "text": part.text})
|
||||
elif hasattr(part, "inline_data") and part.inline_data:
|
||||
parts_desc.append(
|
||||
{
|
||||
"type": "image",
|
||||
"mime_type": part.inline_data.mime_type,
|
||||
"length": len(part.inline_data.data),
|
||||
}
|
||||
)
|
||||
elif hasattr(part, "function_call") and part.function_call:
|
||||
parts_desc.append(
|
||||
{
|
||||
"type": "function_call",
|
||||
"name": part.function_call.name,
|
||||
"args": (
|
||||
dict(part.function_call.args)
|
||||
if part.function_call.args
|
||||
else {}
|
||||
),
|
||||
}
|
||||
)
|
||||
elif hasattr(part, "function_call") and part.function_response:
|
||||
parts_desc.append(
|
||||
{
|
||||
"type": "function_response",
|
||||
"name": part.function_response.name,
|
||||
"response": (
|
||||
dict(part.function_response.response)
|
||||
if part.function_response.response
|
||||
else {}
|
||||
),
|
||||
}
|
||||
)
|
||||
else:
|
||||
parts_desc.append({"type": "raw", "value": str(content)})
|
||||
|
||||
prompt_data["contents"].append(
|
||||
{
|
||||
"role": getattr(content, "role", "unknown"),
|
||||
"parts": parts_desc,
|
||||
}
|
||||
)
|
||||
|
||||
record = {"prompt": prompt_data, "response": None, "error": None}
|
||||
llm_calls_log.append(record)
|
||||
|
||||
try:
|
||||
res = await original_generate_content(
|
||||
self,
|
||||
model=model,
|
||||
contents=contents,
|
||||
config=config,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
thinking_parts = []
|
||||
if hasattr(res, "candidates") and res.candidates and res.candidates[0].content:
|
||||
for part in res.candidates[0].content.parts:
|
||||
if getattr(part, "thought", False):
|
||||
thinking_parts.append(part.text)
|
||||
|
||||
resp_data = {
|
||||
"text": res.text if hasattr(res, "text") else None,
|
||||
"function_calls": [],
|
||||
"thought": ("\n".join(thinking_parts) if thinking_parts else None),
|
||||
}
|
||||
if hasattr(res, "function_calls") and res.function_calls:
|
||||
for fc in res.function_calls:
|
||||
resp_data["function_calls"].append(
|
||||
{
|
||||
"name": fc.name,
|
||||
"args": dict(fc.args) if fc.args else {},
|
||||
}
|
||||
)
|
||||
record["response"] = resp_data
|
||||
return res
|
||||
except Exception as err:
|
||||
err_str = str(err)
|
||||
record["error"] = (
|
||||
f"{err.__class__.__name__}: {err_str}" if err_str else err.__class__.__name__
|
||||
)
|
||||
raise err
|
||||
result = await original_generate(model, messages, *args, **kwargs)
|
||||
record["response"] = [
|
||||
generation.message.model_dump(mode="json") for generation in result.generations
|
||||
]
|
||||
return result
|
||||
except Exception as error:
|
||||
record["error"] = error.__class__.__name__
|
||||
raise
|
||||
|
||||
try:
|
||||
try:
|
||||
@@ -2042,7 +1955,7 @@ class ReplayManager:
|
||||
val = resolved_val
|
||||
tool_args[name] = val
|
||||
|
||||
with patch.object(AsyncModels, "generate_content", wrapped_generate_content):
|
||||
with patch.object(ChatOpenAI, "_agenerate", wrapped_generate):
|
||||
outcome = await tool_fn(**tool_args)
|
||||
|
||||
replay_result["success"] = True
|
||||
|
||||
Reference in new issue
Block a user