chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

runtime: harden LLM retry handling and operator workflows

Browse Source
This commit is contained in:
somew1nd committed 2026-08-21 00:34:44 -07:00
1 parent 0f5fb1d32c
commit 71e5bba551
18 files changed
+937 -348
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

artemis/agents/operator/operator.py
+12 -12
View File
@@ -93,6 +93,7 @@ from artemis.agents.operator.prompts import (
ToolLimitWarningPromptComponent,
InjectedInstructionPromptComponent,
)
from artemis.agents.operator.prompt_builder import load_operator_prompts
class OperatorNode:
@@ -107,13 +108,11 @@ class OperatorNode:
self.tools = tools or []
self.prompt_components = prompt_components or []
self.last_n_detailed = last_n_detailed
self.prompts = {}
prompts_path = Path(__file__).parent / "operator.json"
if prompts_path.exists():
try:
self.prompts = json.loads(prompts_path.read_text(encoding="utf-8"))
except Exception as e:
logger.error(f"Failed to load operator prompts: {e}")
try:
self.prompts = load_operator_prompts()
except (OSError, ValueError, json.JSONDecodeError) as e:
logger.error(f"Failed to load operator prompts: {e}")
self.prompts = {}
def _check_infinite_loop(self, state: State):
subagent_calls = state.subagent_calls or []
@@ -594,8 +593,9 @@ class OperatorNode:
" screen action you attempted to"
" execute was rejected because you are"
" not allowed to simultaneously"
" read/write memory and execute"
" physical actions in the same turn."
" use result-dependent pre-decision"
" tools and execute physical actions in"
" the same turn."
" Currently, no screen actions have"
" been executed. Please review your"
" updated context and re-output your"
@@ -614,9 +614,9 @@ class OperatorNode:
" successfully processed. However, your"
" response to the checker was rejected"
" because you are not allowed to"
" simultaneously read/write memory and"
" reply to the checker in the same"
" turn. Currently, no response has been"
" simultaneously use result-dependent"
" pre-decision tools and reply to the"
" checker. Currently, no response has been"
" sent to the checker. Please review"
" your updated context and re-output"
" your reply."
artemis/agents/operator/prompt_builder.py
+20 -14
View File
@@ -12,21 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Dynamic prompt template builder for Operator Agent."""
"""Compatibility prompt builder backed by the canonical Operator templates."""
import json
from functools import lru_cache
from pathlib import Path
from jinja2 import Template
OPERATOR_SYSTEM_PROMPT = """You are ARTEMIS Operator, an expert autonomous mobile device interaction agent.
Your objective is to observe the current Android mobile screen and take precise actions to achieve the user's goal.
Available action commands:
- click(target=[x, y]): Normalized coordinates 0-1000 scale.
- swipe(action="up"|"down"|"left"|"right"|[x1, y1, x2, y2]): Drag across screen.
- input_text(text="...", target=[x, y]): Click field and type.
- press_key(key="home"|"back"|"enter"|"delete"): Press hardware key.
- wait_for_delay(seconds=...): Pause execution.
"""
OPERATOR_HUMAN_TEMPLATE = """Goal: {{ goal }}
Active Sub-Goal: {{ sub_goal }}
Current Turn: {{ current_turn }}
@@ -35,12 +28,25 @@ Execution History:
"""
@lru_cache(maxsize=1)
def load_operator_prompts() -> dict[str, str]:
"""Load the single canonical prompt source shared by all Operator entry points."""
prompts_path = Path(__file__).with_name("operator.json")
return json.loads(prompts_path.read_text(encoding="utf-8"))
class OperatorPromptBuilder:
"""Builds prompt messages and multimodal contents for the Operator Agent."""
@classmethod
def build_system_message(cls) -> str:
return OPERATOR_SYSTEM_PROMPT
def build_system_message(cls, template_name: str = "main_template") -> str:
from artemis.agents.operator.prompts import apply_operator_prompt_contract
prompts = load_operator_prompts()
try:
return apply_operator_prompt_contract(prompts[template_name])
except KeyError as exc:
raise ValueError(f"Unknown Operator template: {template_name}") from exc
@classmethod
def build_human_message(
artemis/agents/operator/prompts.py
+48 -13
View File
@@ -27,14 +27,59 @@ from artemis.tools.command_tool import (
_format_long_output_response,
_is_output_long,
)
from artemis.agents.explorer.constants import EXPLORE_DESCRIPTIONS
from artemis.config import resolve_explorer_version
from artemis.utils.logger import get_logger
from artemis.utils.task_tree import get_active_subgoal_hashes
logger = get_logger(__name__)
_LEGACY_TOOL_CALLING_RULE = """- **Tool Calling Iron Rule**: Do NOT submit a Turn-Ending Action at the same time (in the same tool-call list) as a Pre-Decision Exploratory Tool. You must first gather information (e.g., finding coordinates or diagnosing a state), manage ADB background tasks, or update memory notes. You can use pre-decision exploratory tools to complete this task. You will be prompted again with the tool's result, at which point you can output your final physical/turn-ending action."""
_TOOL_CALLING_RULE = """- **Tool Calling Contract**: Tools that return information needed for the decision (`ask_explorer`, `ask_diagnoser`, `video_analyzer`, `run_adb_command`, `manage_task`, `analyze_task_output`, `read_note`, `list_notes`, and `save_note`) are result-dependent Pre-Decision Tools. Do not submit them in the same tool-call list as a Turn-Ending Action; inspect their result first. The write-through tools `update_note` and `append_note` may accompany at most one Turn-Ending Action when their content is based entirely on evidence already available before that action. Never write an action's expected outcome as if it had already been observed."""
_LEGACY_TOOL_PROTOCOL = """1. **Pre-Decision Exploratory Tools**:
- *What they are*: Helper/Subagent tools (such as `ask_explorer`, `ask_diagnoser`, `video_analyzer`), ADB command tools (`run_adb_command`, `manage_task`), and memory note tools (`read_note`, `list_notes`, `save_note`, `update_note`, `append_note`).
- *How they work*: These tools gather details or update/read memory. When you invoke these tools, the framework will immediately execute them and return the results, allowing you to continue thinking and make your final decision.
2. **Turn-Ending Actions**:"""
_TOOL_PROTOCOL = """1. **Result-Dependent Pre-Decision Tools**:
- *What they are*: Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`), ADB/task tools (`run_adb_command`, `manage_task`, `analyze_task_output`), and memory tools whose result must be inspected (`read_note`, `list_notes`, `save_note`).
- *How they work*: Invoke these without a Turn-Ending Action, inspect the returned result, and then decide.
2. **Write-Through Memory Tools**:
- `update_note` and `append_note` may run alongside at most one Turn-Ending Action only when recording facts already observed in the current context. If the note content depends on the action's result, wait for the next observation before writing it.
3. **Turn-Ending Actions**:"""
_LEGACY_CHECKER_REJECTION_TRIGGER = """ - **Validation/Checker Rejection**: The verification agent (Checker) rejected your subgoal completion (i.e., you are in troubleshooter mode and this is a retry)."""
_AMBIGUOUS_CHECKER_REJECTION_TRIGGER = """ - **Ambiguous Validation/Checker Rejection**: Use `ask_diagnoser` only when the rejection cause is unclear, the evidence conflicts, or no safe local correction is evident. A Checker rejection alone is not a mandatory diagnosis trigger."""
_LEGACY_TROUBLESHOOTER_CHALLENGE = """ - *Challenge Checker*: If you visually confirm the subgoal's target state is *already achieved*, you **MUST** invoke the `reply_to_checker` tool to state your observation."""
_TROUBLESHOOTER_CHALLENGE = """ - *Challenge Checker*: If current evidence clearly proves the subgoal is already achieved, invoke `reply_to_checker` directly. This branch takes precedence over diagnosis; do not call `ask_diagnoser` first.
- *Clear Local Correction*: If the rejection cause is explicit and a safe correction is evident from the current screen, perform that correction directly. This branch also takes precedence over generic diagnosis triggers. Use `ask_diagnoser` only for ambiguity, conflicting evidence, or repeated failure."""
def apply_operator_prompt_contract(prompt_template: str) -> str:
"""Align prompt-level tool and recovery rules with Operator runtime semantics."""
prompt_template = prompt_template.replace(
_LEGACY_TOOL_CALLING_RULE,
_TOOL_CALLING_RULE,
)
prompt_template = prompt_template.replace(
_LEGACY_TOOL_PROTOCOL,
_TOOL_PROTOCOL,
)
prompt_template = prompt_template.replace(
_LEGACY_CHECKER_REJECTION_TRIGGER,
_AMBIGUOUS_CHECKER_REJECTION_TRIGGER,
)
prompt_template = prompt_template.replace(
_LEGACY_TROUBLESHOOTER_CHALLENGE,
_TROUBLESHOOTER_CHALLENGE,
)
return prompt_template
class PromptBuilder:
def __init__(self):
self.system_parts = []
@@ -83,17 +128,7 @@ class TemplatePromptComponent(PromptComponent):
"Failed to format prompt, template not found in operator prompts config."
)
op_version = resolve_explorer_version(ctx, agent_or_profile_name="operator")
rule_prompt_info = EXPLORE_DESCRIPTIONS.get(op_version, EXPLORE_DESCRIPTIONS["pro"])
rule_prompt = rule_prompt_info["rule_prompt"]
if "{max_iterations}" in rule_prompt:
max_iterations = 8 if op_version == "ultra" else 3
rule_prompt = rule_prompt.format(max_iterations=max_iterations)
prompt_template = prompt_template.replace(
"# OPERATIONAL PRINCIPLES\n",
f"# OPERATIONAL PRINCIPLES\n- **Visual Explorer Rule**: {rule_prompt}\n",
)
prompt_template = apply_operator_prompt_contract(prompt_template)
plan_and_history = kwargs.get("plan_and_history", "No plan or history yet.")
artemis/context.py
+29 -3
View File
@@ -163,6 +163,7 @@ class ArtemisContext(BaseModel):
checker_task: asyncio.Task[Any] | None = None
planner_task: asyncio.Task[Any] | None = None
background_tasks: list[asyncio.Task[Any]] = Field(default_factory=list)
background_task_grace_period_seconds: float = 2.0
package_cache: dict[str, str | None] = Field(default_factory=dict)
background_jobs: dict[str, dict[str, Any]] = Field(default_factory=dict)
@@ -178,9 +179,34 @@ class ArtemisContext(BaseModel):
async def __aexit__(self, exc_type, exc_val, exc_tb):
if self.background_tasks:
logger.info(f"Waiting for {len(self.background_tasks)} background tasks to complete...")
results = await asyncio.gather(*self.background_tasks, return_exceptions=True)
for task, result in zip(self.background_tasks, results):
tasks = list(self.background_tasks)
pending_tasks = [task for task in tasks if not task.done()]
grace_period = (
max(0.0, self.background_task_grace_period_seconds)
if exc_type is None
else 0.0
)
if pending_tasks and grace_period > 0:
logger.info(
f"Draining {len(pending_tasks)} background tasks for up to "
f"{grace_period:.1f}s..."
)
_, pending_tasks = await asyncio.wait(
pending_tasks,
timeout=grace_period,
)
if pending_tasks:
logger.info(
f"Cancelling {len(pending_tasks)} unfinished background tasks; "
"primary automation is already complete."
)
for task in pending_tasks:
task.cancel()
results = await asyncio.gather(*tasks, return_exceptions=True)
for task, result in zip(tasks, results):
if isinstance(result, Exception):
logger.error(
f"Background task {task.get_name()} failed: {result}",
artemis/services/llm.py
+87 -22
View File
@@ -20,6 +20,7 @@ thought stream recording, and role-based dynamic dispatching.
import asyncio
from collections.abc import Awaitable, Callable, Coroutine
from contextvars import ContextVar, Token
from dataclasses import replace
import functools
import logging
@@ -59,6 +60,33 @@ T = TypeVar("T")
# global after this module has already been imported.
_CURRENT_DATA_ENGINE = None
# Provider retry logs do not carry an Artemis trace id. Keep the active model
# request in async-local state so observable SDK retries and the terminal error
# are emitted as one lifecycle, even when several model calls run concurrently.
_ACTIVE_LLM_REQUEST: ContextVar[dict[str, Any] | None] = ContextVar(
"active_llm_request", default=None
)
def _provider_name_from_call(args: tuple[Any, ...]) -> str | None:
wrapper = args[0] if args else None
endpoint = getattr(wrapper, "endpoint", None)
provider = getattr(endpoint, "provider", None)
if provider is None:
return None
return str(getattr(provider, "value", provider))
def _begin_llm_request(args: tuple[Any, ...]) -> Token:
return _ACTIVE_LLM_REQUEST.set(
{
"request_id": str(uuid4()),
"provider": _provider_name_from_call(args),
"started_at": time.time(),
"retries": [],
}
)
def _get_current_data_engine():
if _CURRENT_DATA_ENGINE is not None:
@@ -77,6 +105,11 @@ def _record_llm_retry(
source: str = "artemis",
) -> None:
"""Persist and publish one recoverable LLM retry for UI transparency."""
request = _ACTIVE_LLM_REQUEST.get()
timestamp = time.time()
request_id = request.get("request_id") if request else None
provider = provider or (request.get("provider") if request else None)
engine = _get_current_data_engine()
if not engine or not getattr(engine, "current_session_id", None):
return
@@ -93,12 +126,12 @@ def _record_llm_retry(
"provider": provider,
"source": source,
"recoverable": True,
"request_id": request_id,
"scheduled_at": timestamp,
}
payload = {key: value for key, value in payload.items() if value is not None}
step_id = getattr(engine, "current_step_id", None)
parent_trace_id = CURRENT_TRACE_ID.get()
timestamp = time.time()
try:
trace_id = engine.record_trace(
type="llm_call",
@@ -112,6 +145,15 @@ def _record_llm_retry(
llm_logger.warning("Failed to persist LLM retry trace: %s", trace_error)
trace_id = None
if request is not None:
request["retries"].append(
{
**payload,
"trace_id": str(trace_id) if trace_id else None,
"timestamp": timestamp,
}
)
try:
engine._publish(
"llm_retrying",
@@ -145,6 +187,10 @@ class _ProviderRetryTelemetryHandler(logging.Handler):
def emit(self, record: logging.LogRecord) -> None:
try:
request = _ACTIVE_LLM_REQUEST.get()
if not request or request.get("provider") != ModelProvider.GOOGLE.value:
return
message = record.getMessage()
lowered = message.lower()
if "retrying" not in lowered or not any(
@@ -159,7 +205,7 @@ class _ProviderRetryTelemetryHandler(logging.Handler):
_record_llm_retry(
error,
delay,
provider="google",
provider=request["provider"],
source="provider_sdk",
)
except Exception:
@@ -197,13 +243,29 @@ def _handle_llm_pause_and_resume(last_error: Exception) -> Path:
if current_engine:
step_id = getattr(current_engine, "current_step_id", None)
timestamp = time.time()
request = _ACTIVE_LLM_REQUEST.get()
failure_payload: dict[str, Any] = {"error": err_msg, "pause": True}
if request:
failure_payload.update(
{
"request_id": request["request_id"],
"provider": request.get("provider"),
"waited_seconds": max(0.0, timestamp - request["started_at"]),
# Only SDK-observed retries are included. Other providers
# may retry internally, but their attempt data is opaque.
"retries": list(request["retries"]),
}
)
failure_payload = {
key: value for key, value in failure_payload.items() if value is not None
}
try:
# Persist exhausted LLM retries as a failed call so the same error
# card is available both in the live stream and after a refresh.
current_engine.record_trace(
type="llm_call",
name="llm_pause",
payload={"error": err_msg, "pause": True},
payload=failure_payload,
step_id=step_id,
status="failed",
)
@@ -213,7 +275,7 @@ def _handle_llm_pause_and_resume(last_error: Exception) -> Path:
current_engine._publish(
"task_paused",
{
"error": err_msg,
**failure_payload,
"step_id": str(step_id) if step_id else None,
"timestamp": timestamp,
},
@@ -228,12 +290,15 @@ def robust_retry_async(func: Callable) -> Callable:
@functools.wraps(func)
async def wrapper(*args, **kwargs):
while True:
request_token = _begin_llm_request(args)
max_retries = 3
retry_delay = 0.0
last_error = None
for attempt in range(max_retries):
try:
return await func(*args, **kwargs)
result = await func(*args, **kwargs)
_ACTIVE_LLM_REQUEST.reset(request_token)
return result
except Exception as e:
last_error = e
err_str = str(e).lower()
@@ -256,16 +321,15 @@ def robust_retry_async(func: Callable) -> Callable:
if retry_delay > 0:
llm_logger.warning(f"LLM call failed, retrying in {retry_delay:.2f}s...")
_record_llm_retry(
str(e),
retry_delay,
attempt=attempt + 1,
max_retries=max_retries,
source="artemis_wrapper",
)
await asyncio.sleep(retry_delay)
except BaseException:
_ACTIVE_LLM_REQUEST.reset(request_token)
raise
pause_file = _handle_llm_pause_and_resume(last_error)
try:
pause_file = _handle_llm_pause_and_resume(last_error)
finally:
_ACTIVE_LLM_REQUEST.reset(request_token)
while pause_file.exists():
await asyncio.sleep(1)
@@ -284,6 +348,7 @@ def robust_retry_astream(func: Callable) -> Callable:
@functools.wraps(func)
async def wrapper(*args, **kwargs):
while True:
request_token = _begin_llm_request(args)
max_retries = 3
retry_delay = 0.0
last_error = None
@@ -294,6 +359,7 @@ def robust_retry_astream(func: Callable) -> Callable:
yield first
async for chunk in gen:
yield chunk
_ACTIVE_LLM_REQUEST.reset(request_token)
return
except Exception as e:
last_error = e
@@ -319,16 +385,15 @@ def robust_retry_astream(func: Callable) -> Callable:
f"LLM stream handshake throttled, retrying in {retry_delay:.2f}s..."
)
_record_llm_retry(
str(e),
retry_delay,
attempt=attempt + 1,
max_retries=max_retries,
source="artemis_wrapper",
)
await asyncio.sleep(retry_delay)
except BaseException:
_ACTIVE_LLM_REQUEST.reset(request_token)
raise
pause_file = _handle_llm_pause_and_resume(last_error)
try:
pause_file = _handle_llm_pause_and_resume(last_error)
finally:
_ACTIVE_LLM_REQUEST.reset(request_token)
while pause_file.exists():
await asyncio.sleep(1)