chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

refactor(validator): split validator responsibilities into modules

Browse Source
This commit is contained in:
somew1nd committed 2026-09-01 21:34:39 -07:00
1 parent 9cad5b690f
commit b7dd8c6f4d
5 files changed
+1554 -1122
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

artemis/agents/validator/execution_loop.py
+445
View File
@@ -0,0 +1,445 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Per-turn action execution loop for the Validator.
Orchestrates, for each Operator decision: precondition safety-net checks
(XML-first with pixel fallback), local execution with retries, screenshot and
trace bookkeeping, and FailureAnalyzer-driven local repair.
Extracted from ``validator.py``; the public entry point remains
``ValidatorNode.__call__``, which delegates here. All patchable seams
(_exec_action, _validate_action_precondition*, _get_initial_screenshot, ...)
are dispatched through the node so existing test patches keep working.
"""
import asyncio
import base64
from dataclasses import dataclass
from pathlib import Path
import time
import uuid
from uuid import UUID
from artemis.agents.validator.failure_analyzer import (
FailureAnalyzer,
ValidationErrorCategory,
)
from artemis.data_engine.trace import CURRENT_TRACE_ID
from artemis.graph.state import State
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
@dataclass
class _ActionOutcome:
"""Result of processing a single action item (pre-check + execution)."""
action_item: dict
success: bool = False
error_msg: str = ""
error_category: ValidationErrorCategory = ValidationErrorCategory.GENERAL
post_screenshot_b64: str | None = None
async def _run_precondition_gate(
node, session, action_item: dict, state: State, pre_screenshot_b64: str, original_coords: list
) -> tuple[bool, ValidationErrorCategory, str]:
"""Runs the pre-execution Safety Net: XML-first with pixel fallback."""
is_explorer_candidate = action_item.get("target_class") == "ExplorerCandidate"
is_ocr_element = "[OCR]" in str(action_item.get("target_class") or "")
has_index_metadata = bool(
(
action_item.get("target_text")
or action_item.get("target_bounds")
or action_item.get("target_resource_id")
)
and not is_explorer_candidate
and not is_ocr_element
)
if not has_index_metadata:
# Fall back to Pixel-based validation for pure coordinate/non-index interactions
return await node._validate_action_precondition_pixel(
session, action_item, pre_screenshot_b64, original_coords, state=state
)
# Index-based validation is fastest and safest when metadata is present
(
validation_passed,
validation_category,
validation_error,
) = await node._validate_action_precondition(session, action_item, state=state)
if validation_passed:
return validation_passed, validation_category, validation_error
if validation_category == ValidationErrorCategory.XML_BYPASSED:
logger.info(
"Pre-execution XML-based validation bypassed"
f" ({validation_error}). Falling back to"
" Pixel-based validation..."
)
return await node._validate_action_precondition_pixel(
session, action_item, pre_screenshot_b64, original_coords, state=state
)
logger.info(
"Pre-execution XML-based validation failed:"
f" {validation_error}. Attempting Pixel-based"
" validation fallback..."
)
(
pixel_passed,
pixel_category,
pixel_error,
) = await node._validate_action_precondition_pixel(
session, action_item, pre_screenshot_b64, original_coords, state=state
)
if pixel_passed and pixel_category == ValidationErrorCategory.NONE:
logger.success(
"Pixel-based validation fallback PASSED! Overriding XML validation failure."
)
return True, ValidationErrorCategory.NONE, ""
return validation_passed, validation_category, validation_error
async def _attempt_local_execution(
node, session, action_item: dict, action_name, attempts_log: list
) -> tuple[bool, str]:
"""Executes the action locally with a short retry loop."""
max_local_retries = 1 if action_name == "launch_app" else 2
success = False
error_msg = ""
for attempt in range(max_local_retries):
try:
exec_success, exec_error = await node._exec_action(session, action_item)
except Exception as e:
exec_success = False
exec_error = str(e)
# Skip polling: assume execution success implies action success
if exec_success:
success = True
error_msg = ""
attempts_log.append("Success")
break
else:
success = False
error_msg = exec_error
attempts_log.append(error_msg)
if attempt < max_local_retries - 1:
await asyncio.sleep(0.5)
return success, error_msg
async def _capture_live_screenshot(session, context_desc: str) -> str | None:
try:
return await session.screenshot_b64()
except Exception as e:
logger.error(f"Failed to capture {context_desc}: {e!r}")
return None
async def _process_action(
node, state: State, session, action_item: dict, action_name, pre_screenshot_b64: str
) -> _ActionOutcome:
"""Runs precondition checks and execution for one action item."""
action_item = dict(action_item) # Make a copy to avoid mutating original state actions
outcome = _ActionOutcome(action_item=action_item)
attempts_log = []
# 1. Pre-execution validation (Safety Net)
original_coords = list(action_item.get("coordinates") or [])
(
validation_passed,
validation_category,
validation_error,
) = await _run_precondition_gate(
node, session, action_item, state, pre_screenshot_b64, original_coords
)
if not validation_passed:
outcome.success = False
outcome.error_msg = f"Pre-execution validation failed: {validation_error}"
outcome.error_category = validation_category
attempts_log.append(outcome.error_msg)
# Capture the live mismatch screenshot to provide context for FailureAnalyzer
outcome.post_screenshot_b64 = await _capture_live_screenshot(
session, "live screenshot for failure analysis"
)
else:
# 2. Local execution attempts
outcome.success, outcome.error_msg = await _attempt_local_execution(
node, session, action_item, action_name, attempts_log
)
if not outcome.success and not outcome.post_screenshot_b64:
logger.info("Action failed, capturing failure screenshot...")
outcome.post_screenshot_b64 = await _capture_live_screenshot(
session, "failure screenshot"
)
# Enrich the executed action record
if len(attempts_log) > 1 or not outcome.success:
action_item["attempts"] = list(attempts_log)
return outcome
def _absorb_post_screenshot(ctx, state: State, post_screenshot_b64: str) -> str | None:
"""Persists the post-action screenshot and points state at it.
Returns the stored image name (None when no data engine is attached).
"""
decoded_bytes = base64.b64decode(post_screenshot_b64)
logger.info(f"Successfully decoded post_screenshot_b64 ({len(decoded_bytes)} bytes)")
if not ctx.data_engine:
return None
post_image_name = ctx.data_engine.get_or_create_image(decoded_bytes)
screenshot_path = str(ctx.data_engine.get_image_path(post_image_name))
state.latest_screenshot = screenshot_path
logger.info(f"Validator updated state.latest_screenshot to: {screenshot_path}")
return post_image_name
def _record_action_start(ctx, step_id, action_name, action_item, parent_id, action_trace_id):
if ctx.data_engine and step_id:
ctx.data_engine.record_trace(
type="action",
name=action_name,
payload={"action": action_item, "status": "running"},
status="running",
parent_trace_id=parent_id,
step_id=step_id,
trace_id=action_trace_id,
)
def _record_action_end(
ctx,
step_id,
action_name,
outcome: _ActionOutcome,
post_image_name,
parent_id,
action_trace_id,
start_time: float,
):
duration = time.time() - start_time
if ctx.data_engine and step_id:
relative_time = ctx.data_engine.get_relative_time(time.time())
ctx.data_engine.record_trace(
type="action",
name=action_name,
payload={
"action": outcome.action_item,
"success": outcome.success,
"error_msg": outcome.error_msg,
"post_screenshot": post_image_name,
"timestamp": time.time(),
"relative_time": relative_time,
},
status="success" if outcome.success else "failed",
duration=duration,
parent_trace_id=parent_id,
step_id=step_id,
trace_id=action_trace_id,
)
def _reload_post_repair_screenshot(
ctx, state: State, last_screenshot_b64: str, last_screenshot_name: str | None
) -> tuple[str, str | None]:
"""Reloads the latest screenshot updated by FailureAnalyzer during repair."""
try:
screenshot_path = getattr(state, "latest_screenshot", None)
if screenshot_path:
if Path(screenshot_path).exists():
with open(screenshot_path, "rb") as f:
last_screenshot_b64 = base64.b64encode(f.read()).decode("utf-8")
if ctx.data_engine:
last_screenshot_name = ctx.data_engine.get_or_create_image(
base64.b64decode(last_screenshot_b64)
)
logger.info(f"Validator reloaded post-repair screenshot from: {screenshot_path}")
except Exception as e:
logger.error(f"Failed to reload post-repair screenshot: {e}")
return last_screenshot_b64, last_screenshot_name
async def _analyze_failure(
node,
state: State,
outcome: _ActionOutcome,
*,
decision_screenshot_b64,
decision_screenshot_name,
post_image_name,
executed_actions,
unexecuted_actions,
) -> dict:
"""Triggers the FailureAnalyzer for a failed action and records its analysis."""
logger.warning(f"Action failed: {outcome.action_item}. Triggering failure analysis.")
analyzer = FailureAnalyzer(node.ctx)
analysis_result = await analyzer.analyze(
state,
outcome.action_item,
outcome.error_msg,
pre_screenshot=decision_screenshot_b64,
post_screenshot=outcome.post_screenshot_b64,
pre_screenshot_name=decision_screenshot_name,
post_screenshot_name=post_image_name,
executed_actions=executed_actions,
unexecuted_actions=unexecuted_actions,
error_category=outcome.error_category,
)
outcome.action_item["repair"] = analysis_result.get("analysis", "No analysis provided.")
return analysis_result
async def run_validation_loop(node, state: State) -> dict:
"""Executes all pending Operator decisions and returns the execution report.
``node`` is the ValidatorNode facade; every overridable seam is invoked
through it.
"""
ctx = node.ctx
actions, error_msg = node._parse_decisions(state.structured_decisions)
if error_msg:
return {}
execution: list[dict] = []
failed_action = None
actions_to_execute = list(actions)
session = await node._get_mcp_session()
try:
last_screenshot_b64, last_screenshot_name = await node._get_initial_screenshot(
session, state
)
# Preserve the turn-initial screenshot (what the operator saw when making decisions)
decision_screenshot_b64 = last_screenshot_b64
decision_screenshot_name = last_screenshot_name
except Exception:
return {}
step_id = UUID(state.current_step_id) if state.current_step_id else None
analysis_result: dict = {}
success = False
while actions_to_execute:
action_item = actions_to_execute.pop(0)
action_name = action_item.get("action")
logger.info(f"Executing action: {action_name}")
action_trace_id = uuid.uuid4()
parent_id = CURRENT_TRACE_ID.get()
token = CURRENT_TRACE_ID.set(action_trace_id)
start_time = time.time()
_record_action_start(ctx, step_id, action_name, action_item, parent_id, action_trace_id)
try:
outcome = await _process_action(
node, state, session, action_item, action_name, last_screenshot_b64
)
success = outcome.success
execution.append(outcome.action_item)
post_image_name = None
if outcome.post_screenshot_b64:
post_image_name = _absorb_post_screenshot(
ctx, state, outcome.post_screenshot_b64
)
last_screenshot_b64 = outcome.post_screenshot_b64
if post_image_name:
last_screenshot_name = post_image_name
_record_action_end(
ctx,
step_id,
action_name,
outcome,
post_image_name,
parent_id,
action_trace_id,
start_time,
)
finally:
CURRENT_TRACE_ID.reset(token)
if not success:
failed_action = outcome.action_item
analysis_result = await _analyze_failure(
node,
state,
outcome,
decision_screenshot_b64=decision_screenshot_b64,
decision_screenshot_name=decision_screenshot_name,
post_image_name=post_image_name,
executed_actions=execution[:-1],
unexecuted_actions=list(actions_to_execute),
)
if analysis_result.get("status") == "fixed":
logger.success("Failure repaired locally!")
actions_to_execute = []
last_screenshot_b64, last_screenshot_name = _reload_post_repair_screenshot(
ctx, state, last_screenshot_b64, last_screenshot_name
)
continue
else:
break
# Append any skipped actions to the execution log
for skipped in actions_to_execute:
skipped_copy = dict(skipped)
skipped_copy["attempts"] = ["Skipped"]
execution.append(skipped_copy)
report = {
"execution": execution,
"status": (
"success"
if success or (failed_action and analysis_result.get("status") == "fixed")
else "failed"
),
"repair_status": (analysis_result.get("status") if failed_action else None),
}
# Determine if a distinct post-action screenshot was captured
distinct_post_image_name = (
last_screenshot_name
if (last_screenshot_name and last_screenshot_name != decision_screenshot_name)
else None
)
if ctx.data_engine and step_id:
ctx.data_engine.update_step_execution_result(
step_id, report, post_image_name=distinct_post_image_name
)
return {"last_execution_result": report}