# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Per-turn action execution loop for the Validator. Orchestrates, for each Operator decision: precondition safety-net checks (XML-first with pixel fallback), local execution with retries, screenshot and trace bookkeeping, and FailureAnalyzer-driven local repair. Extracted from ``validator.py``; the public entry point remains ``ValidatorNode.__call__``, which delegates here. All patchable seams (_exec_action, _validate_action_precondition*, _get_initial_screenshot, ...) are dispatched through the node so existing test patches keep working. """ import asyncio import base64 from dataclasses import dataclass from pathlib import Path import time import uuid from uuid import UUID from artemis.agents.validator.failure_analyzer import ( FailureAnalyzer, ValidationErrorCategory, ) from artemis.data_engine.trace import CURRENT_TRACE_ID from artemis.graph.state import State from artemis.utils.logger import get_logger logger = get_logger(__name__) @dataclass class _ActionOutcome: """Result of processing a single action item (pre-check + execution).""" action_item: dict success: bool = False error_msg: str = "" error_category: ValidationErrorCategory = ValidationErrorCategory.GENERAL post_screenshot_b64: str | None = None async def _run_precondition_gate( node, session, action_item: dict, state: State, pre_screenshot_b64: str, original_coords: list ) -> tuple[bool, ValidationErrorCategory, str]: """Runs the pre-execution Safety Net: XML-first with pixel fallback.""" is_explorer_candidate = action_item.get("target_class") == "ExplorerCandidate" is_ocr_element = "[OCR]" in str(action_item.get("target_class") or "") has_index_metadata = bool( ( action_item.get("target_text") or action_item.get("target_bounds") or action_item.get("target_resource_id") ) and not is_explorer_candidate and not is_ocr_element ) if not has_index_metadata: # Fall back to Pixel-based validation for pure coordinate/non-index interactions return await node._validate_action_precondition_pixel( session, action_item, pre_screenshot_b64, original_coords, state=state ) # Index-based validation is fastest and safest when metadata is present ( validation_passed, validation_category, validation_error, ) = await node._validate_action_precondition(session, action_item, state=state) if validation_passed: return validation_passed, validation_category, validation_error if validation_category == ValidationErrorCategory.XML_BYPASSED: logger.info( "Pre-execution XML-based validation bypassed" f" ({validation_error}). Falling back to" " Pixel-based validation..." ) return await node._validate_action_precondition_pixel( session, action_item, pre_screenshot_b64, original_coords, state=state ) logger.info( "Pre-execution XML-based validation failed:" f" {validation_error}. Attempting Pixel-based" " validation fallback..." ) ( pixel_passed, pixel_category, pixel_error, ) = await node._validate_action_precondition_pixel( session, action_item, pre_screenshot_b64, original_coords, state=state ) if pixel_passed and pixel_category == ValidationErrorCategory.NONE: logger.success( "Pixel-based validation fallback PASSED! Overriding XML validation failure." ) return True, ValidationErrorCategory.NONE, "" return validation_passed, validation_category, validation_error async def _attempt_local_execution( node, session, action_item: dict, action_name, attempts_log: list ) -> tuple[bool, str]: """Executes the action locally with a short retry loop.""" max_local_retries = 1 if action_name == "launch_app" else 2 success = False error_msg = "" for attempt in range(max_local_retries): try: exec_success, exec_error = await node._exec_action(session, action_item) except Exception as e: exec_success = False exec_error = str(e) # Skip polling: assume execution success implies action success if exec_success: success = True error_msg = "" attempts_log.append("Success") break else: success = False error_msg = exec_error attempts_log.append(error_msg) if attempt < max_local_retries - 1: await asyncio.sleep(0.5) return success, error_msg async def _capture_live_screenshot(session, context_desc: str) -> str | None: try: return await session.screenshot_b64() except Exception as e: logger.error(f"Failed to capture {context_desc}: {e!r}") return None async def _process_action( node, state: State, session, action_item: dict, action_name, pre_screenshot_b64: str ) -> _ActionOutcome: """Runs precondition checks and execution for one action item.""" action_item = dict(action_item) # Make a copy to avoid mutating original state actions outcome = _ActionOutcome(action_item=action_item) attempts_log = [] # 1. Pre-execution validation (Safety Net) original_coords = list(action_item.get("coordinates") or []) ( validation_passed, validation_category, validation_error, ) = await _run_precondition_gate( node, session, action_item, state, pre_screenshot_b64, original_coords ) if not validation_passed: outcome.success = False outcome.error_msg = f"Pre-execution validation failed: {validation_error}" outcome.error_category = validation_category attempts_log.append(outcome.error_msg) # Capture the live mismatch screenshot to provide context for FailureAnalyzer outcome.post_screenshot_b64 = await _capture_live_screenshot( session, "live screenshot for failure analysis" ) else: # 2. Local execution attempts outcome.success, outcome.error_msg = await _attempt_local_execution( node, session, action_item, action_name, attempts_log ) if not outcome.success and not outcome.post_screenshot_b64: logger.info("Action failed, capturing failure screenshot...") outcome.post_screenshot_b64 = await _capture_live_screenshot( session, "failure screenshot" ) # Enrich the executed action record if len(attempts_log) > 1 or not outcome.success: action_item["attempts"] = list(attempts_log) return outcome def _absorb_post_screenshot(ctx, state: State, post_screenshot_b64: str) -> str | None: """Persists the post-action screenshot and points state at it. Returns the stored image name (None when no data engine is attached). """ decoded_bytes = base64.b64decode(post_screenshot_b64) logger.info(f"Successfully decoded post_screenshot_b64 ({len(decoded_bytes)} bytes)") if not ctx.data_engine: return None post_image_name = ctx.data_engine.get_or_create_image(decoded_bytes) screenshot_path = str(ctx.data_engine.get_image_path(post_image_name)) state.latest_screenshot = screenshot_path logger.info(f"Validator updated state.latest_screenshot to: {screenshot_path}") return post_image_name def _record_action_start(ctx, step_id, action_name, action_item, parent_id, action_trace_id): if ctx.data_engine and step_id: ctx.data_engine.record_trace( type="action", name=action_name, payload={"action": action_item, "status": "running"}, status="running", parent_trace_id=parent_id, step_id=step_id, trace_id=action_trace_id, ) def _record_action_end( ctx, step_id, action_name, outcome: _ActionOutcome, post_image_name, parent_id, action_trace_id, start_time: float, ): duration = time.time() - start_time if ctx.data_engine and step_id: relative_time = ctx.data_engine.get_relative_time(time.time()) ctx.data_engine.record_trace( type="action", name=action_name, payload={ "action": outcome.action_item, "success": outcome.success, "error_msg": outcome.error_msg, "post_screenshot": post_image_name, "timestamp": time.time(), "relative_time": relative_time, }, status="success" if outcome.success else "failed", duration=duration, parent_trace_id=parent_id, step_id=step_id, trace_id=action_trace_id, ) def _reload_post_repair_screenshot( ctx, state: State, last_screenshot_b64: str, last_screenshot_name: str | None ) -> tuple[str, str | None]: """Reloads the latest screenshot updated by FailureAnalyzer during repair.""" try: screenshot_path = getattr(state, "latest_screenshot", None) if screenshot_path: if Path(screenshot_path).exists(): with open(screenshot_path, "rb") as f: last_screenshot_b64 = base64.b64encode(f.read()).decode("utf-8") if ctx.data_engine: last_screenshot_name = ctx.data_engine.get_or_create_image( base64.b64decode(last_screenshot_b64) ) logger.info(f"Validator reloaded post-repair screenshot from: {screenshot_path}") except Exception as e: logger.error(f"Failed to reload post-repair screenshot: {e}") return last_screenshot_b64, last_screenshot_name async def _analyze_failure( node, state: State, outcome: _ActionOutcome, *, decision_screenshot_b64, decision_screenshot_name, post_image_name, executed_actions, unexecuted_actions, ) -> dict: """Triggers the FailureAnalyzer for a failed action and records its analysis.""" logger.warning(f"Action failed: {outcome.action_item}. Triggering failure analysis.") analyzer = FailureAnalyzer(node.ctx) analysis_result = await analyzer.analyze( state, outcome.action_item, outcome.error_msg, pre_screenshot=decision_screenshot_b64, post_screenshot=outcome.post_screenshot_b64, pre_screenshot_name=decision_screenshot_name, post_screenshot_name=post_image_name, executed_actions=executed_actions, unexecuted_actions=unexecuted_actions, error_category=outcome.error_category, ) outcome.action_item["repair"] = analysis_result.get("analysis", "No analysis provided.") return analysis_result async def run_validation_loop(node, state: State) -> dict: """Executes all pending Operator decisions and returns the execution report. ``node`` is the ValidatorNode facade; every overridable seam is invoked through it. """ ctx = node.ctx actions, error_msg = node._parse_decisions(state.structured_decisions) if error_msg: return {} execution: list[dict] = [] failed_action = None actions_to_execute = list(actions) session = await node._get_mcp_session() try: last_screenshot_b64, last_screenshot_name = await node._get_initial_screenshot( session, state ) # Preserve the turn-initial screenshot (what the operator saw when making decisions) decision_screenshot_b64 = last_screenshot_b64 decision_screenshot_name = last_screenshot_name except Exception: return {} step_id = UUID(state.current_step_id) if state.current_step_id else None analysis_result: dict = {} success = False while actions_to_execute: action_item = actions_to_execute.pop(0) action_name = action_item.get("action") logger.info(f"Executing action: {action_name}") action_trace_id = uuid.uuid4() parent_id = CURRENT_TRACE_ID.get() token = CURRENT_TRACE_ID.set(action_trace_id) start_time = time.time() _record_action_start(ctx, step_id, action_name, action_item, parent_id, action_trace_id) try: outcome = await _process_action( node, state, session, action_item, action_name, last_screenshot_b64 ) success = outcome.success execution.append(outcome.action_item) post_image_name = None if outcome.post_screenshot_b64: post_image_name = _absorb_post_screenshot( ctx, state, outcome.post_screenshot_b64 ) last_screenshot_b64 = outcome.post_screenshot_b64 if post_image_name: last_screenshot_name = post_image_name _record_action_end( ctx, step_id, action_name, outcome, post_image_name, parent_id, action_trace_id, start_time, ) finally: CURRENT_TRACE_ID.reset(token) if not success: failed_action = outcome.action_item analysis_result = await _analyze_failure( node, state, outcome, decision_screenshot_b64=decision_screenshot_b64, decision_screenshot_name=decision_screenshot_name, post_image_name=post_image_name, executed_actions=execution[:-1], unexecuted_actions=list(actions_to_execute), ) if analysis_result.get("status") == "fixed": logger.success("Failure repaired locally!") actions_to_execute = [] last_screenshot_b64, last_screenshot_name = _reload_post_repair_screenshot( ctx, state, last_screenshot_b64, last_screenshot_name ) continue else: break # Append any skipped actions to the execution log for skipped in actions_to_execute: skipped_copy = dict(skipped) skipped_copy["attempts"] = ["Skipped"] execution.append(skipped_copy) report = { "execution": execution, "status": ( "success" if success or (failed_action and analysis_result.get("status") == "fixed") else "failed" ), "repair_status": (analysis_result.get("status") if failed_action else None), } # Determine if a distinct post-action screenshot was captured distinct_post_image_name = ( last_screenshot_name if (last_screenshot_name and last_screenshot_name != decision_screenshot_name) else None ) if ctx.data_engine and step_id: ctx.data_engine.update_step_execution_result( step_id, report, post_image_name=distinct_post_image_name ) return {"last_execution_result": report}