# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Pixel-level Safety Net: VLM comparison of the target crop before execution. Validates that the visual target button/component at the action coordinates is still present, visible, and interactive on the live screen compared to the original (decision-time) screenshot. Extracted from ``validator.py``; the public entry point remains the ``ValidatorNode`` method, which delegates here. The ``get_llm_fn`` dependency is injected by the facade so that tests patching ``artemis.agents.validator.validator.get_llm`` keep working. """ import asyncio import base64 import hashlib import math from io import BytesIO from pathlib import Path from langchain_core.messages import HumanMessage, SystemMessage from PIL import Image from artemis.agents.validator.categories import ValidationErrorCategory from artemis.context import ArtemisContext from artemis.graph.state import State from artemis.services.llm import acomplete_structured from artemis.utils import visualization from artemis.utils.logger import get_logger logger = get_logger(__name__) # Only interactive, element-focused actions are subject to pixel validation. PIXEL_VALIDATED_ACTIONS = ( "tap", "long_press_on", "focus_and_input_text", "input_text", "click_coordinate", "click", "long_press", ) _MAX_ATTEMPTS = 3 _RETRY_DELAY_SECONDS = 0.3 def _annotate_target(image_bytes, coords, coordinate_size, **kwargs) -> bytes: """Map device pixels into each encoded screenshot before marking/cropping it.""" if coordinate_size is not None: width, height = coordinate_size with Image.open(BytesIO(image_bytes)) as image: image_width, image_height = image.size if abs((image_width / image_height) / (width / height) - 1) > 0.01: raise ValueError("Screenshot orientation/geometry changed; observe the target again.") x, y = coords if not (0 <= x < width and 0 <= y < height): raise ValueError("Target lies outside the observed device screen.") coords = [round(x * image_width / width), round(y * image_height / height)] return visualization.crop_and_annotate_target(image_bytes, coords, **kwargs) def _strict_target(action_item: dict) -> bool: return any( action_item.get(key) for key in ( "target_text", "target_resource_id", "target_class", "target_bounds", "target_description", ) ) or action_item.get("action") in {"input_text", "focus_and_input_text"} def _unverified(action_item: dict, message: str) -> tuple[bool, ValidationErrorCategory, str]: if _strict_target(action_item): return False, ValidationErrorCategory.PIXEL_VERIFICATION_UNAVAILABLE, message return True, ValidationErrorCategory.PIXEL_BYPASSED, "" def _init_llm_and_prompt(ctx: ArtemisContext, get_llm_fn): """Resolves the VLM and loads the prompt template (once per validation).""" try: llm = get_llm_fn(ctx, name="validator_pixel_safety_net") except Exception: llm = get_llm_fn(ctx, name="validator") prompt_path = Path(__file__).parent.joinpath("pixel_safety_net.md") prompt = prompt_path.read_text(encoding="utf-8") return llm, prompt def _describe_target(action_item: dict) -> str: """Describe the structured action target for the pixel judge. Three kinds, keyed on provenance: an index-addressed target carries observed element metadata; a coordinate target carries only the Operator's own statement of what it aimed at (``target_description``); a legacy record may carry neither. """ label = action_item.get("target_text") resource_id = action_item.get("target_resource_id") class_name = action_item.get("target_class") bounds = action_item.get("target_bounds") description = action_item.get("target_description") lines = ["[Target]", f"Action: {action_item.get('action')}"] if label or resource_id or class_name or bounds: lines.append("Kind: specific UI control") if label: lines.append(f"Label: {label}") if resource_id: lines.append(f"Resource ID: {resource_id}") if class_name: lines.append(f"Class: {class_name}") if not label and not resource_id: lines.append("Label: (unlabelled; identify it by its shape in Image 1)") if bounds: lines.append(f"Bounds at decision time: {bounds}") elif description: lines.append("Kind: described target") lines.append(f"Operator's description: {description}") else: lines.append( "Kind: coordinates only (no target metadata recorded;" " inspect Image 1 for a control at the red dot)" ) return "\n".join(lines) def _build_messages( prompt: str, orig_crop_bytes: bytes, live_crop_bytes: bytes, action_item: dict, state: State | None, ) -> list: """Formulates the multi-modal verification message pair.""" orig_b64 = base64.b64encode(orig_crop_bytes).decode("utf-8") live_b64 = base64.b64encode(live_crop_bytes).decode("utf-8") action_name = action_item.get("action") user_content = [ {"type": "text", "text": _describe_target(action_item)}, {"type": "text", "text": "[Image 1 (Reference)]"}, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{orig_b64}"}, }, {"type": "text", "text": "[Image 2 (Current State)]"}, { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{live_b64}"}, }, ] if state: thoughts = [] native_thought = getattr(state, "operator_native_thinking", None) if native_thought and native_thought.strip(): thoughts.append(native_thought.strip()) raw_thought = getattr(state, "operator_raw_thinking", None) if raw_thought and raw_thought.strip(): thoughts.append(raw_thought.strip()) operator_thought = "\n\n".join(thoughts) if operator_thought.strip(): context_text = ( "[Planned Action & Original Thinking]\nAction:" f" {action_name}\nOriginal" f" Thinking:\n{operator_thought.strip()}" ) user_content.append({"type": "text", "text": f"\n{context_text.strip()}"}) return [ SystemMessage(content=prompt), HumanMessage(content=user_content), ] async def _run_attempt( session, llm, prompt: str, orig_crop_bytes: bytes, current_coords, action_item: dict, state: State | None, attempt: int, *, coordinate_size: tuple[int, int] | None = None, ) -> tuple[bool, ValidationErrorCategory, str] | None: """Runs one pixel validation attempt. Returns a final (passed, category, reason) verdict, or None when the attempt yielded no verdict (never happens today: every branch is final). Raises on infrastructure errors so the caller's retry loop can decide. """ # A. Take fresh live screenshot (captures state changes / dynamic loading settling) try: live_screenshot_b64 = await session.screenshot_b64() except Exception as exc: raise RuntimeError(f"Failed to acquire live screenshot: {exc}") from exc if not live_screenshot_b64: raise RuntimeError("Live screenshot was empty; target identity could not be checked.") # B. Decode and crop the live screen target live_bytes = base64.b64decode(live_screenshot_b64) live_crop_bytes = _annotate_target( live_bytes, current_coords, coordinate_size, crop_size=None, dot_radius=15 ) # C. Formulate multi-modal verification message messages = _build_messages(prompt, orig_crop_bytes, live_crop_bytes, action_item, state) # D. Invoke Universal VLM. Parsing (fences, repair) and one # corrective re-ask on unparseable JSON are handled by the # structured-output layer; a StructuredOutputError lands in # the attempt loop's except like any other attempt failure. res_json = await acomplete_structured(llm, messages) if not isinstance(res_json, dict): raise ValueError( "Pixel validation response parsed to" f" {type(res_json).__name__}, expected a JSON object." ) reasoning = res_json.get("reasoning", "") is_present = res_json.get("is_present") confidence = res_json.get("confidence") if ( not isinstance(is_present, bool) or isinstance(confidence, bool) or not isinstance(confidence, (int, float)) or not math.isfinite(confidence) or not 0 <= confidence <= 1 ): raise ValueError( "Pixel validation must include boolean is_present and confidence in [0, 1]." ) if _strict_target(action_item) and ( res_json.get("target_matches") is not True or res_json.get("target_kind") not in {"control", "surface"} ): is_present = False reasoning = reasoning or "The intended target was not positively identified." if ( action_item.get("action") in {"input_text", "focus_and_input_text"} or any( action_item.get(key) for key in ("target_text", "target_resource_id", "target_class", "target_bounds") ) ) and res_json.get("target_kind") != "control": is_present = False reasoning = "A recorded UI control cannot be validated as a blank surface." logger.info( f"Pixel validation attempt {attempt}/{_MAX_ATTEMPTS} result:" f" is_present={is_present}, confidence={confidence:.2f}, reasoning={reasoning}" ) # E. Branching based on outcome if is_present and (confidence >= 0.7 or not _strict_target(action_item)): logger.info(f"Pixel validation SUCCESS on attempt {attempt}/{_MAX_ATTEMPTS}.") return True, ValidationErrorCategory.NONE, "" # Target not present: fail or bypass based on confidence without retry! if confidence >= 0.7 or _strict_target(action_item): reason_msg = ( f"Pixel-level validation failed: {reasoning}" if reasoning else ( "Pixel-level validation failed: The target UI" " element at the exact coordinates has disappeared," " changed, or is blocked." ) ) return ( False, ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED, reason_msg, ) logger.warning( f"Pixel validation failed but confidence ({confidence:.2f}) is below 0.7. Bypassing check." ) return True, ValidationErrorCategory.PIXEL_BYPASSED, "" async def validate_action_precondition_pixel( ctx: ArtemisContext, session, action_item: dict, pre_screenshot_b64: str, original_coords: list[int] | None = None, state: State | None = None, *, get_llm_fn, ) -> tuple[bool, ValidationErrorCategory, str]: """Pixel-level Safety Net entry point (see module docstring).""" action_name = action_item.get("action") # 1. White-list filtering: Only validate interactive, element-focused actions if action_name not in PIXEL_VALIDATED_ACTIONS: return True, ValidationErrorCategory.PIXEL_BYPASSED, "" current_coords = action_item.get("coordinates") orig_coords = original_coords or current_coords if not orig_coords or len(orig_coords) != 2: logger.info("Pixel safety net skipped: no valid coordinates.") return _unverified(action_item, "No valid coordinates to verify the intended target.") if not current_coords or len(current_coords) != 2: current_coords = orig_coords device = getattr(ctx, "device", None) dimensions = (getattr(device, "device_width", None), getattr(device, "device_height", None)) coordinate_size = ( dimensions if all(type(value) is int and value > 0 for value in dimensions) else None ) try: neighborhood = _annotate_target( base64.b64decode(pre_screenshot_b64), orig_coords, coordinate_size, crop_size=256, dot_radius=0, ) except (ValueError, TypeError, OSError): neighborhood = pre_screenshot_b64.encode() # A changing clock elsewhere on screen is not fresh evidence for a # previously rejected blank point. Grounded control metadata still allows # a new validation; this does not restrict ordinary repeated actions. key = hashlib.sha256(repr(orig_coords).encode() + neighborhood).hexdigest() rejected = getattr(ctx, "rejected_pixel_targets", None) if not isinstance(rejected, dict): rejected = {} ctx.rejected_pixel_targets = rejected grounded = bool(action_item.get("target_bounds") and action_item.get("target_class")) if key in rejected and not grounded: return ( False, ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED, ( "This point was already rejected for this observation. A renamed description " "is not new evidence; locate the intended control again or obtain fresh target evidence." ), ) logger.info( "Pre-execution pixel validation: checking target position" f" {orig_coords} in original image vs {current_coords} on live" " screen..." ) # 2. Decode and crop the static reference (original) image ONCE outside the loop to save CPU try: orig_bytes = base64.b64decode(pre_screenshot_b64) orig_crop_bytes = _annotate_target( orig_bytes, orig_coords, coordinate_size, crop_size=None, dot_radius=15 ) except Exception as e: logger.error( f"Failed to prepare original crop for pixel safety net: {e}. Target unverified." ) return _unverified( action_item, "Reference screenshot unavailable; target identity is unverified." ) # 3. Get LLM Configuration once outside the loop try: llm, prompt = _init_llm_and_prompt(ctx, get_llm_fn) except Exception as e: logger.error( f"Failed to initialize LLM/prompt for pixel safety net: {e}. Target unverified." ) return _unverified( action_item, "Pixel validator unavailable; target identity is unverified." ) # 4. Start high-precision execution validation retry loop last_error = None for attempt in range(1, _MAX_ATTEMPTS + 1): logger.info(f"Pre-execution pixel validation: Attempt {attempt}/{_MAX_ATTEMPTS}...") try: verdict = await _run_attempt( session, llm, prompt, orig_crop_bytes, current_coords, action_item, state, attempt, coordinate_size=coordinate_size, ) if verdict is not None: if not verdict[0]: rejected[key] = True return verdict except Exception as attempt_err: logger.warning( f"Error during pixel validation attempt {attempt}/{_MAX_ATTEMPTS}: {attempt_err}" ) last_error = attempt_err if attempt < _MAX_ATTEMPTS: await asyncio.sleep(_RETRY_DELAY_SECONDS) continue logger.error( "Pixel validation repeatedly failed with errors. Last" f" error: {last_error}. Target identity remains unverified." ) return _unverified( action_item, f"Pixel verification unavailable after {_MAX_ATTEMPTS} attempts: {last_error}. " "The action was not executed; restore observation before locating the target again.", ) return _unverified(action_item, "Pixel validation did not establish target identity.")