somew1nd
9a5aaf99b7
- Showcase UI Reorganization: * Move web UI dashboard from root directory into apps/showcase_ui/ for cleaner project structure * Add 'artemis ui' CLI command (artemis/interfaces/cli/commands/ui.py) and launcher scripts for the interactive showcase console - Standalone MCP Server & Notifiers (mcp_server/): * Implement dedicated MCP server package with mobile automation tools (device_state, inspect_trace, task_manager, task_runner) * Add multi-channel notifiers (AgentAPI, Desktop notifications, Webhooks, File output, and Composite routing) * Introduce background task runner and trace store for autonomous mobile execution monitoring - Cross-Platform Runtime & Toolchain Support: * Add artemis/platform/ with OS-specific implementations for Linux, macOS (Darwin), and Windows * Add artemis/runtime/supervisor.py for resilient process and agent runtime supervision * Add artemis/toolchain/ with dependency resolution and SDK descriptors * Add cross-platform launcher scripts (start.sh, start.bat) and installer scripts (scripts/install_deps.*) - Core Enhancements & Diagnostics: * Enhance coordinate normalization, OCR API fallback, and perception pipeline * Add credential validation (credentials_validator.py) and system diagnostics * Update bilingual documentation (README.md, README_CN.md) and architecture diagrams - Test Suite Expansion: * Add integration tests for cloud gateway, remote ADB, and Artemis login * Add unit tests covering core diagnostics, platform detection, toolchain, credentials validator, optional OCR, and MCP tools
1617 lines
69 KiB
Python
1617 lines
69 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import asyncio
|
|
import base64
|
|
import difflib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import psutil
|
|
import re
|
|
import signal
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import uuid
|
|
from uuid import UUID
|
|
|
|
from adbutils import AdbClient
|
|
from mcp import ClientSession, StdioServerParameters
|
|
from mcp.client.stdio import stdio_client
|
|
|
|
from langchain_core.messages import HumanMessage, SystemMessage
|
|
from artemis.agents.validator.failure_analyzer import (
|
|
FailureAnalyzer,
|
|
ValidationErrorCategory,
|
|
)
|
|
from artemis.clients.ui_automator_client import UIAutomatorClient
|
|
from artemis.constants import (
|
|
VALIDATOR_POLL_INTERVAL,
|
|
VALIDATOR_POLL_TIMEOUT,
|
|
VALIDATOR_UI_HIERARCHY_TIMEOUT,
|
|
)
|
|
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
|
|
from artemis.controllers.unified_controller import UnifiedMobileController
|
|
from artemis.data_engine.trace import CURRENT_TRACE_ID, trace
|
|
from artemis.graph.state import State
|
|
from artemis.platform import platform
|
|
from artemis.services.llm import get_llm
|
|
from artemis.tools.mobile.launch_app import find_package
|
|
from artemis.utils import app_launch_utils, image_diff, visualization
|
|
from artemis.utils.decorators import wrap_with_callbacks
|
|
from artemis.utils.logger import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
# Define actions that are permitted to succeed without a detected UI change
|
|
ALLOW_NO_UI_CHANGE = {
|
|
"wait_for_delay",
|
|
"focus_and_clear_text",
|
|
"launch_app",
|
|
"stop_app",
|
|
"press_key",
|
|
"back",
|
|
"erase_one_char",
|
|
"swipe",
|
|
}
|
|
|
|
|
|
class ValidatorNode:
|
|
"""Node responsible for executing actions on the device and verifying the results.
|
|
|
|
Despite its name, this node handles:
|
|
1. Parsing structured decisions from the Operator.
|
|
2. Executing these actions via an MCP session (calling tools like tap,
|
|
swipe, etc.).
|
|
3. Verifying if the action resulted in a UI change.
|
|
4. Triggering failure analysis and local repair if actions fail.
|
|
|
|
In a more strict architecture, execution and validation might be split into
|
|
separate nodes.
|
|
"""
|
|
|
|
def __init__(self, ctx: ArtemisContext):
|
|
self.ctx = ctx
|
|
|
|
def _kill_mcp_server_instantly(self):
|
|
"""Instantly terminate MCP ADB server child processes using SIGKILL."""
|
|
|
|
try:
|
|
current_process = psutil.Process()
|
|
children = current_process.children(recursive=True)
|
|
for child in children:
|
|
try:
|
|
cmdline = child.cmdline()
|
|
if any("adb_server.py" in part for part in cmdline):
|
|
logger.warning(f"Instantly killing MCP Server child process: {child.pid}")
|
|
os.kill(child.pid, signal.SIGKILL)
|
|
except psutil.NoSuchProcess:
|
|
pass
|
|
except Exception as e:
|
|
logger.error(f"Failed to instantly kill MCP server child processes: {e}")
|
|
|
|
async def _get_mcp_session(self):
|
|
if getattr(self.ctx, "mcp_session", None) is not None:
|
|
return self.ctx.mcp_session
|
|
|
|
root_dir = Path(__file__).parent.parent.parent
|
|
venv_python = root_dir / ".venv" / "bin" / "python3"
|
|
python_exe = str(venv_python) if venv_python.exists() else sys.executable
|
|
|
|
server_params = StdioServerParameters(
|
|
command=python_exe,
|
|
args=[str(root_dir / "mcp" / "adb_server.py")],
|
|
env=os.environ.copy(),
|
|
)
|
|
|
|
logger.info("Starting persistent MCP server session...")
|
|
self.ctx.mcp_client_ctx = stdio_client(server_params)
|
|
try:
|
|
read, write = await self.ctx.mcp_client_ctx.__aenter__()
|
|
self.ctx.mcp_session = ClientSession(read, write)
|
|
await self.ctx.mcp_session.__aenter__()
|
|
# Handshake timeout protection
|
|
await asyncio.wait_for(self.ctx.mcp_session.initialize(), timeout=15.0)
|
|
return self.ctx.mcp_session
|
|
except Exception as e:
|
|
err_stack = traceback.format_exc()
|
|
logger.critical(
|
|
"CRITICAL: Failed to initialize MCP server session:"
|
|
f" {e}\n{err_stack}. Cleaning up child processes."
|
|
)
|
|
self._kill_mcp_server_instantly()
|
|
self.ctx.mcp_session = None
|
|
self.ctx.mcp_client_ctx = None
|
|
|
|
logger.warning(
|
|
"Falling back to robust local in-process execution bypassing MCP protocol."
|
|
)
|
|
|
|
try:
|
|
adb = AdbClient(host="localhost", port=5037)
|
|
devices = adb.device_list()
|
|
if devices:
|
|
device_id = devices[0].serial
|
|
ui_client = UIAutomatorClient(device_id=device_id)
|
|
try:
|
|
ui_data = ui_client.get_screen_data()
|
|
w, h = ui_data.width, ui_data.height
|
|
except Exception:
|
|
w, h = 1080, 2400
|
|
local_ctx = ArtemisContext(
|
|
trace_id="local-fallback",
|
|
device=DeviceContext(
|
|
host_platform=platform.os_type.name,
|
|
mobile_platform=DevicePlatform.ANDROID,
|
|
device_id=device_id,
|
|
device_width=w,
|
|
device_height=h,
|
|
),
|
|
adb_client=adb,
|
|
ui_adb_client=ui_client,
|
|
)
|
|
self._local_controller = UnifiedMobileController(local_ctx)
|
|
except Exception as le:
|
|
logger.error(f"Failed to setup local fallback controller: {le}")
|
|
|
|
return None
|
|
|
|
def _parse_decisions(self, structured_decisions: str) -> tuple[list[dict] | None, str | None]:
|
|
if not structured_decisions:
|
|
return None, "No structured decisions found, nothing to execute."
|
|
try:
|
|
return json.loads(structured_decisions), None
|
|
except json.JSONDecodeError as e:
|
|
return None, f"Failed to parse structured decisions: {e}"
|
|
|
|
async def _get_initial_screenshot(self, session, state: State) -> tuple[str, str | None]:
|
|
screenshot_path = getattr(state, "latest_screenshot", None)
|
|
if not screenshot_path:
|
|
logger.error("No screenshot path found in state.latest_screenshot")
|
|
raise ValueError("No screenshot path found in state.latest_screenshot")
|
|
|
|
if not Path(screenshot_path).exists():
|
|
logger.error(f"Screenshot file does not exist: {screenshot_path}")
|
|
raise FileNotFoundError(f"Screenshot file does not exist: {screenshot_path}")
|
|
|
|
try:
|
|
with open(screenshot_path, "rb") as f:
|
|
screenshot_b64 = base64.b64encode(f.read()).decode("utf-8")
|
|
except Exception as e:
|
|
logger.error(f"Failed to read screenshot from {screenshot_path}: {e}")
|
|
raise e
|
|
|
|
screenshot_name = Path(screenshot_path).stem
|
|
return screenshot_b64, screenshot_name
|
|
|
|
async def _poll_for_ui_change(
|
|
self, session, pre_screenshot_b64, action_item
|
|
) -> tuple[bool, str, str | None]:
|
|
|
|
start_time = time.time()
|
|
timeout = VALIDATOR_POLL_TIMEOUT
|
|
ui_changed = False
|
|
post_screenshot_b64 = None
|
|
error_msg = ""
|
|
|
|
while time.time() - start_time < timeout:
|
|
if session is not None:
|
|
result = await session.call_tool("take_screenshot", {})
|
|
success_img, content_img = self._parse_mcp_result(result)
|
|
elif getattr(self, "_local_controller", None):
|
|
try:
|
|
content_img = await self._local_controller.take_screenshot()
|
|
success_img = True
|
|
except Exception as e:
|
|
success_img, content_img = False, str(e)
|
|
else:
|
|
success_img, content_img = (
|
|
False,
|
|
"No session or local controller",
|
|
)
|
|
if success_img:
|
|
post_screenshot_b64 = content_img
|
|
img_before_bytes = base64.b64decode(pre_screenshot_b64)
|
|
img_after_bytes = base64.b64decode(post_screenshot_b64)
|
|
|
|
ui_changed = image_diff.check_ui_change(
|
|
img_before_bytes, img_after_bytes, action_item
|
|
)
|
|
if ui_changed:
|
|
logger.info("UI change detected during polling.")
|
|
break
|
|
else:
|
|
logger.error(f"Failed to take screenshot via MCP: {content_img}")
|
|
|
|
await asyncio.sleep(VALIDATOR_POLL_INTERVAL)
|
|
|
|
if not ui_changed:
|
|
error_msg = "Screen did not change"
|
|
|
|
return ui_changed, error_msg, post_screenshot_b64
|
|
|
|
async def _execute_validation_loop(self, state: State):
|
|
actions, error_msg = self._parse_decisions(state.structured_decisions)
|
|
if error_msg:
|
|
return await state.asanitize_update(
|
|
ctx=self.ctx,
|
|
update={},
|
|
agent="validator",
|
|
)
|
|
|
|
execution = []
|
|
failed_action = None
|
|
failure_reason = None
|
|
|
|
actions_to_execute = list(actions)
|
|
|
|
session = await self._get_mcp_session()
|
|
|
|
try:
|
|
last_screenshot_b64, last_screenshot_name = await self._get_initial_screenshot(
|
|
session, state
|
|
)
|
|
# Preserve the turn-initial screenshot (what the operator saw when making decisions)
|
|
decision_screenshot_b64 = last_screenshot_b64
|
|
decision_screenshot_name = last_screenshot_name
|
|
except Exception:
|
|
return await state.asanitize_update(
|
|
ctx=self.ctx,
|
|
update={},
|
|
agent="validator",
|
|
)
|
|
|
|
step_id = None
|
|
if state.current_step_id:
|
|
step_id = UUID(state.current_step_id)
|
|
|
|
analysis_result = {}
|
|
|
|
while actions_to_execute:
|
|
action_item = actions_to_execute.pop(0)
|
|
action_name = action_item.get("action")
|
|
|
|
logger.info(f"Executing action: {action_name}")
|
|
|
|
action_trace_id = uuid.uuid4()
|
|
parent_id = CURRENT_TRACE_ID.get()
|
|
token = CURRENT_TRACE_ID.set(action_trace_id)
|
|
start_time = time.time()
|
|
|
|
if self.ctx.data_engine and step_id:
|
|
self.ctx.data_engine.record_trace(
|
|
type="action",
|
|
name=action_name,
|
|
payload={"action": action_item, "status": "running"},
|
|
status="running",
|
|
parent_trace_id=parent_id,
|
|
step_id=step_id,
|
|
trace_id=action_trace_id,
|
|
)
|
|
|
|
try:
|
|
pre_screenshot_b64 = last_screenshot_b64
|
|
success = False
|
|
error_msg = ""
|
|
error_category = ValidationErrorCategory.GENERAL
|
|
post_screenshot_b64 = None
|
|
post_image_name = None
|
|
|
|
action_item = dict(
|
|
action_item
|
|
) # Make a copy to avoid mutating original state actions
|
|
attempts_log = []
|
|
|
|
# 1. Pre-execution validation (Safety Net)
|
|
original_coords = list(action_item.get("coordinates") or [])
|
|
|
|
is_explorer_candidate = action_item.get("target_class") == "ExplorerCandidate"
|
|
is_ocr_element = "[OCR]" in str(action_item.get("target_class") or "")
|
|
has_index_metadata = bool(
|
|
(
|
|
action_item.get("target_text")
|
|
or action_item.get("target_bounds")
|
|
or action_item.get("target_resource_id")
|
|
)
|
|
and not is_explorer_candidate
|
|
and not is_ocr_element
|
|
)
|
|
|
|
if has_index_metadata:
|
|
# Index-based validation is fastest and safest when metadata is present
|
|
(
|
|
validation_passed,
|
|
validation_category,
|
|
validation_error,
|
|
) = await self._validate_action_precondition(session, action_item, state=state)
|
|
|
|
if not validation_passed:
|
|
if validation_category == ValidationErrorCategory.XML_BYPASSED:
|
|
logger.info(
|
|
"Pre-execution XML-based validation bypassed"
|
|
f" ({validation_error}). Falling back to"
|
|
" Pixel-based validation..."
|
|
)
|
|
(
|
|
validation_passed,
|
|
validation_category,
|
|
validation_error,
|
|
) = await self._validate_action_precondition_pixel(
|
|
session,
|
|
action_item,
|
|
pre_screenshot_b64,
|
|
original_coords,
|
|
state=state,
|
|
)
|
|
else:
|
|
logger.info(
|
|
"Pre-execution XML-based validation failed:"
|
|
f" {validation_error}. Attempting Pixel-based"
|
|
" validation fallback..."
|
|
)
|
|
(
|
|
pixel_passed,
|
|
pixel_category,
|
|
pixel_error,
|
|
) = await self._validate_action_precondition_pixel(
|
|
session,
|
|
action_item,
|
|
pre_screenshot_b64,
|
|
original_coords,
|
|
state=state,
|
|
)
|
|
if pixel_passed and pixel_category == ValidationErrorCategory.NONE:
|
|
logger.success(
|
|
"Pixel-based validation fallback PASSED!"
|
|
" Overriding XML validation failure."
|
|
)
|
|
validation_passed = True
|
|
validation_category = ValidationErrorCategory.NONE
|
|
validation_error = ""
|
|
else:
|
|
# Fall back to Pixel-based validation for pure coordinate/non-index interactions
|
|
(
|
|
validation_passed,
|
|
validation_category,
|
|
validation_error,
|
|
) = await self._validate_action_precondition_pixel(
|
|
session,
|
|
action_item,
|
|
pre_screenshot_b64,
|
|
original_coords,
|
|
state=state,
|
|
)
|
|
|
|
if not validation_passed:
|
|
success = False
|
|
error_msg = f"Pre-execution validation failed: {validation_error}"
|
|
error_category = validation_category
|
|
attempts_log.append(error_msg)
|
|
|
|
# Capture the live mismatch screenshot to provide context for FailureAnalyzer
|
|
try:
|
|
if session is not None:
|
|
result = await session.call_tool("take_screenshot", {})
|
|
success_img, post_screenshot_b64 = self._parse_mcp_result(result)
|
|
elif getattr(self, "_local_controller", None):
|
|
post_screenshot_b64 = await self._local_controller.take_screenshot()
|
|
success_img = True
|
|
else:
|
|
success_img, post_screenshot_b64 = False, None
|
|
|
|
if success_img and post_screenshot_b64:
|
|
if self.ctx.data_engine:
|
|
post_image_name = self.ctx.data_engine.get_or_create_image(
|
|
base64.b64decode(post_screenshot_b64)
|
|
)
|
|
screenshot_path = str(
|
|
self.ctx.data_engine.get_image_path(post_image_name)
|
|
)
|
|
state.latest_screenshot = screenshot_path
|
|
except Exception as e:
|
|
logger.error(f"Failed to capture live screenshot for failure analysis: {e}")
|
|
post_screenshot_b64 = None
|
|
post_image_name = None
|
|
else:
|
|
# 2. Local execution attempts
|
|
max_local_retries = 1 if action_name == "launch_app" else 2
|
|
for attempt in range(max_local_retries):
|
|
try:
|
|
exec_success, exec_error = await self._exec_action(session, action_item)
|
|
except Exception as e:
|
|
exec_success = False
|
|
exec_error = str(e)
|
|
|
|
# Skip polling: assume execution success implies action success
|
|
if exec_success:
|
|
success = True
|
|
error_msg = ""
|
|
attempts_log.append("Success")
|
|
break
|
|
else:
|
|
success = False
|
|
error_msg = exec_error
|
|
attempts_log.append(error_msg)
|
|
|
|
if attempt < max_local_retries - 1:
|
|
await asyncio.sleep(0.5)
|
|
|
|
if not success and not post_screenshot_b64:
|
|
logger.info("Action failed, capturing failure screenshot...")
|
|
try:
|
|
if session is not None:
|
|
result = await session.call_tool("take_screenshot", {})
|
|
success_img, post_screenshot_b64 = self._parse_mcp_result(result)
|
|
elif getattr(self, "_local_controller", None):
|
|
post_screenshot_b64 = await self._local_controller.take_screenshot()
|
|
success_img = True
|
|
else:
|
|
success_img, post_screenshot_b64 = False, None
|
|
|
|
if success_img and post_screenshot_b64:
|
|
if self.ctx.data_engine:
|
|
post_image_name = self.ctx.data_engine.get_or_create_image(
|
|
base64.b64decode(post_screenshot_b64)
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"Failed to capture failure screenshot: {e}")
|
|
|
|
# Enrich and record execution
|
|
if len(attempts_log) > 1 or not success:
|
|
action_item["attempts"] = list(attempts_log)
|
|
execution.append(action_item)
|
|
|
|
if post_screenshot_b64:
|
|
last_screenshot_b64 = post_screenshot_b64
|
|
if self.ctx.data_engine:
|
|
post_image_name = self.ctx.data_engine.get_or_create_image(
|
|
base64.b64decode(post_screenshot_b64)
|
|
)
|
|
last_screenshot_name = post_image_name
|
|
|
|
screenshot_path = str(self.ctx.data_engine.get_image_path(post_image_name))
|
|
state.latest_screenshot = screenshot_path
|
|
logger.info(
|
|
f"Validator updated state.latest_screenshot to: {screenshot_path}"
|
|
)
|
|
|
|
# Update action trace with final results
|
|
duration = time.time() - start_time
|
|
if self.ctx.data_engine and step_id:
|
|
relative_time = self.ctx.data_engine.get_relative_time(time.time())
|
|
self.ctx.data_engine.record_trace(
|
|
type="action",
|
|
name=action_name,
|
|
payload={
|
|
"action": action_item,
|
|
"success": success,
|
|
"error_msg": error_msg,
|
|
"post_screenshot": post_image_name,
|
|
"timestamp": time.time(),
|
|
"relative_time": relative_time,
|
|
},
|
|
status="success" if success else "failed",
|
|
duration=duration,
|
|
parent_trace_id=parent_id,
|
|
step_id=step_id,
|
|
trace_id=action_trace_id,
|
|
)
|
|
finally:
|
|
CURRENT_TRACE_ID.reset(token)
|
|
|
|
if not success:
|
|
failed_action = action_item
|
|
failure_reason = error_msg
|
|
|
|
logger.warning(f"Action failed: {failed_action}. Triggering failure analysis.")
|
|
|
|
analyzer = FailureAnalyzer(self.ctx)
|
|
analysis_result = await analyzer.analyze(
|
|
state,
|
|
failed_action,
|
|
failure_reason,
|
|
pre_screenshot=decision_screenshot_b64,
|
|
post_screenshot=post_screenshot_b64,
|
|
pre_screenshot_name=decision_screenshot_name,
|
|
post_screenshot_name=post_image_name,
|
|
executed_actions=execution[:-1],
|
|
unexecuted_actions=list(actions_to_execute),
|
|
error_category=error_category,
|
|
)
|
|
action_item["repair"] = analysis_result.get("analysis", "No analysis provided.")
|
|
|
|
status = analysis_result.get("status")
|
|
if status == "fixed":
|
|
logger.success("Failure repaired locally!")
|
|
actions_to_execute = []
|
|
|
|
# Reload the latest screenshot updated by FailureAnalyzer during repair
|
|
try:
|
|
screenshot_path = getattr(state, "latest_screenshot", None)
|
|
if screenshot_path:
|
|
if Path(screenshot_path).exists():
|
|
with open(screenshot_path, "rb") as f:
|
|
last_screenshot_b64 = base64.b64encode(f.read()).decode("utf-8")
|
|
if self.ctx.data_engine:
|
|
last_screenshot_name = self.ctx.data_engine.get_or_create_image(
|
|
base64.b64decode(last_screenshot_b64)
|
|
)
|
|
logger.info(
|
|
"Validator reloaded post-repair screenshot"
|
|
f" from: {screenshot_path}"
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"Failed to reload post-repair screenshot: {e}")
|
|
|
|
continue
|
|
else:
|
|
break
|
|
|
|
# Append any skipped actions to the execution log
|
|
for skipped in actions_to_execute:
|
|
skipped_copy = dict(skipped)
|
|
skipped_copy["attempts"] = ["Skipped"]
|
|
execution.append(skipped_copy)
|
|
|
|
report = {
|
|
"execution": execution,
|
|
"status": (
|
|
"success"
|
|
if success or (failed_action and analysis_result.get("status") == "fixed")
|
|
else "failed"
|
|
),
|
|
"repair_status": (analysis_result.get("status") if failed_action else None),
|
|
}
|
|
|
|
# Determine if a distinct post-action screenshot was captured
|
|
distinct_post_image_name = (
|
|
last_screenshot_name
|
|
if (last_screenshot_name and last_screenshot_name != decision_screenshot_name)
|
|
else None
|
|
)
|
|
|
|
if self.ctx.data_engine and step_id:
|
|
self.ctx.data_engine.update_step_execution_result(
|
|
step_id, report, post_image_name=distinct_post_image_name
|
|
)
|
|
|
|
return await state.asanitize_update(
|
|
ctx=self.ctx,
|
|
update={"last_execution_result": report},
|
|
agent="validator",
|
|
)
|
|
|
|
@wrap_with_callbacks(
|
|
before=lambda: logger.info("Starting Validator Agent..."),
|
|
on_success=lambda _: logger.success("Validator Agent"),
|
|
on_failure=lambda _: logger.error("Validator Agent"),
|
|
)
|
|
@trace(type="agent", name="validator")
|
|
async def __call__(self, state: State):
|
|
|
|
step_id = None
|
|
if state.current_step_id:
|
|
step_id = UUID(state.current_step_id)
|
|
|
|
original_step_id = getattr(self.ctx.data_engine, "current_step_id", None)
|
|
if self.ctx.data_engine and step_id:
|
|
self.ctx.data_engine.current_step_id = step_id
|
|
|
|
try:
|
|
return await self._execute_validation_loop(state)
|
|
except asyncio.CancelledError:
|
|
logger.warning(
|
|
"Validator task cancelled (Stop requested). Instantly killing MCP server processes."
|
|
)
|
|
self._kill_mcp_server_instantly()
|
|
raise
|
|
except Exception as e:
|
|
err_stack = traceback.format_exc()
|
|
logger.critical(f"CRITICAL ERROR in Validator execution loop: {e}\n{err_stack}")
|
|
if self.ctx.data_engine and step_id:
|
|
try:
|
|
self.ctx.data_engine.update_step_execution_result(
|
|
step_id,
|
|
{
|
|
"status": "error",
|
|
"error_msg": str(e),
|
|
"traceback": err_stack,
|
|
},
|
|
)
|
|
except Exception:
|
|
pass
|
|
self._kill_mcp_server_instantly()
|
|
raise e
|
|
finally:
|
|
if self.ctx.data_engine:
|
|
self.ctx.data_engine.current_step_id = original_step_id
|
|
|
|
async def _exec_action(
|
|
self, session: ClientSession | None, action_item: dict
|
|
) -> tuple[bool, str]:
|
|
action_name = action_item.get("action")
|
|
coordinates = action_item.get("coordinates")
|
|
|
|
# Intercept non-wait_for_text actions for local fallback
|
|
if session is None and getattr(self, "_local_controller", None) is not None:
|
|
ctrl = self._local_controller
|
|
logger.info(f"Executing in-process action fallback: {action_name}")
|
|
try:
|
|
if action_name == "tap":
|
|
if coordinates and len(coordinates) == 2:
|
|
times = action_item.get("times", 1)
|
|
delay_ms = action_item.get("delay_ms", 100)
|
|
res = await ctrl.tap_at(
|
|
coordinates[0],
|
|
coordinates[1],
|
|
times=times,
|
|
delay_ms=delay_ms,
|
|
)
|
|
return (True, "Success") if not res.error else (False, res.error)
|
|
return False, "Invalid coords"
|
|
elif action_name == "long_press_on":
|
|
d = action_item.get("duration", 1000)
|
|
if coordinates and len(coordinates) == 2:
|
|
res = await ctrl.tap_at(
|
|
coordinates[0],
|
|
coordinates[1],
|
|
long_press=True,
|
|
long_press_duration=d,
|
|
)
|
|
return (True, "Success") if not res.error else (False, res.error)
|
|
return False, "Invalid coords"
|
|
elif action_name == "swipe":
|
|
d = action_item.get("duration", 400)
|
|
if coordinates and len(coordinates) == 4:
|
|
err = await ctrl.swipe_coords(
|
|
coordinates[0],
|
|
coordinates[1],
|
|
coordinates[2],
|
|
coordinates[3],
|
|
duration=d,
|
|
)
|
|
return (True, "Success") if not err else (False, str(err))
|
|
return False, "Invalid coords"
|
|
elif action_name == "focus_and_input_text":
|
|
t = action_item.get("text")
|
|
clear_before = action_item.get("clear_before_input", False)
|
|
if coordinates and len(coordinates) == 2:
|
|
await ctrl.tap_at(coordinates[0], coordinates[1])
|
|
|
|
await asyncio.sleep(0.5)
|
|
if clear_before:
|
|
await ctrl.erase_text()
|
|
else:
|
|
await ctrl.press_key("123") # Move cursor to end for clean append
|
|
success = await ctrl.type_text(t, clear_existing=False)
|
|
return (True, "Success") if success else (False, "Failed typing")
|
|
return False, "Invalid coords"
|
|
elif action_name == "focus_and_clear_text":
|
|
if coordinates and len(coordinates) == 2:
|
|
await ctrl.tap_at(coordinates[0], coordinates[1])
|
|
|
|
await asyncio.sleep(0.5)
|
|
success = await ctrl.erase_text()
|
|
return (True, "Success") if success else (False, "Failed erase")
|
|
return False, "Invalid coords"
|
|
elif action_name == "erase_one_char":
|
|
success = await ctrl.erase_text(nb_chars=1)
|
|
return (True, "Success") if success else (False, "Failed erase")
|
|
elif action_name == "press_key":
|
|
keycode = action_item.get("keycode")
|
|
success = await ctrl.press_key(keycode)
|
|
return (True, "Success") if success else (False, f"Failed press {keycode}")
|
|
elif action_name == "back":
|
|
success = await ctrl.go_back()
|
|
return (True, "Success") if success else (False, "Failed back")
|
|
elif action_name == "launch_app":
|
|
app_name = action_item.get("app_name")
|
|
package_name = await find_package(self.ctx, app_name, use_fallback=False)
|
|
if package_name:
|
|
success, err_msg = await app_launch_utils.launch_app_with_retries(
|
|
ctrl.ctx, package_name
|
|
)
|
|
return (True, "Success") if success else (False, err_msg)
|
|
return False, "Package not found"
|
|
elif action_name == "stop_app":
|
|
app_name = action_item.get("app_name")
|
|
package_name = await find_package(self.ctx, app_name, use_fallback=False)
|
|
if package_name:
|
|
success = await ctrl.terminate_app(package_name)
|
|
return (True, "Success") if success else (False, "Failed terminate")
|
|
return False, "Package not found"
|
|
elif action_name == "open_link":
|
|
url = action_item.get("url")
|
|
success = await ctrl.open_url(url)
|
|
return (True, "Success") if success else (False, "Failed open url")
|
|
elif action_name == "wait_for_delay":
|
|
time_in_ms = action_item.get("time_in_ms", 0)
|
|
|
|
await asyncio.sleep(time_in_ms / 1000.0)
|
|
return True, ""
|
|
else:
|
|
return (
|
|
False,
|
|
f"Unsupported action for local fallback: {action_name}",
|
|
)
|
|
except Exception as e:
|
|
return False, f"In-process fallback failed: {e}"
|
|
|
|
# Normal MCP Execution Path
|
|
try:
|
|
if action_name == "tap":
|
|
coordinates = action_item.get("coordinates")
|
|
times = action_item.get("times") or action_item.get("click_times") or 1
|
|
delay_ms = action_item.get("delay_ms") or action_item.get("delay") or 100
|
|
result = await session.call_tool(
|
|
"tap",
|
|
{
|
|
"coordinates": coordinates,
|
|
"times": times,
|
|
"delay_ms": delay_ms,
|
|
},
|
|
)
|
|
return self._parse_mcp_result(result)
|
|
|
|
elif action_name == "long_press_on":
|
|
coordinates = action_item.get("coordinates")
|
|
duration = action_item.get("duration", 1000)
|
|
result = await session.call_tool(
|
|
"long_press_on",
|
|
{"coordinates": coordinates, "duration": duration},
|
|
)
|
|
return self._parse_mcp_result(result)
|
|
|
|
elif action_name == "swipe":
|
|
coordinates = action_item.get("coordinates")
|
|
duration = action_item.get("duration", 400)
|
|
result = await session.call_tool(
|
|
"swipe", {"coordinates": coordinates, "duration": duration}
|
|
)
|
|
return self._parse_mcp_result(result)
|
|
|
|
elif action_name == "focus_and_input_text":
|
|
coordinates = action_item.get("coordinates")
|
|
text = action_item.get("text", "")
|
|
clear_before_input = action_item.get("clear_before_input", False)
|
|
result = await session.call_tool(
|
|
"focus_and_input_text",
|
|
{
|
|
"coordinates": coordinates,
|
|
"text": text,
|
|
"clear_before_input": clear_before_input,
|
|
},
|
|
)
|
|
return self._parse_mcp_result(result)
|
|
|
|
elif action_name == "focus_and_clear_text":
|
|
coordinates = action_item.get("coordinates")
|
|
result = await session.call_tool(
|
|
"focus_and_clear_text", {"coordinates": coordinates}
|
|
)
|
|
return self._parse_mcp_result(result)
|
|
|
|
elif action_name == "erase_one_char":
|
|
result = await session.call_tool("erase_one_char", {})
|
|
return self._parse_mcp_result(result)
|
|
|
|
elif action_name == "press_key":
|
|
keycode = action_item.get("keycode", "")
|
|
result = await session.call_tool("press_key", {"keycode": keycode})
|
|
return self._parse_mcp_result(result)
|
|
|
|
elif action_name == "back":
|
|
result = await session.call_tool("back", {})
|
|
return self._parse_mcp_result(result)
|
|
|
|
elif action_name == "launch_app":
|
|
app_name = action_item.get("app_name")
|
|
package_name = await find_package(self.ctx, app_name, use_fallback=False)
|
|
if not package_name:
|
|
return False, f"Failed to find package for app: {app_name}"
|
|
|
|
success, err_msg = await app_launch_utils.launch_app_with_retries(
|
|
self.ctx, package_name
|
|
)
|
|
return success, "Success" if success else err_msg
|
|
|
|
elif action_name == "stop_app":
|
|
app_name = action_item.get("app_name")
|
|
package_name = await find_package(self.ctx, app_name, use_fallback=False)
|
|
if not package_name:
|
|
return False, f"Failed to find package for app: {app_name}"
|
|
result = await session.call_tool("stop_app", {"package_name": package_name})
|
|
return self._parse_mcp_result(result)
|
|
|
|
elif action_name == "open_link":
|
|
url = action_item.get("url")
|
|
result = await session.call_tool("open_link", {"url": url})
|
|
return self._parse_mcp_result(result)
|
|
|
|
elif action_name == "wait_for_delay":
|
|
time_in_ms = action_item.get("time_in_ms", 0)
|
|
logger.info(f"Validator performing wait_for_delay for {time_in_ms}ms...")
|
|
|
|
await asyncio.sleep(time_in_ms / 1000.0)
|
|
return True, ""
|
|
|
|
else:
|
|
return False, f"Unsupported action: {action_name}"
|
|
except Exception as e:
|
|
return False, f"MCP call failed: {e}"
|
|
|
|
@trace(type="tool", name="safety_net_validation")
|
|
async def _validate_action_precondition(
|
|
self, session, action_item: dict, state: State | None = None
|
|
) -> tuple[bool, ValidationErrorCategory, str]:
|
|
"""Safety Net: Validates if the target element for interactive actions
|
|
|
|
is still present on the live screen and is substantially consistent,
|
|
with a short retry loop to accommodate screen transitions and loading
|
|
delays.
|
|
"""
|
|
max_pre_attempts = 3
|
|
pre_retry_delay = 0.4
|
|
|
|
passed = False
|
|
category = ValidationErrorCategory.NONE
|
|
reason = ""
|
|
|
|
for pre_attempt in range(1, max_pre_attempts + 1):
|
|
passed, category, reason = await self._validate_action_precondition_single(
|
|
session, action_item, state
|
|
)
|
|
if passed:
|
|
return True, category, reason
|
|
|
|
if category == ValidationErrorCategory.XML_BYPASSED:
|
|
# Do not retry on XML bypass/timeout; return immediately
|
|
# so it falls back to Pixel-based VLM validation
|
|
return False, category, reason
|
|
|
|
if pre_attempt < max_pre_attempts:
|
|
logger.info(
|
|
"Pre-execution validation failed on attempt"
|
|
f" {pre_attempt}/{max_pre_attempts}: {reason}. Retrying in"
|
|
f" {pre_retry_delay}s..."
|
|
)
|
|
await asyncio.sleep(pre_retry_delay)
|
|
else:
|
|
break
|
|
|
|
return passed, category, reason
|
|
|
|
async def _validate_action_precondition_single(
|
|
self, session, action_item: dict, state: State | None = None
|
|
) -> tuple[bool, ValidationErrorCategory, str]:
|
|
"""Safety Net: Validates if the target element for interactive actions
|
|
|
|
is still present on the live screen and is substantially consistent.
|
|
|
|
Returns (True, ValidationErrorCategory.NONE, "") if validation passes or
|
|
is skipped.
|
|
Returns (False, category, error_msg) if validation fails.
|
|
"""
|
|
action_name = action_item.get("action")
|
|
|
|
# 1. White-list filtering: Only validate interactive, element-focused actions
|
|
if action_name not in [
|
|
"tap",
|
|
"long_press_on",
|
|
"focus_and_input_text",
|
|
"focus_and_clear_text",
|
|
]:
|
|
return True, ValidationErrorCategory.NONE, ""
|
|
|
|
target_text = action_item.get("target_text")
|
|
target_bounds = action_item.get("target_bounds") # [l, t, r, b]
|
|
target_resource_id = action_item.get("target_resource_id")
|
|
|
|
width = 1080
|
|
height = 2400
|
|
operator_raw_data = getattr(state, "operator_raw_data", {}) or {}
|
|
w_raw = operator_raw_data.get("width")
|
|
h_raw = operator_raw_data.get("height")
|
|
if isinstance(w_raw, int) and isinstance(h_raw, int):
|
|
width = w_raw
|
|
height = h_raw
|
|
else:
|
|
device = getattr(self.ctx, "device", None)
|
|
if device:
|
|
width = getattr(device, "device_width", 1080)
|
|
height = getattr(device, "device_height", 2400)
|
|
|
|
# Calculate diagonal-based scale factor for resolution-independent thresholds
|
|
|
|
ref_diagonal = math.sqrt(1080**2 + 2400**2) # ~2631.8
|
|
current_diagonal = math.sqrt(width**2 + height**2)
|
|
scale_factor = current_diagonal / ref_diagonal
|
|
|
|
# 2. Skip validation if there's no target metadata to match against
|
|
if not target_text and not target_bounds and not target_resource_id:
|
|
return True, ValidationErrorCategory.NONE, ""
|
|
|
|
logger.info(
|
|
f"Pre-execution validation: checking target '{target_text}'"
|
|
f" ({target_resource_id}) at {target_bounds}..."
|
|
)
|
|
|
|
# 3. Pull live XML tree with strict timeout to avoid hanging the execution flow
|
|
|
|
try:
|
|
elements = None
|
|
if session is not None:
|
|
try:
|
|
result = await asyncio.wait_for(
|
|
session.call_tool("get_ui_hierarchy", {}),
|
|
timeout=VALIDATOR_UI_HIERARCHY_TIMEOUT,
|
|
)
|
|
if hasattr(result, "content") and result.content:
|
|
text_content = (
|
|
result.content[0].text
|
|
if hasattr(result.content[0], "text")
|
|
else str(result.content[0])
|
|
)
|
|
elements = json.loads(text_content)
|
|
except Exception as e:
|
|
logger.warning(
|
|
f"Failed to get live XML via MCP: {e}. Falling back to"
|
|
" Pixel-based validation."
|
|
)
|
|
return (
|
|
False,
|
|
ValidationErrorCategory.XML_BYPASSED,
|
|
f"XML hierarchy fetch timed out or errored: {e}",
|
|
)
|
|
elif getattr(self, "_local_controller", None):
|
|
try:
|
|
elements = await asyncio.wait_for(
|
|
self._local_controller.get_ui_elements(),
|
|
timeout=VALIDATOR_UI_HIERARCHY_TIMEOUT,
|
|
)
|
|
except Exception as e:
|
|
logger.warning(
|
|
f"Failed to get live XML via local controller: {e}."
|
|
" Falling back to Pixel-based validation."
|
|
)
|
|
return (
|
|
False,
|
|
ValidationErrorCategory.XML_BYPASSED,
|
|
f"XML hierarchy fetch timed out or errored: {e}",
|
|
)
|
|
|
|
if not elements or not isinstance(elements, list):
|
|
logger.warning(
|
|
"Live hierarchy empty or invalid. Falling back to Pixel-based validation."
|
|
)
|
|
return (
|
|
False,
|
|
ValidationErrorCategory.XML_BYPASSED,
|
|
"Live hierarchy empty or invalid.",
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(
|
|
f"Unexpected error during live XML fetch: {e}. Falling back to"
|
|
" Pixel-based validation."
|
|
)
|
|
return (
|
|
False,
|
|
ValidationErrorCategory.XML_BYPASSED,
|
|
f"Unexpected error during live XML fetch: {e}",
|
|
)
|
|
|
|
# 4. Matching Heuristics & Scoring
|
|
|
|
# Normalization Helper
|
|
def normalize_text(s: str | None) -> str:
|
|
if not s:
|
|
return ""
|
|
s = s.lower()
|
|
s = re.sub(r"\(\d+\+?\)", "", s)
|
|
s = re.sub(r"\[\d+\+?\]", "", s)
|
|
s = re.sub(r"[^\w\s\d]", "", s)
|
|
return s.strip()
|
|
|
|
# Recursive text aggregator for candidate container nodes
|
|
def aggregate_text(elem: dict) -> str:
|
|
text = (
|
|
elem.get("text") or elem.get("content-desc") or elem.get("accessibilityText") or ""
|
|
)
|
|
if not isinstance(text, str):
|
|
text = str(text)
|
|
children = elem.get("children") or []
|
|
if children:
|
|
for child in children:
|
|
if isinstance(child, dict):
|
|
text += " " + aggregate_text(child)
|
|
return text
|
|
|
|
norm_target_text = normalize_text(target_text)
|
|
|
|
candidates = []
|
|
for elem in elements:
|
|
if not isinstance(elem, dict):
|
|
continue
|
|
|
|
elem_bounds_str = elem.get("bounds")
|
|
elem_bounds = visualization.parse_bounds(elem_bounds_str)
|
|
if not elem_bounds:
|
|
continue
|
|
|
|
elem_raw_text = aggregate_text(elem)
|
|
elem_norm_text = normalize_text(elem_raw_text)
|
|
elem_res_id = elem.get("resource-id") or elem.get("resourceId") or ""
|
|
|
|
elem_cx, elem_cy = visualization.get_center_coordinates(*elem_bounds)
|
|
|
|
original_coords = action_item.get("coordinates") or [0, 0]
|
|
dist = math.sqrt(
|
|
(elem_cx - original_coords[0]) ** 2 + (elem_cy - original_coords[1]) ** 2
|
|
)
|
|
|
|
id_match = target_resource_id and elem_res_id and target_resource_id == elem_res_id
|
|
|
|
text_score = 0.0
|
|
if norm_target_text and elem_norm_text:
|
|
if norm_target_text == elem_norm_text:
|
|
text_score = 1.0
|
|
elif norm_target_text in elem_norm_text or elem_norm_text in norm_target_text:
|
|
text_score = 0.8
|
|
else:
|
|
text_score = difflib.SequenceMatcher(
|
|
None, norm_target_text, elem_norm_text
|
|
).ratio()
|
|
|
|
# Calculate bounds match (IoU)
|
|
bounds_score = 0.0
|
|
size_mismatch = False
|
|
if target_bounds and elem_bounds and len(target_bounds) == 4 and len(elem_bounds) == 4:
|
|
l1, t1, r1, b1 = target_bounds
|
|
l2, t2, r2, b2 = elem_bounds
|
|
|
|
# Check size ratio mismatch to prevent matching small widget to giant container
|
|
w1, h1 = r1 - l1, b1 - t1
|
|
w2, h2 = r2 - l2, b2 - t2
|
|
if w1 > 0 and h1 > 0 and w2 > 0 and h2 > 0:
|
|
size_ratio_w = max(w1, w2) / min(w1, w2)
|
|
size_ratio_h = max(h1, h2) / min(h1, h2)
|
|
if size_ratio_w > 2.5 or size_ratio_h > 2.5:
|
|
size_mismatch = True
|
|
|
|
li = max(l1, l2)
|
|
ti = max(t1, t2)
|
|
ri = min(r1, r2)
|
|
bi = min(b1, b2)
|
|
if ri > li and bi > ti:
|
|
inter_area = (ri - li) * (bi - ti)
|
|
union_area = (r1 - l1) * (b1 - t1) + (r2 - l2) * (b2 - t2) - inter_area
|
|
if union_area > 0:
|
|
bounds_score = inter_area / union_area
|
|
|
|
# Calculate Coordinate Containment
|
|
contains_coord = False
|
|
if (
|
|
original_coords
|
|
and len(original_coords) == 2
|
|
and elem_bounds
|
|
and len(elem_bounds) == 4
|
|
):
|
|
cx, cy = original_coords
|
|
l2, t2, r2, b2 = elem_bounds
|
|
contains_coord = l2 <= cx <= r2 and t2 <= cy <= b2
|
|
|
|
# Apply size mismatch penalty
|
|
if size_mismatch and not id_match:
|
|
text_score = 0.0
|
|
bounds_score = 0.0
|
|
contains_coord = False
|
|
|
|
# Define signal weights
|
|
W_id = 0.5
|
|
W_text = 0.4
|
|
W_bounds = 0.3
|
|
W_coord = 0.3
|
|
|
|
signals = []
|
|
weights = []
|
|
identity_signals = []
|
|
identity_weights = []
|
|
|
|
# 1. Resource ID Signal
|
|
if target_resource_id:
|
|
weights.append(W_id)
|
|
identity_weights.append(W_id)
|
|
if id_match:
|
|
signals.append(W_id * 1.0)
|
|
identity_signals.append(W_id * 1.0)
|
|
elif not elem_res_id:
|
|
signals.append(W_id * 0.3) # Soft penalty for missing ID
|
|
identity_signals.append(W_id * 0.3)
|
|
else:
|
|
signals.append(W_id * -0.5) # Hard penalty for mismatched ID
|
|
identity_signals.append(W_id * -0.5)
|
|
|
|
# 2. Text Signal
|
|
if target_text:
|
|
weights.append(W_text)
|
|
identity_weights.append(W_text)
|
|
if norm_target_text and elem_norm_text:
|
|
signals.append(W_text * text_score)
|
|
identity_signals.append(W_text * text_score)
|
|
elif not elem_norm_text:
|
|
signals.append(W_text * 0.0) # Penalty for missing expected text
|
|
identity_signals.append(W_text * 0.0)
|
|
|
|
# 3. Bounds Signal
|
|
if target_bounds:
|
|
weights.append(W_bounds)
|
|
signals.append(W_bounds * bounds_score)
|
|
|
|
# 4. Coordinate Containment Signal
|
|
if original_coords:
|
|
weights.append(W_coord)
|
|
signals.append(W_coord * (1.0 if contains_coord else 0.0))
|
|
|
|
if weights:
|
|
score = sum(signals) / sum(weights)
|
|
else:
|
|
score = 1.0
|
|
|
|
if identity_weights:
|
|
identity_score = sum(identity_signals) / sum(identity_weights)
|
|
else:
|
|
identity_score = 1.0
|
|
|
|
decay = max(0.5, 1.0 - (dist / 800.0))
|
|
score *= decay
|
|
|
|
candidates.append(
|
|
{
|
|
"element": elem,
|
|
"center": [elem_cx, elem_cy],
|
|
"bounds": elem_bounds,
|
|
"text": elem_raw_text,
|
|
"resource_id": elem_res_id,
|
|
"score": score,
|
|
"identity_score": identity_score,
|
|
"distance": dist,
|
|
"text_score": text_score,
|
|
"id_match": id_match,
|
|
}
|
|
)
|
|
|
|
if not candidates:
|
|
logger.warning(
|
|
"No candidates with valid bounds found. Falling back to Pixel-based validation."
|
|
)
|
|
return (
|
|
False,
|
|
ValidationErrorCategory.XML_BYPASSED,
|
|
"No candidates with valid bounds found.",
|
|
)
|
|
|
|
candidates.sort(key=lambda x: x["score"], reverse=True)
|
|
best = candidates[0]
|
|
|
|
passed = False
|
|
category = ValidationErrorCategory.NONE
|
|
reason = ""
|
|
|
|
threshold = 0.55 if best["distance"] <= 150 * scale_factor else 0.75
|
|
|
|
if best["score"] >= threshold:
|
|
passed = True
|
|
if best["distance"] <= 200 * scale_factor:
|
|
new_coords = best["center"]
|
|
old_coords = action_item.get("coordinates")
|
|
if old_coords != new_coords:
|
|
logger.success(
|
|
"Pre-execution validation SUCCESS"
|
|
f" (score={best['score']:.2f}). Target element"
|
|
f" '{target_text}' shifted by {best['distance']:.1f}px."
|
|
f" Self-healing: correcting coordinates {old_coords} ->"
|
|
f" {new_coords}."
|
|
)
|
|
action_item["coordinates"] = new_coords
|
|
else:
|
|
logger.info(
|
|
"Pre-execution validation SUCCESS"
|
|
f" (score={best['score']:.2f}). Target element matched but"
|
|
f" shifted by {best['distance']:.1f}px"
|
|
f" (>{200 * scale_factor:.1f}px). Bypassing coordinate"
|
|
" self-healing to remain conservative."
|
|
)
|
|
else:
|
|
original_coords = action_item.get("coordinates")
|
|
|
|
# 1. Shifted Case: we must be conservative to avoid confusing disappearance as shift.
|
|
# Shift misclassification as disappeared is acceptable, but disappeared as shift
|
|
# causes errors.
|
|
is_valid_shift = False
|
|
|
|
# Enforce strict shift criteria (preferring disappeared over shifted)
|
|
max_allowed_distance = 100.0 * scale_factor
|
|
if best.get("id_match", False):
|
|
# Allow larger distance threshold only for strong resource ID matches
|
|
max_allowed_distance = 300.0 * scale_factor
|
|
|
|
if best["distance"] <= max_allowed_distance:
|
|
if best["identity_score"] >= 0.85:
|
|
# High confidence identity match (e.g. exact text or exact ID)
|
|
is_valid_shift = True
|
|
elif best["identity_score"] >= 0.5 and best.get("id_match", False):
|
|
# Moderate confidence with exact resource ID match
|
|
is_valid_shift = True
|
|
|
|
if is_valid_shift:
|
|
category = ValidationErrorCategory.TARGET_SHIFTED
|
|
reason = (
|
|
f"Target element '{target_text}' ({target_resource_id})"
|
|
f" expected at {original_coords} (bounds: {target_bounds})"
|
|
f" has shifted.\n- New location: {best['center']} (bounds:"
|
|
f" {best['bounds']})"
|
|
)
|
|
else:
|
|
# 2. Find the most specific element containing expected coordinates
|
|
occupant = None
|
|
if original_coords and len(original_coords) == 2:
|
|
orig_cx, orig_cy = original_coords
|
|
for elem in elements:
|
|
if not isinstance(elem, dict):
|
|
continue
|
|
elem_bounds_str = elem.get("bounds")
|
|
elem_bounds = visualization.parse_bounds(elem_bounds_str)
|
|
if elem_bounds:
|
|
left, top, right, bottom = elem_bounds
|
|
if left <= orig_cx <= right and top <= orig_cy <= bottom:
|
|
elem_area = (right - left) * (bottom - top)
|
|
if not occupant or elem_area < occupant[2]:
|
|
occupant = (elem, elem_bounds, elem_area)
|
|
|
|
def is_interactive(e: dict) -> bool:
|
|
clickable = e.get("clickable")
|
|
long_clickable = e.get("long-clickable") or e.get("longClickable")
|
|
focusable = e.get("focusable")
|
|
enabled = e.get("enabled")
|
|
|
|
def to_bool(v) -> bool:
|
|
if v is None:
|
|
return False
|
|
if isinstance(v, bool):
|
|
return v
|
|
return str(v).lower() == "true"
|
|
|
|
return (
|
|
to_bool(clickable)
|
|
or to_bool(long_clickable)
|
|
or to_bool(focusable)
|
|
or to_bool(enabled)
|
|
)
|
|
|
|
is_occupied = False
|
|
occupant_desc = "anonymous element"
|
|
occupant_bounds = None
|
|
|
|
if occupant:
|
|
elem, elem_bounds, _ = occupant
|
|
occupant_text = (
|
|
elem.get("text")
|
|
or elem.get("content-desc")
|
|
or elem.get("accessibilityText")
|
|
or ""
|
|
)
|
|
occupant_res_id = elem.get("resource-id") or elem.get("resourceId") or ""
|
|
occupant_bounds = elem_bounds
|
|
|
|
# Check if the occupant has content
|
|
has_content = bool(occupant_text or occupant_res_id)
|
|
|
|
# Check if this occupant or any container covering this point is interactive
|
|
coords_interactive = False
|
|
if original_coords and len(original_coords) == 2:
|
|
orig_cx, orig_cy = original_coords
|
|
for e in elements:
|
|
if not isinstance(e, dict):
|
|
continue
|
|
eb_str = e.get("bounds")
|
|
eb = visualization.parse_bounds(eb_str)
|
|
if eb:
|
|
e_left, e_top, e_right, e_bottom = eb
|
|
if e_left <= orig_cx <= e_right and e_top <= orig_cy <= e_bottom:
|
|
# Exclude full-screen root background elements
|
|
e_width = e_right - e_left
|
|
e_height = e_bottom - e_top
|
|
is_full_screen = (
|
|
e_width >= width - 10 and e_height >= height - 10
|
|
)
|
|
|
|
if is_interactive(e) and not is_full_screen:
|
|
coords_interactive = True
|
|
break
|
|
|
|
# Exclude full-screen or giant parent containers from being considered blockers
|
|
is_occupant_full_screen = False
|
|
if occupant_bounds:
|
|
o_w = occupant_bounds[2] - occupant_bounds[0]
|
|
o_h = occupant_bounds[3] - occupant_bounds[1]
|
|
is_occupant_full_screen = o_w >= width - 10 and o_h >= height - 10
|
|
|
|
# If target bounds are known, check if occupant is a giant container
|
|
if target_bounds:
|
|
t_w = target_bounds[2] - target_bounds[0]
|
|
t_h = target_bounds[3] - target_bounds[1]
|
|
t_area = t_w * t_h
|
|
o_area = o_w * o_h
|
|
if t_area > 0 and o_area > 3.0 * t_area:
|
|
is_occupant_full_screen = True
|
|
|
|
if (has_content or coords_interactive) and not is_occupant_full_screen:
|
|
is_occupied = True
|
|
occupant_desc = (
|
|
f"'{occupant_text}' ({occupant_res_id})"
|
|
if has_content
|
|
else "interactive anonymous element"
|
|
)
|
|
|
|
# If coordinates are occupied by a different active/labeled element
|
|
if is_occupied:
|
|
category = ValidationErrorCategory.TARGET_OCCUPIED
|
|
reason = (
|
|
f"The expected position {original_coords} for"
|
|
f" '{target_text}' ({target_resource_id}) is"
|
|
" occupied/intercepted by a different element:"
|
|
f" {occupant_desc} at bounds {occupant_bounds}."
|
|
)
|
|
else:
|
|
# 3. Completely Missing Case
|
|
category = ValidationErrorCategory.TARGET_DISAPPEARED
|
|
reason = (
|
|
f"Target element '{target_text}' ({target_resource_id})"
|
|
f" expected at {original_coords} (bounds:"
|
|
f" {target_bounds}) was not found on the screen."
|
|
)
|
|
|
|
return passed, category, reason
|
|
|
|
def _parse_mcp_result(self, result) -> tuple[bool, str]:
|
|
"""Helper to parse MCP tool result."""
|
|
if hasattr(result, "content") and result.content:
|
|
text_content = (
|
|
result.content[0].text
|
|
if hasattr(result.content[0], "text")
|
|
else str(result.content[0])
|
|
)
|
|
if text_content.startswith("Error:") or text_content.startswith("Failed:"):
|
|
return False, text_content
|
|
elif text_content == "Success":
|
|
return True, ""
|
|
elif text_content == "Failed":
|
|
return False, "Action failed on server"
|
|
return True, text_content
|
|
return True, ""
|
|
|
|
@trace(type="tool", name="safety_net_pixel_validation")
|
|
async def _validate_action_precondition_pixel(
|
|
self,
|
|
session,
|
|
action_item: dict,
|
|
pre_screenshot_b64: str,
|
|
original_coords: list[int] | None = None,
|
|
state: State | None = None,
|
|
) -> tuple[bool, ValidationErrorCategory, str]:
|
|
"""Pixel-level Safety Net: Validates if visual target button/component at
|
|
coordinates is still present, visible, and interactive on the live screen
|
|
compared to the original screenshot.
|
|
"""
|
|
action_name = action_item.get("action")
|
|
|
|
# 1. White-list filtering: Only validate interactive, element-focused actions
|
|
if action_name not in [
|
|
"tap",
|
|
"long_press_on",
|
|
"focus_and_input_text",
|
|
"input_text",
|
|
"click_coordinate",
|
|
"click",
|
|
"long_press",
|
|
]:
|
|
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
|
|
|
|
current_coords = action_item.get("coordinates")
|
|
orig_coords = original_coords or current_coords
|
|
|
|
if not orig_coords or len(orig_coords) != 2:
|
|
logger.info("Pixel safety net skipped: no valid coordinates.")
|
|
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
|
|
|
|
if not current_coords or len(current_coords) != 2:
|
|
current_coords = orig_coords
|
|
|
|
logger.info(
|
|
"Pre-execution pixel validation: checking target position"
|
|
f" {orig_coords} in original image vs {current_coords} on live"
|
|
" screen..."
|
|
)
|
|
|
|
# 2. Decode and crop the static reference (original) image ONCE outside the loop to save CPU
|
|
|
|
try:
|
|
orig_bytes = base64.b64decode(pre_screenshot_b64)
|
|
orig_crop_bytes = visualization.crop_and_annotate_target(
|
|
orig_bytes, orig_coords, crop_size=None, dot_radius=15
|
|
)
|
|
except Exception as e:
|
|
logger.error(
|
|
f"Failed to prepare original crop for pixel safety net: {e}. Bypassing check."
|
|
)
|
|
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
|
|
|
|
# 3. Get LLM Configuration once outside the loop
|
|
|
|
try:
|
|
try:
|
|
llm = get_llm(self.ctx, name="validator_pixel_safety_net")
|
|
except Exception:
|
|
llm = get_llm(self.ctx, name="validator")
|
|
|
|
# Load prompt template once
|
|
prompt_path = Path(__file__).parent.joinpath("pixel_safety_net.md")
|
|
prompt = prompt_path.read_text(encoding="utf-8")
|
|
except Exception as e:
|
|
logger.error(
|
|
f"Failed to initialize LLM/prompt for pixel safety net: {e}. Bypassing check."
|
|
)
|
|
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
|
|
|
|
# 4. Start high-precision execution validation retry loop
|
|
max_attempts = 3
|
|
retry_delay_seconds = 0.3
|
|
last_error = None
|
|
|
|
for attempt in range(1, max_attempts + 1):
|
|
logger.info(f"Pre-execution pixel validation: Attempt {attempt}/{max_attempts}...")
|
|
try:
|
|
# A. Take fresh live screenshot (captures state changes / dynamic loading settling)
|
|
live_screenshot_b64 = None
|
|
if session is not None:
|
|
result = await session.call_tool("take_screenshot", {})
|
|
success_img, live_screenshot_b64 = self._parse_mcp_result(result)
|
|
elif getattr(self, "_local_controller", None):
|
|
live_screenshot_b64 = await self._local_controller.take_screenshot()
|
|
success_img = True
|
|
else:
|
|
success_img, live_screenshot_b64 = False, None
|
|
|
|
if not success_img or not live_screenshot_b64:
|
|
raise Exception("Failed to acquire live screenshot from controller/MCP tool.")
|
|
|
|
# B. Decode and crop the live screen target
|
|
live_bytes = base64.b64decode(live_screenshot_b64)
|
|
live_crop_bytes = visualization.crop_and_annotate_target(
|
|
live_bytes, current_coords, crop_size=None, dot_radius=15
|
|
)
|
|
|
|
orig_b64 = base64.b64encode(orig_crop_bytes).decode("utf-8")
|
|
live_b64 = base64.b64encode(live_crop_bytes).decode("utf-8")
|
|
|
|
# C. Formulate multi-modal verification message
|
|
user_content = [
|
|
{"type": "text", "text": "[Image 1 (Reference)]"},
|
|
{
|
|
"type": "image_url",
|
|
"image_url": {"url": f"data:image/jpeg;base64,{orig_b64}"},
|
|
},
|
|
{"type": "text", "text": "[Image 2 (Current State)]"},
|
|
{
|
|
"type": "image_url",
|
|
"image_url": {"url": f"data:image/jpeg;base64,{live_b64}"},
|
|
},
|
|
]
|
|
if state:
|
|
thoughts = []
|
|
native_thought = getattr(state, "operator_native_thinking", None)
|
|
if native_thought and native_thought.strip():
|
|
thoughts.append(native_thought.strip())
|
|
raw_thought = getattr(state, "operator_raw_thinking", None)
|
|
if raw_thought and raw_thought.strip():
|
|
thoughts.append(raw_thought.strip())
|
|
|
|
operator_thought = "\n\n".join(thoughts)
|
|
if operator_thought.strip():
|
|
context_text = (
|
|
"[Planned Action & Original Thinking]\nAction:"
|
|
f" {action_name}\nOriginal"
|
|
f" Thinking:\n{operator_thought.strip()}"
|
|
)
|
|
user_content.append({"type": "text", "text": f"\n{context_text.strip()}"})
|
|
|
|
messages = [
|
|
SystemMessage(content=prompt),
|
|
HumanMessage(content=user_content),
|
|
]
|
|
|
|
# D. Invoke Universal VLM
|
|
response = await llm.ainvoke(messages)
|
|
output = response.content if isinstance(response.content, str) else ""
|
|
if isinstance(response.content, list):
|
|
output = "".join(
|
|
b.get("text", "")
|
|
for b in response.content
|
|
if isinstance(b, dict) and "text" in b
|
|
)
|
|
|
|
if output.startswith("```"):
|
|
code_match = re.search(r"```(?:json)?\s*([\s\S]*?)\s*```", output)
|
|
if code_match:
|
|
output = code_match.group(1).strip()
|
|
|
|
res_json = json.loads(output)
|
|
reasoning = res_json.get("reasoning", "")
|
|
is_present = res_json.get("is_present", True)
|
|
confidence = res_json.get("confidence", 1.0)
|
|
|
|
logger.info(
|
|
f"Pixel validation attempt {attempt}/{max_attempts} result:"
|
|
f" is_present={is_present}, confidence={confidence:.2f}, reasoning={reasoning}"
|
|
)
|
|
|
|
# E. Branching based on outcome
|
|
if is_present:
|
|
logger.info(f"Pixel validation SUCCESS on attempt {attempt}/{max_attempts}.")
|
|
return True, ValidationErrorCategory.NONE, ""
|
|
|
|
# Target not present: fail or bypass based on confidence without retry!
|
|
if confidence >= 0.7:
|
|
reason_msg = (
|
|
f"Pixel-level validation failed: {reasoning}"
|
|
if reasoning
|
|
else (
|
|
"Pixel-level validation failed: The target UI"
|
|
" element at the exact coordinates has disappeared,"
|
|
" changed, or is blocked."
|
|
)
|
|
)
|
|
return (
|
|
False,
|
|
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
|
|
reason_msg,
|
|
)
|
|
else:
|
|
logger.warning(
|
|
"Pixel validation failed but confidence"
|
|
f" ({confidence:.2f}) is below 0.7. Bypassing check."
|
|
)
|
|
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
|
|
|
|
except Exception as attempt_err:
|
|
logger.warning(
|
|
f"Error during pixel validation attempt {attempt}/{max_attempts}: {attempt_err}"
|
|
)
|
|
last_error = attempt_err
|
|
if attempt < max_attempts:
|
|
await asyncio.sleep(retry_delay_seconds)
|
|
continue
|
|
else:
|
|
logger.error(
|
|
"Pixel validation repeatedly failed with errors. Last"
|
|
f" error: {last_error}. Bypassing check to avoid"
|
|
" blocking flow."
|
|
)
|
|
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
|
|
|
|
return True, ValidationErrorCategory.PIXEL_BYPASSED, ""
|