somew1nd
0f5fb1d32c
Persist and publish recoverable provider retries so temporary 503 and throttling events remain visible in live and historical task streams. Aggregate repeated retries into an expandable action card, preserve terminal pause recovery, harden Windows worker isolation and output forwarding, and retain reliable Android hierarchy fallbacks with regression coverage.
111 lines
4.4 KiB
Python
111 lines
4.4 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""MCP Tool: mobile_get_device_state."""
|
|
|
|
import base64
|
|
import os
|
|
|
|
from mcp_server.base import mcp
|
|
from artemis.mcp.adb_server import _get_controller
|
|
from mcp_server.utils import env_utils
|
|
from artemis.utils.ocr_xml_fusion import (
|
|
fuse_ocr_with_xml,
|
|
_detect_status_bar_height,
|
|
_crop_image_remove_status_bar,
|
|
_map_coordinates_back,
|
|
)
|
|
from artemis.utils.ocr_api import is_ocr_configured, perform_ocr
|
|
from artemis.utils.visualization import format_minimal_list_with_elements
|
|
|
|
|
|
@mcp.tool()
|
|
async def mobile_get_device_state(view_type: str) -> str:
|
|
"""Real-time mobile device state observer (for debugging and validation).
|
|
|
|
Use this tool to inspect the latest mobile status or to assist in tracking
|
|
the subagent's task progress.
|
|
|
|
Retrieves a real-time screenshot (returns local file URI) or a simplified
|
|
element tree text list of the currently connected mobile device.
|
|
|
|
Args:
|
|
view_type: The observation type. Supported values: - "screenshot":
|
|
Captures the real-time mobile screen and saves it to the workspace,
|
|
returning the local URI path of the image file (e.g.,
|
|
"file:///path/to/screenshot.jpg"). - "hierarchy": Directly returns the
|
|
simplified text-labeled element list, which is identical to what the
|
|
mobile UI automation subagent sees when making decisions.
|
|
"""
|
|
try:
|
|
controller = _get_controller()
|
|
device_width = controller.ctx.device.device_width
|
|
device_height = controller.ctx.device.device_height
|
|
except Exception as e:
|
|
return f"Error: Failed to initialize/lock Android device controller: {e}"
|
|
|
|
try:
|
|
device_data = await controller.get_screen_data()
|
|
latest_screenshot_b64 = device_data.base64
|
|
xml_hierarchy = device_data.elements
|
|
|
|
project_root = env_utils.get_project_root()
|
|
|
|
if view_type == "screenshot":
|
|
screenshot_bytes = base64.b64decode(latest_screenshot_b64)
|
|
device_id = controller.ctx.device.device_id
|
|
safe_device_id = "".join(
|
|
[c if c.isalnum() or c in ("-", "_") else "_" for c in device_id]
|
|
)
|
|
screenshot_filename = f"live_screenshot_{safe_device_id}.jpg"
|
|
screenshot_path = os.path.join(project_root, screenshot_filename)
|
|
|
|
with open(screenshot_path, "wb") as f:
|
|
f.write(screenshot_bytes)
|
|
|
|
return f"file://{screenshot_path}"
|
|
|
|
elif view_type == "hierarchy":
|
|
ocr_results = []
|
|
if is_ocr_configured():
|
|
try:
|
|
screen_height = device_data.height
|
|
status_bar_height = _detect_status_bar_height(
|
|
xml_hierarchy, screen_height
|
|
)
|
|
if status_bar_height > 0:
|
|
cropped_b64, _, _ = _crop_image_remove_status_bar(
|
|
latest_screenshot_b64, status_bar_height
|
|
)
|
|
raw_ocr_results = await perform_ocr(cropped_b64)
|
|
ocr_results = _map_coordinates_back(
|
|
raw_ocr_results, status_bar_height
|
|
)
|
|
else:
|
|
ocr_results = await perform_ocr(latest_screenshot_b64)
|
|
except Exception:
|
|
ocr_results = []
|
|
|
|
fused_xml = fuse_ocr_with_xml(xml_hierarchy, ocr_results)
|
|
hierarchy_text, elements, labels = format_minimal_list_with_elements(
|
|
fused_xml, width=device_width, height=device_height
|
|
)
|
|
return hierarchy_text
|
|
|
|
else:
|
|
return f"Error: Invalid view_type '{view_type}'. Supported types are 'screenshot' and 'hierarchy'."
|
|
|
|
except Exception as e:
|
|
return f"Error: An unexpected error occurred while communicating with the device: {e}"
|