feat: introduce MCP server, cross-platform runtime, and Showcase UI reorganization
- Showcase UI Reorganization: * Move web UI dashboard from root directory into apps/showcase_ui/ for cleaner project structure * Add 'artemis ui' CLI command (artemis/interfaces/cli/commands/ui.py) and launcher scripts for the interactive showcase console - Standalone MCP Server & Notifiers (mcp_server/): * Implement dedicated MCP server package with mobile automation tools (device_state, inspect_trace, task_manager, task_runner) * Add multi-channel notifiers (AgentAPI, Desktop notifications, Webhooks, File output, and Composite routing) * Introduce background task runner and trace store for autonomous mobile execution monitoring - Cross-Platform Runtime & Toolchain Support: * Add artemis/platform/ with OS-specific implementations for Linux, macOS (Darwin), and Windows * Add artemis/runtime/supervisor.py for resilient process and agent runtime supervision * Add artemis/toolchain/ with dependency resolution and SDK descriptors * Add cross-platform launcher scripts (start.sh, start.bat) and installer scripts (scripts/install_deps.*) - Core Enhancements & Diagnostics: * Enhance coordinate normalization, OCR API fallback, and perception pipeline * Add credential validation (credentials_validator.py) and system diagnostics * Update bilingual documentation (README.md, README_CN.md) and architecture diagrams - Test Suite Expansion: * Add integration tests for cloud gateway, remote ADB, and Artemis login * Add unit tests covering core diagnostics, platform detection, toolchain, credentials validator, optional OCR, and MCP tools
This commit is contained in:
1 parent
78e478f502
commit
9a5aaf99b7
249 files changed
+37226
-9339
No files matched your search
+244
-15
@@ -20,7 +20,7 @@ from typing import Any
|
||||
|
||||
def parse_swipe_parameters(
|
||||
args: Any,
|
||||
default_duration: int | None = 400,
|
||||
default_duration: int | None = 800,
|
||||
) -> tuple[str | None, str | list[int] | None, int | None]:
|
||||
"""Parses swipe/drag/scroll parameters from various possible input formats into a standardized tuple:
|
||||
|
||||
@@ -130,6 +130,188 @@ def parse_swipe_parameters(
|
||||
return None, None, duration_int
|
||||
|
||||
|
||||
def _extract_bounds(val: Any) -> list[int] | None:
|
||||
"""Safely parses bounds representation into [left, top, right, bottom]."""
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, (list, tuple)) and len(val) == 4:
|
||||
try:
|
||||
return [int(float(x)) for x in val]
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
if isinstance(val, str):
|
||||
match = re.match(r"\[(\-?\d+),(\-?\d+)\]\[(\-?\d+),(\-?\d+)\]", val.strip())
|
||||
if match:
|
||||
return [int(x) for x in match.groups()]
|
||||
nums = re.findall(r"-?\d+", val)
|
||||
if len(nums) == 4:
|
||||
return [int(x) for x in nums]
|
||||
return None
|
||||
|
||||
|
||||
def resolve_scrollable_container_bounds(
|
||||
target: Any = None,
|
||||
indexed_elements: list[dict[str, Any]] | None = None,
|
||||
ui_hierarchy: list[dict[str, Any]] | None = None,
|
||||
screen_width: int = 1080,
|
||||
screen_height: int = 2400,
|
||||
) -> list[int]:
|
||||
"""Resolves the best-effort bounding box [left, top, right, bottom] for scrolling.
|
||||
|
||||
Heuristic Priority:
|
||||
1. Explicit target element bounds (if target index/element/bounds passed).
|
||||
2. Primary active scrollable container (RecyclerView, ScrollView, ListView, WebView, ViewPager).
|
||||
3. Fallback to full screen [0, 0, screen_width, screen_height].
|
||||
"""
|
||||
# 1. Target element reference
|
||||
if target is not None:
|
||||
direct_b = _extract_bounds(target)
|
||||
if direct_b and len(direct_b) == 4:
|
||||
return direct_b
|
||||
|
||||
target_idx = None
|
||||
if isinstance(target, (int, float)):
|
||||
target_idx = int(target)
|
||||
elif isinstance(target, str) and target.strip().isdigit():
|
||||
target_idx = int(target.strip())
|
||||
|
||||
if target_idx is not None and indexed_elements:
|
||||
if 1 <= target_idx <= len(indexed_elements):
|
||||
elem = indexed_elements[target_idx - 1]
|
||||
b = _extract_bounds(elem.get("bounds"))
|
||||
if b and len(b) == 4:
|
||||
return b
|
||||
|
||||
# 2. Look for primary scrollable container in ui_hierarchy or indexed_elements
|
||||
candidates: list[list[int]] = []
|
||||
|
||||
def scan_hierarchy(nodes: list[dict[str, Any]]):
|
||||
for node in nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
is_scroll = (
|
||||
node.get("scrollable") is True or str(node.get("scrollable", "")).lower() == "true"
|
||||
)
|
||||
cls_name = str(node.get("class") or node.get("className") or "").lower()
|
||||
if not is_scroll:
|
||||
for kw in ("scrollview", "recyclerview", "listview", "webview", "viewpager"):
|
||||
if kw in cls_name:
|
||||
is_scroll = True
|
||||
break
|
||||
if is_scroll:
|
||||
b = _extract_bounds(node.get("bounds"))
|
||||
if b and len(b) == 4:
|
||||
w = b[2] - b[0]
|
||||
h = b[3] - b[1]
|
||||
if w >= int(screen_width * 0.25) and h >= int(screen_height * 0.15):
|
||||
candidates.append(b)
|
||||
children = node.get("children")
|
||||
if isinstance(children, list) and children:
|
||||
scan_hierarchy(children)
|
||||
|
||||
if ui_hierarchy:
|
||||
scan_hierarchy(ui_hierarchy)
|
||||
elif indexed_elements:
|
||||
for elem in indexed_elements:
|
||||
b = _extract_bounds(elem.get("bounds"))
|
||||
cls_name = str(elem.get("class") or elem.get("className") or "").lower()
|
||||
if any(
|
||||
kw in cls_name
|
||||
for kw in ("scrollview", "recyclerview", "listview", "webview", "viewpager")
|
||||
):
|
||||
if b and len(b) == 4:
|
||||
candidates.append(b)
|
||||
|
||||
if candidates:
|
||||
|
||||
def area(b: list[int]) -> int:
|
||||
return max(0, b[2] - b[0]) * max(0, b[3] - b[1])
|
||||
|
||||
candidates.sort(key=area, reverse=True)
|
||||
return candidates[0]
|
||||
|
||||
return [0, 0, screen_width, screen_height]
|
||||
|
||||
|
||||
def compute_smart_swipe_coordinates(
|
||||
direction: str,
|
||||
target: Any = None,
|
||||
indexed_elements: list[dict[str, Any]] | None = None,
|
||||
ui_hierarchy: list[dict[str, Any]] | None = None,
|
||||
width: int = 1080,
|
||||
height: int = 2400,
|
||||
duration: int | None = None,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
"""Computes best-effort container-aware intelligent swipe coordinates and duration.
|
||||
|
||||
Returns: (start_x, start_y, end_x, end_y, duration_ms)
|
||||
"""
|
||||
container = resolve_scrollable_container_bounds(
|
||||
target=target,
|
||||
indexed_elements=indexed_elements,
|
||||
ui_hierarchy=ui_hierarchy,
|
||||
screen_width=width,
|
||||
screen_height=height,
|
||||
)
|
||||
|
||||
c_left = max(0, min(width - 1, container[0]))
|
||||
c_top = max(0, min(height - 1, container[1]))
|
||||
c_right = max(c_left + 50, min(width, container[2]))
|
||||
c_bottom = max(c_top + 50, min(height, container[3]))
|
||||
|
||||
c_width = c_right - c_left
|
||||
c_height = c_bottom - c_top
|
||||
|
||||
dir_str = str(direction or "up").strip().lower()
|
||||
|
||||
if dir_str == "up":
|
||||
# Drag upwards -> reveal content below
|
||||
sx = ex = int(c_left + c_width * 0.60)
|
||||
sy = int(c_top + c_height * 0.70)
|
||||
ey = int(c_top + c_height * 0.30)
|
||||
elif dir_str == "down":
|
||||
# Drag downwards -> reveal content above
|
||||
sx = ex = int(c_left + c_width * 0.60)
|
||||
sy = int(c_top + c_height * 0.30)
|
||||
ey = int(c_top + c_height * 0.70)
|
||||
elif dir_str == "left":
|
||||
# Drag leftwards -> page right
|
||||
sy = ey = int(c_top + c_height * 0.50)
|
||||
sx = int(c_left + c_width * 0.75)
|
||||
ex = int(c_left + c_width * 0.25)
|
||||
elif dir_str == "right":
|
||||
# Drag rightwards -> page left
|
||||
sy = ey = int(c_top + c_height * 0.50)
|
||||
sx = int(c_left + c_width * 0.25)
|
||||
ex = int(c_left + c_width * 0.75)
|
||||
else:
|
||||
# Default up
|
||||
sx = ex = int(c_left + c_width * 0.60)
|
||||
sy = int(c_top + c_height * 0.70)
|
||||
ey = int(c_top + c_height * 0.30)
|
||||
|
||||
# Clamp coordinates to screen bounds
|
||||
sx = max(0, min(width - 1, sx))
|
||||
ex = max(0, min(width - 1, ex))
|
||||
sy = max(0, min(height - 1, sy))
|
||||
ey = max(0, min(height - 1, ey))
|
||||
|
||||
if duration is None:
|
||||
dist = ((ex - sx) ** 2 + (ey - sy) ** 2) ** 0.5
|
||||
if dist <= 300:
|
||||
duration_ms = 350
|
||||
elif dist <= 600:
|
||||
duration_ms = 550
|
||||
elif dist <= 1000:
|
||||
duration_ms = 800
|
||||
else:
|
||||
duration_ms = 900
|
||||
else:
|
||||
duration_ms = int(duration)
|
||||
|
||||
return sx, sy, ex, ey, duration_ms
|
||||
|
||||
|
||||
def normalize_point(x: int, y: int, width: int, height: int) -> list[int]:
|
||||
"""Convert physical pixels to normalized 0-1000 coordinates."""
|
||||
nx = int(round(x * 1000.0 / width))
|
||||
@@ -216,13 +398,13 @@ def normalize_step_actions(step_dict: dict[str, Any]) -> dict[str, Any]:
|
||||
item["coordinates"] = target
|
||||
elif kind == "direction" and isinstance(target, str):
|
||||
if target == "up":
|
||||
norm = [600, 800, 600, 200]
|
||||
norm = [600, 700, 600, 300]
|
||||
elif target == "down":
|
||||
norm = [600, 200, 600, 800]
|
||||
norm = [600, 300, 600, 700]
|
||||
elif target == "left":
|
||||
norm = [800, 500, 200, 500]
|
||||
norm = [750, 500, 250, 500]
|
||||
elif target == "right":
|
||||
norm = [200, 500, 800, 500]
|
||||
norm = [250, 500, 750, 500]
|
||||
else:
|
||||
norm = None
|
||||
if norm:
|
||||
@@ -255,6 +437,43 @@ def normalize_step_actions(step_dict: dict[str, Any]) -> dict[str, Any]:
|
||||
item["normalized_end_coordinates"] = [nx2, ny2]
|
||||
else:
|
||||
item["normalized_coordinates"] = coords
|
||||
|
||||
seq = item.get("sequence") or item.get("targets")
|
||||
if seq:
|
||||
if isinstance(seq, str):
|
||||
try:
|
||||
seq = json.loads(seq)
|
||||
except Exception:
|
||||
try:
|
||||
import ast
|
||||
|
||||
seq = ast.literal_eval(seq)
|
||||
except Exception:
|
||||
pass
|
||||
if isinstance(seq, (list, tuple)):
|
||||
item["sequence"] = list(seq)
|
||||
if isinstance(item.get("sequence"), list) and "normalized_sequence" not in item:
|
||||
norm_seq = []
|
||||
for pt in item["sequence"]:
|
||||
if (
|
||||
isinstance(pt, (list, tuple))
|
||||
and len(pt) == 2
|
||||
and all(isinstance(c, (int, float)) for c in pt)
|
||||
):
|
||||
if any(c > 1000 for c in pt) or item.get("target_bounds"):
|
||||
norm_seq.append(
|
||||
[
|
||||
int(round(pt[0] * 1000.0 / width)),
|
||||
int(round(pt[1] * 1000.0 / height)),
|
||||
]
|
||||
)
|
||||
else:
|
||||
norm_seq.append([int(round(pt[0])), int(round(pt[1]))])
|
||||
elif isinstance(pt, int):
|
||||
norm_seq.append(pt)
|
||||
if norm_seq:
|
||||
item["normalized_sequence"] = norm_seq
|
||||
|
||||
return item
|
||||
|
||||
action_taken = step_dict.get("action_taken")
|
||||
@@ -326,16 +545,26 @@ def normalize_step_actions(step_dict: dict[str, Any]) -> dict[str, Any]:
|
||||
and isinstance(action_taken[0], dict)
|
||||
else (action_taken if isinstance(action_taken, dict) else {})
|
||||
)
|
||||
# Enrich missing coordinates/target/input from action_taken
|
||||
if "target" not in args and "coordinates" not in args:
|
||||
if first_act.get("coordinates"):
|
||||
args["coordinates"] = first_act["coordinates"]
|
||||
if first_act.get("args", {}).get("target"):
|
||||
args["target"] = first_act["args"]["target"]
|
||||
if "normalized_coordinates" not in args and first_act.get("normalized_coordinates"):
|
||||
args["normalized_coordinates"] = first_act["normalized_coordinates"]
|
||||
if "text" not in args and first_act.get("text"):
|
||||
args["text"] = first_act["text"]
|
||||
|
||||
is_seq_tool = (
|
||||
raw_name in ("click_sequence", "tap_sequence")
|
||||
or "sequence" in args
|
||||
or "targets" in args
|
||||
)
|
||||
|
||||
# Enrich missing coordinates/target/input from action_taken only if not a multi-point sequence action
|
||||
if not is_seq_tool:
|
||||
if "target" not in args and "coordinates" not in args:
|
||||
if first_act.get("coordinates"):
|
||||
args["coordinates"] = first_act["coordinates"]
|
||||
if first_act.get("args", {}).get("target"):
|
||||
args["target"] = first_act["args"]["target"]
|
||||
if "normalized_coordinates" not in args and first_act.get(
|
||||
"normalized_coordinates"
|
||||
):
|
||||
args["normalized_coordinates"] = first_act["normalized_coordinates"]
|
||||
if "text" not in args and first_act.get("text"):
|
||||
args["text"] = first_act["text"]
|
||||
|
||||
args = _enrich_action_item(args)
|
||||
payload["args"] = args
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Validation utility to verify LLM and Vision API keys against live provider endpoints."""
|
||||
|
||||
import httpx
|
||||
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Minimal 1x1 PNG transparent pixel base64 for Vision API testing
|
||||
_TINY_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
|
||||
|
||||
|
||||
def _extract_error_message(resp: httpx.Response) -> str:
|
||||
"""Extract a human-friendly error message from a provider HTTP response."""
|
||||
try:
|
||||
data = resp.json()
|
||||
if isinstance(data, dict):
|
||||
# Google style: {"error": {"message": "..."}}
|
||||
err_obj = data.get("error")
|
||||
if isinstance(err_obj, dict):
|
||||
msg = err_obj.get("message")
|
||||
if msg:
|
||||
return str(msg)
|
||||
elif isinstance(err_obj, str):
|
||||
return err_obj
|
||||
|
||||
# OpenAI / Anthropic / OpenRouter style
|
||||
if "message" in data:
|
||||
return str(data["message"])
|
||||
if "detail" in data:
|
||||
return str(data["detail"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback to response text or HTTP status
|
||||
text = resp.text.strip()
|
||||
if text and len(text) < 200:
|
||||
return text
|
||||
return f"HTTP {resp.status_code}: {resp.reason_phrase}"
|
||||
|
||||
|
||||
async def validate_api_key(
|
||||
provider: str,
|
||||
api_key: str,
|
||||
base_url: str | None = None,
|
||||
timeout: float = 12.0,
|
||||
) -> tuple[bool, str]:
|
||||
"""Tests if the provided API key is valid and usable with the corresponding provider.
|
||||
|
||||
Args:
|
||||
provider: Provider identifier (e.g. 'google', 'gemini', 'ocr', 'openai', 'anthropic', etc.)
|
||||
api_key: Secret API key string.
|
||||
base_url: Optional custom endpoint URL.
|
||||
timeout: Request timeout in seconds.
|
||||
|
||||
Returns:
|
||||
tuple[bool, str]: (is_valid, descriptive_message)
|
||||
"""
|
||||
clean_provider = provider.strip().lower()
|
||||
clean_key = api_key.strip()
|
||||
|
||||
if not clean_key:
|
||||
return False, "API key cannot be empty."
|
||||
|
||||
# Placeholders / default template values should be rejected
|
||||
if clean_key in (
|
||||
"API_KEY",
|
||||
"your_gemini_api_key_here",
|
||||
"your_google_cloud_vision_api_key_here",
|
||||
"your_openai_api_key_here",
|
||||
"your_anthropic_api_key_here",
|
||||
):
|
||||
return False, "Please enter a real API key instead of the placeholder."
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
if clean_provider in ("google", "gemini", "vertexai", "vertex"):
|
||||
# Test Google Gemini via generativelanguage API
|
||||
url = f"https://generativelanguage.googleapis.com/v1beta/models?key={clean_key}&pageSize=1"
|
||||
resp = await client.get(url)
|
||||
if resp.status_code == 200:
|
||||
return True, "Google Gemini API key verified successfully."
|
||||
err_msg = _extract_error_message(resp)
|
||||
return False, f"Gemini API verification failed ({resp.status_code}): {err_msg}"
|
||||
|
||||
elif clean_provider in ("ocr", "vision", "google_vision"):
|
||||
# Test Google Cloud Vision OCR API
|
||||
url = f"https://vision.googleapis.com/v1/images:annotate?key={clean_key}"
|
||||
payload = {
|
||||
"requests": [
|
||||
{
|
||||
"image": {"content": _TINY_PNG_B64},
|
||||
"features": [{"type": "TEXT_DETECTION"}],
|
||||
}
|
||||
]
|
||||
}
|
||||
resp = await client.post(
|
||||
url, json=payload, headers={"Content-Type": "application/json"}
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
return True, "Google Cloud Vision OCR API key verified successfully."
|
||||
err_msg = _extract_error_message(resp)
|
||||
return False, f"Vision OCR API verification failed ({resp.status_code}): {err_msg}"
|
||||
|
||||
elif clean_provider == "openai":
|
||||
endpoint = (
|
||||
f"{base_url.rstrip('/')}/models"
|
||||
if base_url
|
||||
else "https://api.openai.com/v1/models"
|
||||
)
|
||||
resp = await client.get(endpoint, headers={"Authorization": f"Bearer {clean_key}"})
|
||||
if resp.status_code == 200:
|
||||
return True, "OpenAI API key verified successfully."
|
||||
err_msg = _extract_error_message(resp)
|
||||
return False, f"OpenAI API verification failed ({resp.status_code}): {err_msg}"
|
||||
|
||||
elif clean_provider in ("anthropic", "claude"):
|
||||
url = "https://api.anthropic.com/v1/models"
|
||||
resp = await client.get(
|
||||
url,
|
||||
headers={
|
||||
"x-api-key": clean_key,
|
||||
"anthropic-version": "2023-06-01",
|
||||
},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
return True, "Anthropic Claude API key verified successfully."
|
||||
err_msg = _extract_error_message(resp)
|
||||
return False, f"Anthropic API verification failed ({resp.status_code}): {err_msg}"
|
||||
|
||||
elif clean_provider == "openrouter":
|
||||
url = "https://openrouter.ai/api/v1/auth/key"
|
||||
resp = await client.get(url, headers={"Authorization": f"Bearer {clean_key}"})
|
||||
if resp.status_code == 200:
|
||||
return True, "OpenRouter API key verified successfully."
|
||||
err_msg = _extract_error_message(resp)
|
||||
return False, f"OpenRouter API verification failed ({resp.status_code}): {err_msg}"
|
||||
|
||||
elif clean_provider in ("xai", "grok"):
|
||||
url = "https://api.x.ai/v1/models"
|
||||
resp = await client.get(url, headers={"Authorization": f"Bearer {clean_key}"})
|
||||
if resp.status_code == 200:
|
||||
return True, "xAI API key verified successfully."
|
||||
err_msg = _extract_error_message(resp)
|
||||
return False, f"xAI API verification failed ({resp.status_code}): {err_msg}"
|
||||
|
||||
elif clean_provider in ("ollama", "vllm", "custom"):
|
||||
if base_url:
|
||||
endpoint = f"{base_url.rstrip('/')}/models"
|
||||
headers = (
|
||||
{"Authorization": f"Bearer {clean_key}"} if clean_key != "EMPTY" else {}
|
||||
)
|
||||
resp = await client.get(endpoint, headers=headers)
|
||||
if resp.status_code == 200:
|
||||
return True, f"Custom endpoint ({base_url}) verified successfully."
|
||||
err_msg = _extract_error_message(resp)
|
||||
return (
|
||||
False,
|
||||
f"Custom endpoint verification failed ({resp.status_code}): {err_msg}",
|
||||
)
|
||||
return True, "Custom endpoint configured."
|
||||
|
||||
else:
|
||||
# Unknown provider - fallback to basic length check
|
||||
if len(clean_key) >= 8:
|
||||
return True, f"Credentials for {provider} configured."
|
||||
return False, f"Invalid API key format for {provider}."
|
||||
|
||||
except httpx.TimeoutException:
|
||||
logger.warning(f"Timeout verifying API key for {provider}")
|
||||
return (
|
||||
False,
|
||||
f"Connection to {provider.capitalize()} API timed out. Please check network/proxy settings.",
|
||||
)
|
||||
except httpx.RequestError as exc:
|
||||
logger.warning(f"Network error verifying API key for {provider}: {exc}")
|
||||
return False, f"Network error connecting to {provider.capitalize()} API: {exc}"
|
||||
except Exception as exc:
|
||||
logger.error(f"Unexpected error validating API key for {provider}: {exc}")
|
||||
return False, f"Validation error: {exc}"
|
||||
+27
-12
@@ -30,13 +30,33 @@ def get_http_client() -> httpx.AsyncClient:
|
||||
return _HTTP_CLIENT
|
||||
|
||||
|
||||
def is_ocr_configured() -> bool:
|
||||
"""Checks if a valid Google Vision OCR API key is configured."""
|
||||
ocr_secret = settings.get_api_key("ocr")
|
||||
if ocr_secret and ocr_secret.get_secret_value().strip():
|
||||
val = ocr_secret.get_secret_value().strip()
|
||||
if val not in ("API_KEY", "your_google_cloud_vision_api_key_here"):
|
||||
return True
|
||||
for env_var in ("OCR_API_KEY", "VISION_API_KEY"):
|
||||
val = os.environ.get(env_var)
|
||||
if (
|
||||
val
|
||||
and val.strip()
|
||||
and val.strip() not in ("API_KEY", "your_google_cloud_vision_api_key_here")
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def perform_ocr(
|
||||
screenshot_b64: str, client: httpx.AsyncClient = None
|
||||
screenshot_b64: str,
|
||||
client: httpx.AsyncClient | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Calls Google Vision API for OCR using a persistent client to reuse TLS handshakes.
|
||||
"""Calls Google Vision API for text recognition on an image.
|
||||
|
||||
Args:
|
||||
screenshot_b64: Base64 encoded screenshot image.
|
||||
client: Optional persistent httpx.AsyncClient.
|
||||
|
||||
Returns:
|
||||
A list of dictionaries containing detected text and position vertices.
|
||||
@@ -46,10 +66,9 @@ async def perform_ocr(
|
||||
(ocr_secret.get_secret_value() if ocr_secret else None)
|
||||
or os.environ.get("OCR_API_KEY")
|
||||
or os.environ.get("VISION_API_KEY")
|
||||
or os.environ.get("API_KEY")
|
||||
or os.environ.get("GOOGLE_API_KEY")
|
||||
or "API_KEY"
|
||||
)
|
||||
if not api_key:
|
||||
return []
|
||||
|
||||
url = f"https://vision.googleapis.com/v1/images:annotate?key={api_key}"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
@@ -62,13 +81,9 @@ async def perform_ocr(
|
||||
]
|
||||
}
|
||||
|
||||
try:
|
||||
active_client = client if client is not None else get_http_client()
|
||||
response = await active_client.post(url, json=data, headers=headers, timeout=30.0)
|
||||
return _parse_ocr_response(response)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to call Vision API for OCR: {e}")
|
||||
raise e
|
||||
active_client = client if client is not None else get_http_client()
|
||||
response = await active_client.post(url, json=data, headers=headers, timeout=30.0)
|
||||
return _parse_ocr_response(response)
|
||||
|
||||
|
||||
def _parse_ocr_response(response: httpx.Response) -> list[dict[str, Any]]:
|
||||
|
||||
@@ -583,35 +583,88 @@ def draw_action_overlay_on_image(
|
||||
|
||||
# 2. Click Sequence
|
||||
elif action in ("click_sequence", "tap_sequence"):
|
||||
sequence = action_args.get("sequence") or action_args.get("targets") or []
|
||||
sequence = (
|
||||
action_args.get("sequence")
|
||||
or action_args.get("targets")
|
||||
or action_args.get("coordinates")
|
||||
or []
|
||||
)
|
||||
if isinstance(sequence, str):
|
||||
sequence_str = sequence.strip()
|
||||
try:
|
||||
import json
|
||||
|
||||
sequence = json.loads(sequence_str)
|
||||
except Exception:
|
||||
try:
|
||||
import ast
|
||||
|
||||
sequence = ast.literal_eval(sequence_str)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if isinstance(sequence, list):
|
||||
for idx, coord in enumerate(sequence, 1):
|
||||
pts = []
|
||||
for coord in sequence:
|
||||
pt = _resolve_coordinates(coord, width, height)
|
||||
if pt:
|
||||
x, y = pt
|
||||
draw.ellipse(
|
||||
[x - dot_radius, y - dot_radius, x + dot_radius, y + dot_radius],
|
||||
fill="red",
|
||||
outline="white",
|
||||
width=line_width,
|
||||
pts.append(pt)
|
||||
|
||||
# Draw connecting lines/arrows between consecutive sequence points
|
||||
for i in range(len(pts) - 1):
|
||||
p1, p2 = pts[i], pts[i + 1]
|
||||
draw.line([p1, p2], fill="white", width=line_width + 3)
|
||||
draw.line([p1, p2], fill="red", width=line_width)
|
||||
|
||||
# Arrow head
|
||||
dx = p2[0] - p1[0]
|
||||
dy = p2[1] - p1[1]
|
||||
angle = math.atan2(dy, dx)
|
||||
arrow_len = max(16, int(base_dim * 0.025))
|
||||
arrow_angle = math.pi / 6
|
||||
x1 = p2[0] - arrow_len * math.cos(angle - arrow_angle)
|
||||
y1 = p2[1] - arrow_len * math.sin(angle - arrow_angle)
|
||||
x2 = p2[0] - arrow_len * math.cos(angle + arrow_angle)
|
||||
y2 = p2[1] - arrow_len * math.sin(angle + arrow_angle)
|
||||
draw.polygon([p2, (x1, y1), (x2, y2)], fill="red", outline="white")
|
||||
|
||||
# Draw numbered circular touch points
|
||||
for idx, (x, y) in enumerate(pts, 1):
|
||||
# Outer aura
|
||||
draw.ellipse(
|
||||
[
|
||||
x - dot_radius * 1.3,
|
||||
y - dot_radius * 1.3,
|
||||
x + dot_radius * 1.3,
|
||||
y + dot_radius * 1.3,
|
||||
],
|
||||
outline="white",
|
||||
width=line_width + 1,
|
||||
)
|
||||
# Colored circle
|
||||
draw.ellipse(
|
||||
[x - dot_radius, y - dot_radius, x + dot_radius, y + dot_radius],
|
||||
fill="red",
|
||||
outline="white",
|
||||
width=line_width,
|
||||
)
|
||||
label = str(idx)
|
||||
try:
|
||||
font = ImageFont.load_default(size=int(dot_radius * 1.2))
|
||||
draw.text(
|
||||
(x - dot_radius // 3, y - dot_radius // 2),
|
||||
label,
|
||||
fill="white",
|
||||
font=font,
|
||||
stroke_width=2,
|
||||
stroke_fill="black",
|
||||
)
|
||||
except Exception:
|
||||
draw.text(
|
||||
(x - 4, y - 6),
|
||||
label,
|
||||
fill="white",
|
||||
)
|
||||
label = str(idx)
|
||||
try:
|
||||
font = ImageFont.load_default(size=int(dot_radius * 1.2))
|
||||
draw.text(
|
||||
(x - dot_radius // 3, y - dot_radius // 2),
|
||||
label,
|
||||
fill="white",
|
||||
font=font,
|
||||
stroke_width=2,
|
||||
stroke_fill="black",
|
||||
)
|
||||
except Exception:
|
||||
draw.text(
|
||||
(x - 4, y - 6),
|
||||
label,
|
||||
fill="white",
|
||||
)
|
||||
|
||||
# 3. Swipe / Drag / Scroll
|
||||
elif action in ("swipe", "drag", "scroll"):
|
||||
|
||||
Reference in new issue
Block a user