diff --git a/.env.example b/.env.example index f8b492e..8b23d0a 100644 --- a/.env.example +++ b/.env.example @@ -9,7 +9,7 @@ GEMINI_API_KEY= GOOGLE_API_KEY= OPENAI_API_KEY= ANTHROPIC_API_KEY= -OPENROUTER_API_KEY= +OPEN_ROUTER_API_KEY= XAI_API_KEY= # Google Cloud Vision OCR (Optional - for advanced OCR processing) diff --git a/NOTICE b/NOTICE deleted file mode 100644 index e7cd397..0000000 --- a/NOTICE +++ /dev/null @@ -1,27 +0,0 @@ -ARTEMIS Android Agent & Automation Platform -Copyright 2026 Google LLC - -This product includes software developed by Google LLC. - -================================================================================ -Third-Party Software Attribution -================================================================================ - -Portions of this software are derived from and incorporate code originally -developed by the following open-source projects under the Apache License, Version 2.0: - -1. mobile-use - Repository: https://github.com/minitap-ai/mobile-use - Copyright 2025-2026 Minitap, Inc. - Original authors & contributors: - Pierre-Louis Favreau, Jean-Pierre Lo, Nicolas Dehandschoewercker, - Clément Guiguet, Karun Agarwal, and the mobile-use open-source community. - License: Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) - -2. finalrun-agent - Repository: https://github.com/final-run/finalrun-agent - Copyright 2025 FinalRun Inc. - Original authors & contributors: - Ashish Yadav, Arnold Laishram, Srinidhi G S, - and the finalrun-agent open-source community. - License: Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) diff --git a/README.md b/README.md index 2e81c30..0c6e2c6 100644 --- a/README.md +++ b/README.md @@ -314,7 +314,3 @@ Contributions are warmly welcomed! ## License This project is licensed under the [Apache License 2.0](LICENSE). - -This project includes source code developed by [Minitap, Inc.](https://github.com/minitap-ai/mobile-use). - - diff --git a/README_CN.md b/README_CN.md index 7a2e342..9b1995a 100644 --- a/README_CN.md +++ b/README_CN.md @@ -312,14 +312,3 @@ ARTEMIS 提供两种运行模式以适应不同的自动化需求： ## 开源许可证 本项目基于 [Apache License 2.0](LICENSE) 协议开源。 - -## 致谢 - -本项目包含由 [Minitap, Inc.](https://github.com/minitap-ai/mobile-use)（`mobile-use`）团队开发的部分源代码与工具组件，遵循 Apache License 2.0 开源许可。我们感谢其团队及开源社区在移动端自动化领域的贡献。 - -同时感谢 [Finalrun](https://github.com/final-run/finalrun-agent) 在智能滑动与自适应手势机制理念上的启发。 - -关于完整的第三方开源软件版权与归属声明，请参阅根目录的 [NOTICE](NOTICE) 文件。 - - - diff --git a/artemis/agents/entity_extractor/entity_extractor.md b/artemis/agents/entity_extractor/entity_extractor.md deleted file mode 100644 index 6bf1325..0000000 --- a/artemis/agents/entity_extractor/entity_extractor.md +++ /dev/null @@ -1,30 +0,0 @@ -# Artemis Entity Extractor & Package Resolver - -You are an autonomous, high-precision structured data extractor in the ARTEMIS mobile testing runtime. -Your responsibility is to extract the exact requested identifier, package name, or configuration token from raw device dump outputs (such as `pm list packages`, `dumpsys`, logcat records, or accessibility tree dumps). - -## Operating Directives - -1. **Verbatim Extraction**: - - Return the extracted entity exactly as it appears in the source data. - - Do NOT modify capitalization, strip domain segments, translate, or normalize the matched identifier. - -2. **Package Resolution Rules**: - - Match target application titles, codenames, or brand identifiers against standard Android package formats (`com..`, `org..`, etc.). - - Prioritize exact matches over partial or prefix matches. - -3. **Strict Disambiguation & Null-Safety**: - - If the requested entity is absent from the input dataset, return `found: false` with `output: null`. - - If multiple candidates conflict and cannot be deterministically verified from the query, do NOT guess. Set `found: false` and `output: null`. - -## Structured Deliverable Schema - -You must produce your final answer conforming to the following JSON structure: - -```json -{ - "found": true, - "output": "", - "reason": "" -} -``` diff --git a/artemis/agents/entity_extractor/entity_extractor.py b/artemis/agents/entity_extractor/entity_extractor.py deleted file mode 100644 index bac4aff..0000000 --- a/artemis/agents/entity_extractor/entity_extractor.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Lightweight entity extraction and package resolver for ARTEMIS. - -Dispatches bounded LLM extraction turns to identify application package identifiers -and configuration attributes from raw device listings without manual parsing heuristics. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import cast - -from jinja2 import Template -from langchain_core.messages import HumanMessage, SystemMessage -from pydantic import BaseModel, Field - -from artemis.context import ArtemisContext -from artemis.data_engine.trace import trace -from artemis.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback -from artemis.utils.logger import get_logger - -logger = get_logger(__name__) - - -class ExtractionResult(BaseModel): - """Structured extraction outcome for entity resolution.""" - - found: bool = Field(description="True if the requested entity was definitively identified.") - output: str | None = Field(default=None, description="Exact extracted identifier or string value.") - reason: str = Field(description="Brief justification of the match criteria applied.") - - -@trace(type="agent", name="entity_extractor") -async def extract_entity( - ctx: ArtemisContext, - request: str, - data: str, - use_fallback: bool = True, -) -> ExtractionResult: - """Extract a specific identifier or entity from raw batch device data.""" - logger.info(f"Invoking Artemis Entity Extractor (fallback={use_fallback})") - template_path = Path(__file__).parent / "entity_extractor.md" - system_prompt = Template(template_path.read_text(encoding="utf-8")).render() - - messages = [ - SystemMessage(content=system_prompt), - HumanMessage(content=f"Target Query:\n{request}\n\nCandidate Raw Dataset:\n{data}"), - ] - - primary_llm = get_llm(ctx=ctx, name="entity_extractor", is_utils=True).with_structured_output( - ExtractionResult - ) - - try: - if use_fallback: - fallback_llm = get_llm( - ctx=ctx, name="entity_extractor", is_utils=True, use_fallback=True - ).with_structured_output(ExtractionResult) - raw = await with_fallback( - main_call=lambda: invoke_llm_with_timeout_message(primary_llm.ainvoke(messages)), - fallback_call=lambda: invoke_llm_with_timeout_message(fallback_llm.ainvoke(messages)), - ) - else: - raw = await invoke_llm_with_timeout_message(primary_llm.ainvoke(messages)) - - if isinstance(raw, ExtractionResult): - return raw - if isinstance(raw, dict): - return ExtractionResult.model_validate(raw) - return cast(ExtractionResult, raw) - except Exception as exc: - logger.error(f"Entity extraction turn failed: {exc}") - return ExtractionResult( - found=False, - output=None, - reason=f"LLM extraction error: {str(exc)[:400]}", - ) diff --git a/artemis/agents/hopper/hopper.md b/artemis/agents/hopper/hopper.md new file mode 100644 index 0000000..27c9d1a --- /dev/null +++ b/artemis/agents/hopper/hopper.md @@ -0,0 +1,15 @@ +## Hopper + +Extract relevant information from batch data. **Keep extracted data exactly as-is** - no reformatting. + +## Output +- **found**: `true` if data was found, `false` otherwise +- **output**: Extracted information if found, `null` otherwise +- **reason**: Brief explanation of search logic + +## Rules +1. **Search entire input** - may contain hundreds of entries +2. **For app package lookup**: Match app name (or variations) in package identifier + - Common patterns: lowercase app name, company+app, brand name, codenames +3. **Prefer direct matches** over partial matches +4. **Return `null`** if not found or if multiple ambiguous matches exist - don't guess diff --git a/artemis/agents/hopper/hopper.py b/artemis/agents/hopper/hopper.py new file mode 100644 index 0000000..0c1cb9f --- /dev/null +++ b/artemis/agents/hopper/hopper.py @@ -0,0 +1,93 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +from jinja2 import Template +from langchain_core.messages import HumanMessage, SystemMessage +from artemis.context import ArtemisContext +from artemis.data_engine.trace import trace +from artemis.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback +from artemis.utils.logger import get_logger +from pydantic import BaseModel, Field + +logger = get_logger(__name__) + + +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass + + +class HopperOutput(BaseModel): + model_config = {"ignored_types": (CyFunctionDetector,)} + found: bool = Field(description="True if the requested data was found, False otherwise.") + output: str | None = Field(description="The extracted data if found, null otherwise.") + reason: str = Field( + description="A short explanation of what you looked for" + + " and how you decided what to extract." + ) + + +@trace(type="agent", name="hopper") +async def hopper( + ctx: ArtemisContext, + request: str, + data: str, + use_fallback: bool = True, +) -> HopperOutput: + logger.info(f"Starting Hopper Agent (use_fallback={use_fallback})") + system_message = Template( + Path(__file__).parent.joinpath("hopper.md").read_text(encoding="utf-8") + ).render() + messages = [ + SystemMessage(content=system_message), + HumanMessage(content=f"{request}\nHere is the data you must dig:\n{data}"), + ] + + llm = get_llm(ctx=ctx, name="hopper", is_utils=True).with_structured_output(HopperOutput) + try: + if use_fallback: + llm_fallback = get_llm( + ctx=ctx, name="hopper", is_utils=True, use_fallback=True + ).with_structured_output(HopperOutput) + response: HopperOutput = await with_fallback( + main_call=lambda: invoke_llm_with_timeout_message(llm.ainvoke(messages)), + fallback_call=lambda: invoke_llm_with_timeout_message( + llm_fallback.ainvoke(messages) + ), + ) # type: ignore + else: + response: HopperOutput = await invoke_llm_with_timeout_message(llm.ainvoke(messages)) + return response + except Exception as e: + logger.error(f"Hopper LLM invocation failed: {e}") + return HopperOutput( + found=False, + output=None, + reason=f"Failed due to LLM error: {str(e)[:500]}", + ) diff --git a/artemis/agents/log_analyzer/log_analyzer.py b/artemis/agents/log_analyzer/log_analyzer.py index 76e4947..126b770 100644 --- a/artemis/agents/log_analyzer/log_analyzer.py +++ b/artemis/agents/log_analyzer/log_analyzer.py @@ -21,7 +21,26 @@ from langchain_core.tools import BaseTool, StructuredTool from pydantic import BaseModel, Field +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass + + class SpawnLogReaderArgs(BaseModel): + model_config = {"ignored_types": (CyFunctionDetector,)} specific_query: str = Field( ..., description=( @@ -35,6 +54,7 @@ from langgraph.prebuilt import InjectedState from artemis.core.tool_failure import ToolFailure, is_tool_failure from artemis.context import ArtemisContext from artemis.data_engine.trace import ( + CURRENT_TRACE_ID, TraceSpan, trace, ) diff --git a/artemis/agents/outputter/outputter.py b/artemis/agents/outputter/outputter.py index aed600d..b84f429 100644 --- a/artemis/agents/outputter/outputter.py +++ b/artemis/agents/outputter/outputter.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. import json from pathlib import Path diff --git a/artemis/agents/planner/planner.py b/artemis/agents/planner/planner.py index df23d56..9a46038 100644 --- a/artemis/agents/planner/planner.py +++ b/artemis/agents/planner/planner.py @@ -23,7 +23,7 @@ from pydantic import BaseModel, Field from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController -from artemis.data_engine.trace import TraceSpan, trace +from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace from artemis.graph.state import State from artemis.graph.visibility import strict_state from artemis.services.llm import ( @@ -108,7 +108,26 @@ def build_planner_system_blocks(prompts_data: dict, mode: str, include_checks: b return blocks +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass + + class ValidationResult(BaseModel): + model_config = {"ignored_types": (CyFunctionDetector,)} is_approved: bool = Field( description=( "True if the new plan still serves the initial goal and is coherent." diff --git a/artemis/clients/adb_tunnel.py b/artemis/clients/adb_tunnel.py new file mode 100644 index 0000000..2cd79b6 --- /dev/null +++ b/artemis/clients/adb_tunnel.py @@ -0,0 +1,327 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ADB WebSocket tunnel implementation. + +Bridges local TCP connections to a remote ADB server via WebSocket. +This replaces the need for the `lim` CLI for ADB tunneling. + +The tunnel runs in a dedicated thread with its own event loop to ensure +it cannot be blocked by synchronous operations in the main thread. + +Based on the Go SDK implementation which: +1. Creates a TCP listener +2. Accepts TCP connections from ADB +3. Dials the WebSocket +4. Forwards traffic bidirectionally +""" + +import asyncio +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +import socket +import threading + +from artemis.utils.logger import get_logger +import websockets +from websockets.asyncio.client import ClientConnection + +logger = get_logger(__name__) + +BUFFER_SIZE = 32 * 1024 # 32KB, same as Go SDK +PING_INTERVAL = 30 # seconds + + +class AdbTunnel: + """ADB tunnel that bridges local TCP connections to a remote WebSocket endpoint. + + Runs in a dedicated thread with its own event loop to ensure the tunnel + cannot be blocked by synchronous operations in the caller's thread. + + Usage: + tunnel = AdbTunnel(remote_url, token) + addr = await tunnel.start() # Returns "127.0.0.1:PORT" + # Use addr with ADB: adb connect 127.0.0.1:PORT + await tunnel.stop() + """ + + def __init__(self, remote_url: str, token: str): + """Initialize the ADB tunnel. + + Args: + remote_url: WebSocket URL for the remote ADB endpoint + token: Bearer token for authentication + """ + self.remote_url = remote_url + self.token = token + + self._local_port: int | None = None + self._thread: threading.Thread | None = None + self._loop: asyncio.AbstractEventLoop | None = None + self._started = threading.Event() + self._stop_event: asyncio.Event | None = None + + @property + def local_addr(self) -> str: + """Get the local address to connect ADB to.""" + if self._local_port is None: + raise RuntimeError("Tunnel not started") + return f"127.0.0.1:{self._local_port}" + + @property + def local_port(self) -> int: + """Get the local port number.""" + if self._local_port is None: + raise RuntimeError("Tunnel not started") + return self._local_port + + async def start(self) -> str: + """Start the ADB tunnel in a dedicated thread. + + Returns: + The local address (host:port) to connect ADB to. + """ + if self._thread is not None and self._thread.is_alive(): + return self.local_addr + + # Create listener in main thread to get the port immediately + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", 0)) + listener.listen(5) + listener.setblocking(False) + self._local_port = listener.getsockname()[1] + + logger.info(f"ADB tunnel listening on {self.local_addr}") + + # Start tunnel thread + self._thread = threading.Thread( + target=self._run_in_thread, + args=(listener,), + name=f"adb-tunnel-{self._local_port}", + daemon=True, + ) + self._thread.start() + + # Wait for thread to initialize its event loop + self._started.wait(timeout=5.0) + if not self._started.is_set(): + await self.stop() + raise RuntimeError("Tunnel thread failed to start") + + return self.local_addr + + async def stop(self) -> None: + """Stop the ADB tunnel and cleanup resources.""" + if self._loop and self._stop_event: + # Signal the tunnel loop to stop + self._loop.call_soon_threadsafe(self._stop_event.set) + + if self._thread and self._thread.is_alive(): + await asyncio.to_thread(self._thread.join, 5.0) + if self._thread.is_alive(): + logger.warning("ADB tunnel thread did not stop within 5s") + + self._thread = None + self._loop = None + self._stop_event = None + logger.info("ADB tunnel stopped") + + def _run_in_thread(self, listener: socket.socket) -> None: + """Run the tunnel event loop in a dedicated thread.""" + try: + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + self._stop_event = asyncio.Event() + self._started.set() + + self._loop.run_until_complete(self._run_tunnel(listener)) + except Exception as e: + logger.error(f"Tunnel thread error: {e}") + finally: + listener.close() + if self._loop: + self._loop.close() + + async def _run_tunnel(self, listener: socket.socket) -> None: + """Run the tunnel - accept TCP connections and bridge to WebSocket.""" + loop = asyncio.get_event_loop() + connection_tasks: list[asyncio.Task] = [] + + stop_event = self._stop_event + assert stop_event is not None + + logger.debug("Tunnel: ready to accept connections") + + while not stop_event.is_set(): + try: + # Use wait_for to periodically check stop_event + try: + tcp_conn, addr = await asyncio.wait_for( + loop.sock_accept(listener), + timeout=1.0, + ) + except TimeoutError: + # Check if we should stop + continue + + logger.debug(f"Tunnel: new TCP connection from {addr}") + + task = asyncio.create_task( + self._handle_connection(tcp_conn, loop), + name=f"conn-{addr[1]}", + ) + task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None) + connection_tasks.append(task) + connection_tasks = [t for t in connection_tasks if not t.done()] + + except Exception as e: + if not stop_event.is_set(): + logger.error(f"Tunnel accept error: {e}") + break + + # Cleanup pending connections + for task in connection_tasks: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + async def _handle_connection( + self, + tcp_conn: socket.socket, + loop: asyncio.AbstractEventLoop, + ) -> None: + """Handle a single TCP connection.""" + tcp_conn.setblocking(False) + + ws: ClientConnection | None = None + try: + logger.debug(f"Connecting to WebSocket: {self.remote_url[:80]}...") + ws = await websockets.connect( + self.remote_url, + additional_headers={"Authorization": f"Bearer {self.token}"}, + ping_interval=PING_INTERVAL, + ping_timeout=10, + ) + logger.debug("WebSocket connected to remote ADB") + await self._bridge(tcp_conn, ws, loop) + except Exception as e: + logger.error(f"Connection error: {e}") + finally: + if ws is not None: + await ws.close() + tcp_conn.close() + logger.debug("Connection closed") + + async def _bridge( + self, + tcp_conn: socket.socket, + ws: ClientConnection, + loop: asyncio.AbstractEventLoop, + ) -> None: + """Bridge TCP and WebSocket bidirectionally.""" + + stop_event = self._stop_event + assert stop_event is not None + + async def tcp_to_ws() -> None: + """Forward data from TCP to WebSocket.""" + msg_count = 0 + try: + while not stop_event.is_set(): + try: + data = await asyncio.wait_for( + loop.sock_recv(tcp_conn, BUFFER_SIZE), + timeout=1.0, + ) + except TimeoutError: + continue + except (OSError, ConnectionError) as e: + logger.debug(f"tcp->ws: recv error after {msg_count} msgs: {e}") + break + if not data: + logger.debug(f"tcp->ws: closed by client after {msg_count} msgs") + break + msg_count += 1 + logger.debug(f"tcp->ws: [{msg_count}] sending {len(data)} bytes") + await ws.send(data) + logger.debug(f"tcp->ws: loop ended after {msg_count} messages") + except asyncio.CancelledError: + logger.debug(f"tcp->ws: cancelled after {msg_count} messages") + raise + except Exception as e: + logger.error(f"tcp->ws: error after {msg_count} messages: {e}") + + async def ws_to_tcp() -> None: + """Forward data from WebSocket to TCP.""" + msg_count = 0 + try: + logger.debug("ws->tcp: starting message loop") + async for message in ws: + if stop_event.is_set(): + break + if isinstance(message, bytes) and message: + msg_count += 1 + logger.debug(f"ws->tcp: [{msg_count}] received {len(message)} bytes") + await loop.sock_sendall(tcp_conn, message) + elif isinstance(message, str): + logger.debug(f"ws->tcp: received string: {message[:100]}") + logger.debug(f"ws->tcp: loop ended after {msg_count} messages") + except asyncio.CancelledError: + logger.debug(f"ws->tcp: cancelled after {msg_count} messages") + raise + except Exception as e: + logger.error(f"ws->tcp: error after {msg_count} messages: {e}") + + tcp_task = asyncio.create_task(tcp_to_ws()) + ws_task = asyncio.create_task(ws_to_tcp()) + + done, pending = await asyncio.wait( + [tcp_task, ws_task], + return_when=asyncio.FIRST_COMPLETED, + ) + + for task in pending: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + +@asynccontextmanager +async def adb_tunnel(remote_url: str, token: str) -> AsyncGenerator[AdbTunnel, None]: + """Context manager for creating and managing an ADB tunnel. + + Args: + remote_url: WebSocket URL for the remote ADB endpoint + token: Bearer token for authentication + + Yields: + AdbTunnel instance with the tunnel running + + Example: + async with adb_tunnel(ws_url, token) as tunnel: + # Connect ADB to tunnel.local_addr + adb_client = AdbClient() + adb_client.connect(tunnel.local_addr) + """ + tunnel = AdbTunnel(remote_url, token) + try: + await tunnel.start() + yield tunnel + finally: + await tunnel.stop() diff --git a/artemis/clients/ui_automator_client.py b/artemis/clients/ui_automator_client.py index b6b5b41..34c1d21 100644 --- a/artemis/clients/ui_automator_client.py +++ b/artemis/clients/ui_automator_client.py @@ -11,13 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. """UIAutomator2 client for Android device screen data retrieval. -Provides direct device access for hierarchy and screenshot retrieval. +Provides an alternative to the Maestro-based screen API with direct device +access. +Handles Maestro blocker detection and removal before connecting. """ import base64 @@ -42,9 +41,32 @@ if TYPE_CHECKING: logger = get_logger(__name__) +MAESTRO_PACKAGE = "dev.mobile.maestro" + + +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass + + class UIAutomatorScreenData(BaseModel): """Screen data response from UIAutomator2.""" + model_config = {"ignored_types": (CyFunctionDetector,)} + base64: str hierarchy_xml: str elements: list[dict] @@ -148,11 +170,88 @@ def _pil_to_base64(img: Image.Image, format: str = "JPEG", quality: int = 80) -> return base64.b64encode(buffer.getvalue()).decode("utf-8") +def _is_package_installed(device_id: str, pkg: str) -> bool: + """Check if a package is installed on the device.""" + try: + result = subprocess.run( + adb_command(["-s", device_id, "shell", "pm", "list", "packages"]), + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode != 0: + logger.warning(f"Failed to list packages: {result.stderr}") + return False + + lines = result.stdout.splitlines() + target = f"package:{pkg}" + return target in lines + except subprocess.TimeoutExpired: + logger.warning("Timeout checking installed packages") + return False + except (OSError, subprocess.SubprocessError) as e: + logger.warning(f"Error checking installed packages: {e}") + return False + + +def _uninstall_package(device_id: str, pkg: str) -> bool: + """Uninstall a package from the device for the current user.""" + try: + result = subprocess.run( + adb_command( + [ + "-s", + device_id, + "shell", + "pm", + "uninstall", + "--user", + "0", + pkg, + ] + ), + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=30, + ) + if result.returncode == 0: + logger.info(f"Successfully uninstalled {pkg}") + return True + else: + logger.warning(f"Failed to uninstall {pkg}: {result.stderr}") + return False + except subprocess.TimeoutExpired: + logger.warning(f"Timeout uninstalling {pkg}") + return False + except Exception as e: + logger.warning(f"Error uninstalling {pkg}: {e}") + return False + + +def _ensure_maestro_not_installed(device_id: str) -> None: + """Check if Maestro is installed and uninstall it. + + Maestro conflicts with uiautomator2 - if Maestro's package is installed, + u2.connect() will fail. This function ensures Maestro is removed before + attempting to connect. + """ + if _is_package_installed(device_id, MAESTRO_PACKAGE): + logger.warning( + f"Maestro ({MAESTRO_PACKAGE}) detected - uninstalling to enable UIAutomator2..." + ) + _uninstall_package(device_id, MAESTRO_PACKAGE) + + class UIAutomatorClient: """UIAutomator2 client for Android screen data retrieval. - This client uses the uiautomator2 library for direct device communication, - providing high-performance hierarchy and screenshot retrieval. + This client uses uiautomator2 library for direct device communication, + providing faster hierarchy and screenshot retrieval compared to Maestro. + + Important: Maestro must not be installed on the device as it conflicts + with uiautomator2. This client automatically handles Maestro removal. """ def __init__(self, device_id: str): @@ -166,7 +265,7 @@ class UIAutomatorClient: self._awake_strategy: str | None = None def _ensure_connected(self) -> "Device": - """Ensure connection to the device. + """Ensure connection to the device, handling Maestro blocker. Returns: Connected uiautomator2 Device instance @@ -180,6 +279,9 @@ class UIAutomatorClient: logger.warning("UIAutomator2 connection lost, reconnecting...") self._device = None + # Ensure Maestro is not blocking us + _ensure_maestro_not_installed(self._device_id) + # Enroll the device in the containing Artemis service lifetime before # UIAutomator2 reads the screen. Client disconnects do not release it. if self._awake_strategy is None: diff --git a/artemis/config/__init__.py b/artemis/config/__init__.py index d28428a..e539f93 100644 --- a/artemis/config/__init__.py +++ b/artemis/config/__init__.py @@ -83,7 +83,6 @@ from artemis.config.constants import ( ENV_ARTEMIS_TRACES_DIR, ENV_ARTEMIS_USE_FILE_API, ENV_ARTEMIS_USE_USER_DIR, - ENV_OPENROUTER_API_KEY, ENV_OPEN_ROUTER_API_KEY, ENV_OPENAI_API_KEY, ENV_OPENAI_BASE_URL, @@ -101,6 +100,7 @@ from artemis.config.constants import ( LLMUtilsNodeWithFallback, ) from artemis.config.llm import ( + CyFunctionDetector, LLM, LLMConfig, LLMConfigUtils, @@ -196,6 +196,7 @@ __all__ = [ "get_ls_address_file", "is_frozen_bundle", # LLM Config & Models + "CyFunctionDetector", "LLM", "LLMWithFallback", "LLMConfigUtils", @@ -295,7 +296,6 @@ __all__ = [ "ENV_OPENAI_API_KEY", "ENV_OPENAI_BASE_URL", "ENV_ANTHROPIC_API_KEY", - "ENV_OPENROUTER_API_KEY", "ENV_OPEN_ROUTER_API_KEY", "ENV_XAI_API_KEY", "ENV_ARTEMIS_EXPLORER_VERSION", diff --git a/artemis/config/constants.py b/artemis/config/constants.py index 369c43e..972bc6d 100644 --- a/artemis/config/constants.py +++ b/artemis/config/constants.py @@ -62,7 +62,6 @@ ENV_API_KEY = "API_KEY" ENV_OPENAI_API_KEY = "OPENAI_API_KEY" ENV_OPENAI_BASE_URL = "OPENAI_BASE_URL" ENV_ANTHROPIC_API_KEY = "ANTHROPIC_API_KEY" -ENV_OPENROUTER_API_KEY = "OPENROUTER_API_KEY" ENV_OPEN_ROUTER_API_KEY = "OPEN_ROUTER_API_KEY" ENV_XAI_API_KEY = "XAI_API_KEY" @@ -126,7 +125,7 @@ ExplorerVersion = Literal["flash", "pro", "ultra"] LLMUtilsNode = Literal[ "outputter", - "entity_extractor", + "hopper", "video_analyzer", "object_detector", ] diff --git a/artemis/config/llm.py b/artemis/config/llm.py index 5fc6062..36915dd 100644 --- a/artemis/config/llm.py +++ b/artemis/config/llm.py @@ -48,11 +48,30 @@ def validate_vertex_ai_credentials() -> None: ) +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + """Utility type detector for Cython and C-extension functions in Pydantic.""" + pass class LLM(BaseModel): """Base model representing an LLM model provider and runtime parameters.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} provider: LLMProvider model: str temperature: float | None = None @@ -76,8 +95,8 @@ class LLM(BaseModel): if not (settings.ANTHROPIC_API_KEY or os.environ.get("ANTHROPIC_API_KEY")): raise Exception(f"{name} requires ANTHROPIC_API_KEY in .env") elif self.provider == "openrouter": - if not (settings.OPENROUTER_API_KEY or settings.OPEN_ROUTER_API_KEY): - raise Exception(f"{name} requires OPENROUTER_API_KEY in .env") + if not settings.OPEN_ROUTER_API_KEY: + raise Exception(f"{name} requires OPEN_ROUTER_API_KEY in .env") elif self.provider == "xai": if not settings.XAI_API_KEY: raise Exception(f"{name} requires XAI_API_KEY in .env") @@ -114,14 +133,18 @@ def lightweight_judge_default() -> "LLMWithFallback": class LLMConfigUtils(BaseModel): """Configuration container for auxiliary utility agents/nodes.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} outputter: LLMWithFallback - entity_extractor: LLMWithFallback + hopper: LLMWithFallback video_analyzer: LLMWithFallback | None = None object_detector: LLMWithFallback | None = None class LLMConfig(BaseModel): """Comprehensive LLM configuration mapping every node to primary/fallback models.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} planner: LLMWithFallback utils: LLMConfigUtils summarizer: LLMWithFallback @@ -144,7 +167,7 @@ class LLMConfig(BaseModel): """Validate credentials across all configured agent nodes.""" self.planner.validate_provider("Planner") self.utils.outputter.validate_provider("Outputter") - self.utils.entity_extractor.validate_provider("EntityExtractor") + self.utils.hopper.validate_provider("Hopper") if self.utils.video_analyzer: self.utils.video_analyzer.validate_provider("VideoAnalyzer") if self.utils.object_detector: @@ -173,7 +196,7 @@ class LLMConfig(BaseModel): return f""" 📃 Planner: {self.planner} 🧩 Utils: - 🔍 Entity Extractor: {self.utils.entity_extractor} + 🔽 Hopper: {self.utils.hopper} 📝 Outputter: {self.utils.outputter} 🎬 Video Analyzer: {self.utils.video_analyzer or "Not configured"} 👁️ Object Detector: {self.utils.object_detector or "Not configured"} @@ -196,7 +219,7 @@ class LLMConfig(BaseModel): def get_utils(self, item: LLMUtilsNode) -> LLMWithFallback: """Retrieve model configuration for a specific utility node.""" - value = getattr(self.utils, item, None) + value = getattr(self.utils, item) if value is None: raise ValueError( f"Utils '{item}' is not configured. Please add it to your LLM " @@ -241,7 +264,7 @@ def _expand_default_into_nodes(config_dict: dict) -> dict: all_utils_nodes = [ "outputter", - "entity_extractor", + "hopper", "video_analyzer", "object_detector", ] diff --git a/artemis/config/output.py b/artemis/config/output.py index 02967ae..1955b9a 100644 --- a/artemis/config/output.py +++ b/artemis/config/output.py @@ -26,6 +26,7 @@ from artemis.config.constants import ( ENV_EVENTS_OUTPUT_PATH, ENV_RESULTS_OUTPUT_PATH, ) +from artemis.config.llm import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -33,6 +34,8 @@ logger = get_logger(__name__) class OutputConfig(BaseModel): """Configuration for structured output formatting and schemas.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} structured_output: Annotated[ type[BaseModel] | dict | None, Field( diff --git a/artemis/config/settings.py b/artemis/config/settings.py index 0856a06..7129dd8 100644 --- a/artemis/config/settings.py +++ b/artemis/config/settings.py @@ -18,7 +18,7 @@ import os from pathlib import Path from dotenv import load_dotenv -from pydantic import AliasChoices, Field, SecretStr, model_validator +from pydantic import Field, SecretStr, model_validator from pydantic_settings import BaseSettings from artemis.config.constants import ( @@ -32,7 +32,6 @@ from artemis.config.constants import ( ENV_GEMINI_API_KEY, ENV_GOOGLE_API_KEY, ENV_OCR_API_KEY, - ENV_OPENROUTER_API_KEY, ENV_OPEN_ROUTER_API_KEY, ENV_OPENAI_API_KEY, ENV_VISION_API_KEY, @@ -100,10 +99,6 @@ class Settings(BaseSettings): GCP_API_KEY: SecretStr | None = None ANTHROPIC_API_KEY: SecretStr | None = None XAI_API_KEY: SecretStr | None = None - OPENROUTER_API_KEY: SecretStr | None = Field( - default=None, - validation_alias=AliasChoices("OPENROUTER_API_KEY", "OPEN_ROUTER_API_KEY"), - ) OPEN_ROUTER_API_KEY: SecretStr | None = None # Google Cloud Vision OCR Authentication @@ -168,7 +163,6 @@ class Settings(BaseSettings): "GCP_API_KEY", "ANTHROPIC_API_KEY", "XAI_API_KEY", - "OPENROUTER_API_KEY", "OPEN_ROUTER_API_KEY", "OCR_API_KEY", "VISION_API_KEY", @@ -178,12 +172,6 @@ class Settings(BaseSettings): if val and is_placeholder_key(val): setattr(self, attr, None) - # Synchronize OpenRouter keys - if not self.OPENROUTER_API_KEY and self.OPEN_ROUTER_API_KEY: - self.OPENROUTER_API_KEY = self.OPEN_ROUTER_API_KEY - elif not self.OPEN_ROUTER_API_KEY and self.OPENROUTER_API_KEY: - self.OPEN_ROUTER_API_KEY = self.OPENROUTER_API_KEY - if not self.GOOGLE_API_KEY: if self.GEMINI_API_KEY: self.GOOGLE_API_KEY = self.GEMINI_API_KEY @@ -216,8 +204,8 @@ class Settings(BaseSettings): key = self.OPENAI_API_KEY elif provider_lower in ("anthropic", "claude"): key = self.ANTHROPIC_API_KEY - elif provider_lower in ("openrouter", "open_router"): - key = self.OPENROUTER_API_KEY or self.OPEN_ROUTER_API_KEY + elif provider_lower == "openrouter": + key = self.OPEN_ROUTER_API_KEY elif provider_lower in ("xai", "grok"): key = self.XAI_API_KEY @@ -255,11 +243,9 @@ class Settings(BaseSettings): self.ANTHROPIC_API_KEY = secret env_key_name = ENV_ANTHROPIC_API_KEY os.environ[ENV_ANTHROPIC_API_KEY] = key - elif provider_lower in ("openrouter", "open_router"): - self.OPENROUTER_API_KEY = secret + elif provider_lower == "openrouter": self.OPEN_ROUTER_API_KEY = secret - env_key_name = ENV_OPENROUTER_API_KEY - os.environ[ENV_OPENROUTER_API_KEY] = key + env_key_name = ENV_OPEN_ROUTER_API_KEY os.environ[ENV_OPEN_ROUTER_API_KEY] = key elif provider_lower == "xai": self.XAI_API_KEY = secret diff --git a/artemis/context.py b/artemis/context.py index fa698db..9e51ed7 100644 --- a/artemis/context.py +++ b/artemis/context.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. """Context variables for global state management. diff --git a/artemis/controllers/device_controller.py b/artemis/controllers/device_controller.py index 32723da..d5c2457 100644 --- a/artemis/controllers/device_controller.py +++ b/artemis/controllers/device_controller.py @@ -11,42 +11,45 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. - -"""Abstract device controller contract and telemetry interfaces for ARTEMIS. - -Defines the hardware interaction lifecycle including touch gestures, text entry, -process lifecycle operations, display capture, and multi-segment recording. -""" - -from __future__ import annotations from abc import ABC, abstractmethod from typing import Any -from pydantic import BaseModel, ConfigDict, Field - from artemis.controllers.types import Bounds, CoordinatesSelectorRequest, TapOutput from artemis.utils.video import VideoRecordingResult +from pydantic import BaseModel, Field + + +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass class ScreenDataResponse(BaseModel): - """Aggregate snapshot containing live display frames and parsed node hierarchies.""" - - model_config = ConfigDict(extra="ignore") - - base64: str = Field(default="", description="Base64 encoded JPEG/PNG frame capture.") - elements: list[dict[str, Any]] = Field(default_factory=list, description="Parsed hierarchy node elements.") - width: int = Field(default=1080, description="Active viewport width in physical pixels.") - height: int = Field(default=2400, description="Active viewport height in physical pixels.") - platform: str = Field(default="android", description="Target platform identifier.") + model_config = {"ignored_types": (CyFunctionDetector,)} + base64: str = Field(default="", description="Base64 encoded screenshot string") + elements: list[dict[str, Any]] = Field( + default_factory=list, description="Parsed UI hierarchy elements" + ) + width: int = Field(default=1080, description="Device screen width in pixels") + height: int = Field(default=2400, description="Device screen height in pixels") + platform: str = Field(default="android", description="Platform identifier") class MobileDeviceController(ABC): - """Abstract driver interface establishing device actuation and telemetry operations.""" - @abstractmethod async def tap( self, @@ -56,8 +59,7 @@ class MobileDeviceController(ABC): times: int = 1, delay_ms: int = 100, ) -> TapOutput: - """Dispatch single, repeated, or long-press tap events to the target coordinates.""" - raise NotImplementedError + raise NotImplementedError("Subclasses must implement this method") @abstractmethod async def swipe( @@ -66,102 +68,161 @@ class MobileDeviceController(ABC): end: CoordinatesSelectorRequest, duration: int = 400, ) -> str | None: - """Execute directional drag or fling between two coordinate positions.""" - raise NotImplementedError + """Swipe from start to end coordinates. + + Returns error message on failure, None on success. + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod async def screenshot(self) -> str: - """Capture current display frame as an encoded image payload.""" - raise NotImplementedError + """Take a screenshot and return raw image data.""" + raise NotImplementedError("Subclasses must implement this method") @abstractmethod async def input_text(self, text: str) -> bool: - """Stream textual characters into the currently active input field.""" - raise NotImplementedError + """Input text at the currently focused field. + + Returns True on success, False on failure. + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod - async def launch_app(self, package_name: str) -> bool: - """Bring target Android application package into foreground.""" - raise NotImplementedError + async def launch_app(self, package_or_bundle_id: str) -> bool: + """Launch an application by package name (Android). + + Returns True on success, False on failure. + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod - async def terminate_app(self, package_name: str | None) -> bool: - """Force-stop application process matching the target package.""" - raise NotImplementedError + async def terminate_app(self, package_or_bundle_id: str | None) -> bool: + """Terminate an application. + + Returns True on success, False on failure. + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod async def open_url(self, url: str) -> bool: - """Dispatch system VIEW intent for the specified URI.""" - raise NotImplementedError + """Open a URL. + + Returns True on success, False on failure. + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod async def press_back(self) -> bool: - """Trigger Android BACK system navigation key event.""" - raise NotImplementedError + """Press the back button (Android). + + Returns True on success, False on failure. + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod async def press_home(self) -> bool: - """Trigger Android HOME system navigation key event.""" - raise NotImplementedError + """Press the home button. + + Returns True on success, False on failure. + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod async def press_enter(self) -> bool: - """Trigger ENTER / ACTION_DONE key event on virtual keyboard.""" - raise NotImplementedError + """Press the enter/return key. + + Returns True on success, False on failure. + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod async def press_key(self, keycode: str) -> bool: - """Dispatch raw Android keycode event (e.g., 'KEYCODE_TAB').""" - raise NotImplementedError + """Press a specific key event. + + Returns True on success, False on failure. + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod - async def get_ui_hierarchy(self) -> list[dict[str, Any]]: - """Retrieve flattened or nested accessibility tree node descriptors.""" - raise NotImplementedError + async def get_ui_hierarchy(self) -> list[dict]: + """Get the UI element hierarchy. + + Returns a list of UI elements with their properties. + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod def find_element( self, - ui_hierarchy: list[dict[str, Any]], + ui_hierarchy: list[dict], resource_id: str | None = None, text: str | None = None, index: int = 0, - ) -> tuple[dict[str, Any] | None, Bounds | None, str | None]: - """Locate target element within parsed hierarchy by ID or label matching.""" - raise NotImplementedError + ) -> tuple[dict | None, Bounds | None, str | None]: + """Find a UI element in the hierarchy. + + Returns: + Tuple of (element_dict, bounds, error_message) + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod async def cleanup(self) -> None: - """Tear down companion sessions, tunnels, and streaming handles.""" - raise NotImplementedError + """Clean up resources (e.g., stop companion processes).""" + raise NotImplementedError("Subclasses must implement this method") @abstractmethod async def erase_text(self, nb_chars: int | None = None) -> bool: - """Dispatch backspace events to clear trailing characters in focused field.""" - raise NotImplementedError + """Erase the last nb_chars characters. + + Returns True on success, False on failure. + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod - async def get_screen_data(self) -> ScreenDataResponse: - """Capture unified screen frame, layout tree, and viewport dimensions.""" - raise NotImplementedError + async def get_screen_data(self) -> "ScreenDataResponse": + """Get screen data including screenshot (base64), UI hierarchy elements, + + screen dimensions, and platform. + + Returns: + ScreenDataResponse with base64 screenshot, elements, width, height, + platform + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod def get_compressed_b64_screenshot(self, image_base64: str, quality: int = 50) -> str: - """Produce compressed image payload for bandwidth-constrained turns.""" - raise NotImplementedError + """Compress a base64 image. + + Returns the compressed base64 image. + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod async def start_video_recording( self, max_duration_seconds: int = 900, ) -> VideoRecordingResult: - """Initialize session screen capture recording.""" - raise NotImplementedError + """Start screen recording on the device. + + Args: + max_duration_seconds: Maximum recording duration in seconds. + + Returns: + VideoRecordingResult with success status and message. + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod async def stop_video_recording(self) -> VideoRecordingResult: - """Terminate active screen capture and finalize video artifact.""" - raise NotImplementedError + """Stop screen recording and retrieve the video file. + + Returns: + VideoRecordingResult with success status, message, and video_path if + successful. + """ + raise NotImplementedError("Subclasses must implement this method") @abstractmethod async def extract_segment_metadata( @@ -169,5 +230,16 @@ class MobileDeviceController(ABC): start_relative_time: float, end_relative_time: float | None = None, ) -> VideoRecordingResult: - """Slice video segment for the requested relative execution epoch.""" - raise NotImplementedError + """Get a video segment for a specific time range. + + Args: + start_relative_time: Start time in seconds relative to session + start. + end_relative_time: End time in seconds relative to session start, or + None for latest. + + Returns: + VideoRecordingResult with success status, message, and video_path if + successful. + """ + raise NotImplementedError("Subclasses must implement this method") diff --git a/artemis/controllers/platform_specific_commands_controller.py b/artemis/controllers/platform_specific_commands_controller.py index e544899..d32832d 100644 --- a/artemis/controllers/platform_specific_commands_controller.py +++ b/artemis/controllers/platform_specific_commands_controller.py @@ -11,18 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. import os from shutil import which -import subprocess import time from typing import Any from artemis.context import ArtemisContext, DevicePlatform from artemis.utils.logger import ArtemisLogger, get_logger +from artemis.utils.shell_utils import run_shell_command_on_host logger = get_logger(__name__) @@ -60,17 +57,12 @@ def get_first_device( if which("adb"): try: - res = subprocess.run( - ["adb", "devices"], - capture_output=True, - text=True, - check=True, - stdin=subprocess.DEVNULL, - ) - for line in res.stdout.strip().split("\n"): + android_output = run_shell_command_on_host("adb devices") + lines = android_output.strip().split("\n") + for line in lines: if "device" in line and not line.startswith("List of devices"): return line.split()[0], DevicePlatform.ANDROID, None - except Exception as e: + except RuntimeError as e: if logger: logger.error(f"ADB command failed: {e}") diff --git a/artemis/controllers/unified_controller.py b/artemis/controllers/unified_controller.py index 201abfe..56e57b8 100644 --- a/artemis/controllers/unified_controller.py +++ b/artemis/controllers/unified_controller.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. import asyncio import os diff --git a/artemis/graph/graph.py b/artemis/graph/graph.py index 5b03467..ab854a1 100644 --- a/artemis/graph/graph.py +++ b/artemis/graph/graph.py @@ -511,6 +511,24 @@ def _get_active_subgoal_hashes(ctx: ArtemisContext) -> tuple[str, str | None]: return "default", None +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass + + def check_plan_mutation_rejections( content_before: str, content_after: str, state: State | None = None ) -> str | None: @@ -596,6 +614,7 @@ def _reject_checker_note_write(key: str, tool_call_id: str | None) -> ToolMessag class NoteArgs(BaseModel): + model_config = {"ignored_types": (CyFunctionDetector,)} key: str = Field(..., description=SAVE_NOTE_ARG_KEY_DESC) content: str = Field(..., description=SAVE_NOTE_ARG_CONTENT_DESC) @@ -833,6 +852,7 @@ def wrap_note_tool(ctx: ArtemisContext, original_tool): class UpdateNoteArgs(BaseModel): + model_config = {"ignored_types": (CyFunctionDetector,)} key: str = Field(..., description=UPDATE_NOTE_ARG_KEY_DESC) target: str = Field(..., description=UPDATE_NOTE_ARG_TARGET_DESC) replacement: str = Field(..., description=UPDATE_NOTE_ARG_REPLACEMENT_DESC) diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 4649ceb..91a5d04 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -26,6 +26,7 @@ from artemis.config import checker_overrides_for_level, initialize_llm_config, s from artemis.utils.startup_progress import publish_startup_progress from artemis import Agent, Builders from artemis.sdk.types.task import AgentProfile +from artemis.utils.cli_helpers import display_device_status from artemis.utils.logger import get_logger from artemis.utils.video import check_ffmpeg_available import signal @@ -36,23 +37,6 @@ import typer logger = get_logger(__name__) -def _display_device_status(console: Console, adb_client: AdbClient | None = None) -> None: - """Checks for connected devices and displays the status.""" - console.print("\n[bold]📱 Device Status[/bold]") - devices = None - if adb_client is not None: - try: - devices = adb_client.device_list() - except Exception: - devices = None - if devices: - console.print("✅ [bold green]Android device(s) connected:[/bold green]") - for device in devices: - console.print(f" - {device.serial}") - else: - console.print("❌ [bold red]No Android device found.[/bold red]") - - async def execute_task( goal: str, device_serial: str | None = None, @@ -467,7 +451,7 @@ def run_command( # Optional cosmetic device-status display; run continues without it. logger.debug(f"Could not create ADB client for device status display: {exc}") - _display_device_status(console, adb_client=adb_client) + display_device_status(console, adb_client=adb_client) cancelled = False original_sigterm = None diff --git a/artemis/resources/config/artemis.jsonc b/artemis/resources/config/artemis.jsonc index 98ce44b..517d738 100644 --- a/artemis/resources/config/artemis.jsonc +++ b/artemis/resources/config/artemis.jsonc @@ -93,8 +93,8 @@ // 📋 Summarizer: Aggregates step history and generates structured outcome "summarizer": {}, - // 🔍 Entity Extractor: Fast lightweight package name locator and entity extractor - "entity_extractor": { + // 🔽 Hopper: Fast lightweight package name locator + "hopper": { "model": "gemini-3.5-flash-lite", "fallback": { "provider": "google", diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py index dee930e..389e677 100644 --- a/artemis/sdk/agent.py +++ b/artemis/sdk/agent.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. import asyncio import inspect @@ -140,6 +137,7 @@ class Agent: _current_task: asyncio.Task | None = None _task_lock: asyncio.Lock + _cloud_mobile_id: str | None = None def __init__( self, diff --git a/artemis/sdk/builders/agent_config_builder.py b/artemis/sdk/builders/agent_config_builder.py index d4684fb..4786005 100644 --- a/artemis/sdk/builders/agent_config_builder.py +++ b/artemis/sdk/builders/agent_config_builder.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. """Builder for AgentConfig objects using a fluent interface.""" @@ -72,6 +69,7 @@ class AgentConfigBuilder: self._video_recording_tools_enabled: bool = detect_video_tools_enabled() self._force_web_accessibility: bool = False self._disable_checker: bool = False + self._cloud_mobile_id_or_ref: str | None = None self._concurrency_mode: str = "per_device" self._max_concurrency: int | None = None @@ -154,6 +152,12 @@ class AgentConfigBuilder: platform_or_device_id: DevicePlatform or unique identifier for the device device_id: The unique identifier for the device (if platform was passed first) """ + if self._cloud_mobile_id_or_ref is not None: + raise ValueError( + "Device ID cannot be set when a cloud mobile is already" + " configured.\n> for_device() and for_cloud_mobile() are" + " mutually exclusive" + ) if isinstance(platform_or_device_id, DevicePlatform): self._device_platform = platform_or_device_id self._device_id = device_id diff --git a/artemis/sdk/builders/task_request_builder.py b/artemis/sdk/builders/task_request_builder.py index c352752..320c9e5 100644 --- a/artemis/sdk/builders/task_request_builder.py +++ b/artemis/sdk/builders/task_request_builder.py @@ -11,71 +11,123 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. -"""Fluent specification builders for ARTEMIS automation task requests. - -Provides type-safe configuration chaining for task objectives, execution bounds, -device sandboxing constraints, and structured schema validations. -""" - -from __future__ import annotations +"""Builder for TaskRequest objects using a fluent interface.""" from pathlib import Path -from typing import Generic, Self, TypeVar, cast +from typing import Generic, TypeVar, cast -from pydantic import BaseModel +try: + from typing import Self +except ImportError: + from typing import Self from artemis.constants import RECURSION_LIMIT from artemis.sdk.types.agent import AgentProfile from artemis.sdk.types.task import TaskRequest, TaskRequestCommon +from pydantic import BaseModel TIn = TypeVar("TIn", bound=BaseModel | None) TOut = TypeVar("TOut", bound=BaseModel) -class TaskRequestCommonBuilder: - """Configurator for common execution bounds and sandbox constraints.""" +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) - def __init__(self) -> None: - self._max_steps: int = RECURSION_LIMIT - self._record_trace: bool = True - self._trace_path: Path = Path("traces") + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass + + +class TaskRequestCommonBuilder(BaseModel): + """Builder class providing a fluent interface for creating TaskRequestCommon objects.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + + def __init__(self): + self._max_steps = RECURSION_LIMIT + self._record_trace = True + self._trace_path = Path("traces") self._llm_output_path: Path | None = None self._locked_app_package: str | None = None self._app_path: Path | None = None def with_max_steps(self, max_steps: int) -> Self: - """Constrain the maximum interaction turns allowed.""" + """Set the maximum number of steps the task can take. + + Args: + max_steps: Maximum number of steps + """ self._max_steps = max_steps return self - def with_trace_recording(self, enabled: bool = True, path: str | Path | None = None) -> Self: - """Toggle telemetry snapshot capture and set output storage directory.""" + def with_trace_recording(self, enabled: bool = True, path: str | None = None) -> Self: + """Configure trace recording for the task. + + Traces record screenshots and actions during execution. + + Args: + enabled: Whether to enable trace recording + path: Directory path where traces should be saved + """ self._record_trace = enabled if enabled and path: self._trace_path = Path(path) return self - def with_llm_output_saving(self, path: str | Path) -> Self: - """Designate destination path for persisting raw reasoning payloads.""" + def with_llm_output_saving(self, path: str) -> Self: + """Configure LLM output saving for the task. + + Args: + path: Path where to save the LLM output message + """ self._llm_output_path = Path(path) return self def with_locked_app_package(self, package_name: str) -> Self: - """Bind test execution strictly to the target Android package namespace.""" - self._locked_app_package = package_name.strip() + """Set the app package to lock execution to. + + This ensures the specified app is launched and in the foreground before + the agentic loop starts. + + Args: + package_name: Package name (Android, e.g., 'com.whatsapp') + """ + self._locked_app_package = package_name return self def with_app_path(self, app_path: str | Path) -> Self: - """Provide path to local APK for automatic installation prior to run.""" - self._app_path = Path(app_path) + """Set the path to an app to install before running the task. + + For Android: Path to an APK file. + + The app will be installed automatically before the task starts. + + Args: + app_path: Path to the app file to install + """ + self._app_path = Path(app_path) if isinstance(app_path, str) else app_path return self def build(self) -> TaskRequestCommon: - """Materialize configured constraints into a TaskRequestCommon contract.""" + """Build the TaskRequestCommon object. + + Returns: + A configured TaskRequestCommon object + + Raises: + ValueError: If required fields are missing + """ return TaskRequestCommon( max_steps=self._max_steps, record_trace=self._record_trace, @@ -87,66 +139,110 @@ class TaskRequestCommonBuilder: class TaskRequestBuilder(TaskRequestCommonBuilder, Generic[TIn]): - """Fluent configurator constructing immutable TaskRequest specifications.""" + """Builder class providing a fluent interface for creating TaskRequest objects. - def __init__(self, goal: str) -> None: + This builder allows for step-by-step construction of a TaskRequest with + clear methods that make the configuration process intuitive and type-safe. + + Examples: + >>> builder = TaskRequestBuilder[None](goal="Open Gmail and check unread + emails") + >>> task_request = ( + ... builder + ... .with_max_steps(30) + ... .using_profile("LowReasoning") + ... .with_output_description("A list of email subjects and senders") + ... .build() + ... ) + """ + + model_config = {"ignored_types": (CyFunctionDetector,)} + + def __init__(self, goal: str): + """Initialize an empty TaskRequestBuilder.""" super().__init__() - self._goal: str = goal + self._goal = goal self._profile: str | AgentProfile | None = None self._name: str | None = None - self._output_description: str | None = None + self._output_description = None self._output_format: type[TIn] | None = None @classmethod - def from_common(cls, goal: str, common: TaskRequestCommon) -> TaskRequestBuilder[None]: - """Derive a new task builder initialized with existing environment constraints.""" - builder = TaskRequestBuilder[None](goal=goal) - builder._max_steps = common.max_steps - builder._record_trace = common.record_trace - builder._trace_path = common.trace_path - builder._llm_output_path = common.llm_output_path - builder._locked_app_package = common.locked_app_package - builder._app_path = common.app_path - return builder + def from_common(cls, goal: str, common: TaskRequestCommon): + res = cls(goal=goal) + res._max_steps = common.max_steps + res._record_trace = common.record_trace + res._trace_path = common.trace_path + res._llm_output_path = common.llm_output_path + res._locked_app_package = common.locked_app_package + res._app_path = common.app_path + return res - def using_profile(self, profile: str | AgentProfile) -> Self: - """Select execution profile (e.g., 'flash', 'pro', or custom AgentProfile).""" + def using_profile(self, profile: str | AgentProfile) -> "TaskRequestBuilder[TIn]": + """Set the agent profile for executing the task. + + Args: + profile: The agent profile to use + """ self._profile = profile return self - def with_name(self, name: str) -> Self: - """Assign human-readable identifier to this test task.""" - self._name = name.strip() + def with_name(self, name: str) -> "TaskRequestBuilder[TIn]": + """Set the name of the task - useful when recording traces. + + Otherwise, a random name will be generated. + + Args: + name: Name of the task + """ + self._name = name return self def without_llm_output_saving(self) -> Self: - """Omit persisting standalone model reasoning outputs.""" + """Disable LLM output saving for the task.""" self._llm_output_path = None return self - def with_output_description(self, description: str) -> Self: - """Supply natural language extraction instructions for unformatted deliverables.""" + def with_output_description(self, description: str) -> "TaskRequestBuilder[TIn]": + """Set the description of the expected output format. + + This is especially useful for data extraction tasks. + + Args: + description: Description of the expected output format + """ self._output_description = description return self - def with_output_format(self, output_format: type[TOut]) -> TaskRequestBuilder[TOut]: - """Bind output schema to a structured Pydantic model class.""" - self._output_format = cast(type[TIn], output_format) + def with_output_format(self, output_format: type[TOut]) -> "TaskRequestBuilder[TOut]": + """Set the pydantic model for the expected output format. + + Args: + output_format: Pydantic model instance defining the output format + """ + self._output_format = output_format # type: ignore return cast(TaskRequestBuilder[TOut], self) def build(self) -> TaskRequest[TIn]: - """Validate criteria and construct final TaskRequest instance.""" - if not self._goal or not self._goal.strip(): - raise ValueError("Task goal must be a non-empty natural language objective.") + """Build the TaskRequest object. - if self._output_format is not None and self._output_description is not None: - raise ValueError("Cannot specify both output_format and output_description simultaneously.") + Returns: + A configured TaskRequest object - profile_name = self._profile.name if isinstance(self._profile, AgentProfile) else self._profile + Raises: + ValueError: If required fields are missing + """ + if not self._goal: + raise ValueError("Task goal is required") - return TaskRequest( - goal=self._goal.strip(), - profile=profile_name, + if self._output_format and self._output_description: + raise ValueError("Output format and description are mutually exclusive") + + task_request = TaskRequest( + goal=self._goal, + profile=self._profile.name + if isinstance(self._profile, AgentProfile) + else self._profile, task_name=self._name, output_description=self._output_description, output_format=self._output_format, @@ -157,3 +253,4 @@ class TaskRequestBuilder(TaskRequestCommonBuilder, Generic[TIn]): locked_app_package=self._locked_app_package, app_path=self._app_path, ) + return task_request diff --git a/artemis/sdk/examples/accessibility_helper_diagnostics.py b/artemis/sdk/examples/accessibility_helper_diagnostics.py deleted file mode 100644 index 516126e..0000000 --- a/artemis/sdk/examples/accessibility_helper_diagnostics.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Example: Accessibility Helper Diagnostics & Resilient Hierarchy Dumps. - -This example shows how ARTEMIS uses its managed Accessibility Helper APK -to perform conflict-free UI tree inspection alongside external test frameworks -(e.g., Mobly, Appium) without locking the Android UiAutomation service. -""" - -import asyncio -from artemis.controllers.platform_specific_commands_controller import get_first_device -from artemis.drivers.android.adb_driver import AndroidAdbDriver - - -async def main(): - device_id, platform, _ = get_first_device() - if not device_id: - print("⚠️ No connected Android device found via ADB.") - return - - print(f"📱 Connected device: {device_id} (Platform: {platform})") - - # Initialize the Artemis Android ADB Driver - driver = AndroidAdbDriver(device_id=device_id) - - # Capture screen data through Artemis managed hierarchy pipeline - screen_data = await driver.get_screen_data() - - print(f"📐 Screen Dimensions: {screen_data.width}x{screen_data.height}") - print(f"🌲 Hierarchy Elements Captured: {len(screen_data.elements)}") - if screen_data.elements: - first = screen_data.elements[0] - print(f" Root Element Class: {first.get('class', 'N/A')}") - print(f" Package: {first.get('package', 'N/A')}") - - await driver.disconnect() - print("✅ Diagnostics complete.") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/artemis/sdk/examples/android_settings_inspection.py b/artemis/sdk/examples/android_settings_inspection.py deleted file mode 100644 index 878fc70..0000000 --- a/artemis/sdk/examples/android_settings_inspection.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Example: Automated Android System Settings Inspection. - -This example demonstrates how to use the ARTEMIS SDK to launch the Android -Settings app, inspect device battery and storage configurations, and retrieve -verified diagnostics as a structured response. -""" - -import asyncio -from pydantic import BaseModel, Field - -from artemis.interfaces.sdk.client import ArtemisClient -from artemis.sdk.builders.task_request_builder import TaskRequestBuilder - - -class BatteryDiagnostics(BaseModel): - """Structured report of Android battery diagnostics.""" - - battery_percentage: str | None = Field( - default=None, description="Current battery percentage level visible on screen" - ) - battery_saver_status: str | None = Field( - default=None, description="Battery Saver state (e.g. On, Off)" - ) - diagnostics_summary: str = Field( - description="Summary of the battery health and system status" - ) - - -async def main(): - # Initialize the Artemis SDK client - client = ArtemisClient() - - # Build the task request targeting Android Settings - task_request = ( - TaskRequestBuilder(goal="Open Android Settings, navigate to Battery, and extract the current battery level and status.") - .with_name("settings_battery_inspection") - .with_max_steps(15) - .with_locked_app_package("com.android.settings") - .with_output_format(BatteryDiagnostics) - .build() - ) - - print(f"🚀 Running Artemis Task: {task_request.task_name}...") - task = await client.run_task(task_request) - - print(f"✅ Task Status: {task.status}") - if task.result and task.result.content: - report = task.result.get_as_model(BatteryDiagnostics) - print(f"📊 Battery Level: {report.battery_percentage}") - print(f"⚡ Battery Saver: {report.battery_saver_status}") - print(f"📝 Summary: {report.diagnostics_summary}") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/artemis/sdk/examples/app_lock_messaging.py b/artemis/sdk/examples/app_lock_messaging.py new file mode 100644 index 0000000..e89e7fb --- /dev/null +++ b/artemis/sdk/examples/app_lock_messaging.py @@ -0,0 +1,87 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio + +from artemis.sdk import Agent +from pydantic import BaseModel, Field + + +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass + + +class MessageResult(BaseModel): + """Structured result from messaging task.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + + messages_sent: int = Field(..., description="Number of messages successfully sent") + contacts: list[str] = Field(..., description="List of contacts messaged") + success: bool = Field(..., description="Whether all messages were sent successfully") + + +async def main() -> None: + # Create agent with default configuration + agent = Agent() + + try: + await agent.init() + + # Use app lock to keep execution in WhatsApp + # This ensures the agent stays in the app and relaunches if needed + task = ( + agent.new_task("Send 'Happy New Year!' message to Alice, Bob, and Charlie on WhatsApp") + .with_name("send_new_year_messages") + .with_locked_app_package("com.whatsapp") # Lock to WhatsApp + .with_output_format(MessageResult) + .with_max_steps(600) # Messaging tasks may need more steps + .build() + ) + + print("Sending messages with app lock enabled...") + print("The agent will stay in WhatsApp and relaunch if needed.\n") + + result = await agent.run_task(request=task) + + if result: + print("\n=== Messaging Complete ===") + print(f"Messages sent: {result.messages_sent}") + print(f"Contacts: {', '.join(result.contacts)}") + print(f"Success: {result.success}") + else: + print("Failed to send messages") + + except Exception as e: + print(f"Error: {e}") + finally: + await agent.clean() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/artemis/sdk/examples/app_workflow_automation.py b/artemis/sdk/examples/app_workflow_automation.py deleted file mode 100644 index 92cc11b..0000000 --- a/artemis/sdk/examples/app_workflow_automation.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Example: App Workflow Automation with Trace Recording. - -This example demonstrates how to orchestrate multi-step mobile UI navigation, -capture step-by-step traces, and enforce deterministic completion criteria -using ARTEMIS. -""" - -import asyncio -from artemis.interfaces.sdk.client import ArtemisClient -from artemis.sdk.builders.task_request_builder import TaskRequestBuilder - - -async def main(): - client = ArtemisClient() - - # Build a task with full trace recording - task_request = ( - TaskRequestBuilder( - goal="Open Clock app, verify current alarm settings, and ensure the active alarms list is visible." - ) - .with_name("clock_alarm_audit") - .with_max_steps(12) - .with_locked_app_package("com.google.android.deskclock") - .with_trace_recording(enabled=True, path="traces/clock_alarm_audit") - .with_output_description("Summary of configured alarms and their status") - .build() - ) - - print(f"🎬 Initiating Workflow Automation: {task_request.task_name}...") - task = await client.run_task(task_request) - - print(f"🏁 Execution Finished: status={task.status}") - if task.result: - print(f"⏱️ Duration: {task.result.execution_time_seconds:.2f}s, Steps: {task.result.steps_taken}") - print(f"📄 Output: {task.result.content}") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/artemis/sdk/examples/multimodal_vision_testing.py b/artemis/sdk/examples/multimodal_vision_testing.py deleted file mode 100644 index 1859d04..0000000 --- a/artemis/sdk/examples/multimodal_vision_testing.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Example: Multimodal UI Testing and Screen Element Grounding. - -This example demonstrates using ARTEMIS multimodal perception (Explorer & OCR-XML -fusion) to locate visual elements on dynamic or non-standard UI canvases without -relying on static coordinates. -""" - -import asyncio -from pydantic import BaseModel, Field - -from artemis.interfaces.sdk.client import ArtemisClient -from artemis.sdk.builders.task_request_builder import TaskRequestBuilder - - -class VisualAuditReport(BaseModel): - """Structured report verifying visual UI layout integrity.""" - - active_screen_title: str = Field(description="Title of the active screen or view") - interactive_buttons_count: int = Field( - description="Number of actionable buttons identified on the current view" - ) - visual_anomalies_detected: bool = Field( - description="True if unexpected popups, overlapping text, or error banners exist" - ) - observations: str = Field(description="Visual layout observations and audit details") - - -async def main(): - client = ArtemisClient() - - task_request = ( - TaskRequestBuilder( - goal="Open Calculator, verify numeric keypad buttons layout, and confirm display screen is empty." - ) - .with_name("calculator_visual_audit") - .with_max_steps(8) - .with_locked_app_package("com.google.android.calculator") - .with_output_format(VisualAuditReport) - .build() - ) - - print("🔍 Starting Multimodal Vision Audit...") - task = await client.run_task(task_request) - - if task.result and task.result.content: - audit = task.result.get_as_model(VisualAuditReport) - print(f"📱 View: {audit.active_screen_title}") - print(f"🔘 Interactive Buttons: {audit.interactive_buttons_count}") - print(f"⚠️ Anomalies Detected: {audit.visual_anomalies_detected}") - print(f"📝 Observations: {audit.observations}") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/artemis/sdk/examples/simple_photo_organizer.py b/artemis/sdk/examples/simple_photo_organizer.py new file mode 100644 index 0000000..6650d66 --- /dev/null +++ b/artemis/sdk/examples/simple_photo_organizer.py @@ -0,0 +1,111 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Simple Photo Organizer - Basic SDK Usage Example + +This example demonstrates a straightforward way to use the Artemis SDK +without builders or advanced configuration. It performs a real-world automation +task: +1. Opens the photo gallery +2. Finds photos from a specific date +3. Creates an album and moves those photos into it + +Run: +- python artemis/sdk/examples/simple_photo_organizer.py +""" + +import asyncio +from datetime import date, timedelta +from artemis.sdk import Agent +from pydantic import BaseModel, Field + + +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass + + +class PhotosResult(BaseModel): + """Structured result from photo search.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + + found_photos: int = Field(..., description="Number of photos found") + date_range: str = Field(..., description="Date range of photos found") + album_created: bool = Field(..., description="Whether an album was created") + album_name: str = Field(..., description="Name of the created album") + photos_moved: int = Field(0, description="Number of photos moved to the album") + + +async def main() -> None: + # Create a simple agent with default configuration + agent = Agent() + + try: + # Initialize agent (finds a device, starts required servers) + await agent.init() + + # Calculate yesterday's date for the example + yesterday = date.today() - timedelta(days=1) + formatted_date = yesterday.strftime("%B %d") # e.g. "August 22" + + print(f"Looking for photos from {formatted_date}...") + + # First task: search for photos and organize them, with typed output + result = await agent.run_task( + goal=( + "Open the Photos/Gallery app. Find photos taken on" + f" {formatted_date}. Create a new album named '{formatted_date}" + " Memories' and move those photos into it. Count how many" + " photos were moved." + ), + output=PhotosResult, + name="organize_photos", + ) + + # Handle and display the result + if result: + print("\n=== Photo Organization Complete ===") + print(f"Found: {result.found_photos} photos from {result.date_range}") + + if result.album_created: + print(f"Created album: '{result.album_name}'") + print(f"Moved {result.photos_moved} photos to the album") + else: + print("No album was created") + else: + print("Failed to organize photos") + + except Exception as e: + print(f"Error: {e}") + finally: + # Always clean up resources + await agent.clean() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/artemis/sdk/examples/smart_notification_assistant.py b/artemis/sdk/examples/smart_notification_assistant.py new file mode 100644 index 0000000..297ea08 --- /dev/null +++ b/artemis/sdk/examples/smart_notification_assistant.py @@ -0,0 +1,235 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Smart Notification Assistant - Intermediate SDK Usage Example + +This example demonstrates more advanced SDK features including: +- TaskRequestBuilder pattern +- Multiple agent profiles for different reasoning tasks +- Tracing for debugging/visualization +- Structured output with Pydantic +- Exception handling + +It performs a practical automation task: +1. Checks notification panel for unread notifications +2. Categorizes them by priority/app +3. Performs actions based on notification content + +Run: +- python artemis/sdk/examples/smart_notification_assistant.py +""" + +import asyncio +from datetime import datetime + +try: + from enum import StrEnum +except ImportError: + from enum import Enum + + class StrEnum(str, Enum): + pass + + +from artemis.config import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback +from artemis.sdk import Agent +from artemis.sdk.builders import Builders +from artemis.sdk.types import AgentProfile +from artemis.sdk.types.exceptions import AgentError +from pydantic import BaseModel, Field + + +class NotificationPriority(StrEnum): + HIGH = "high" + MEDIUM = "medium" + LOW = "low" + + +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass + + +class Notification(BaseModel): + """Individual notification details.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + + app_name: str = Field(..., description="Name of the app that sent the notification") + title: str = Field(..., description="Title/header of the notification") + message: str = Field(..., description="Message content of the notification") + priority: NotificationPriority = Field( + default=NotificationPriority.MEDIUM, + description="Priority level of notification", + ) + + +class NotificationSummary(BaseModel): + """Summary of all notifications.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + + total_count: int = Field(..., description="Total number of notifications found") + high_priority_count: int = Field(0, description="Count of high priority notifications") + notifications: list[Notification] = Field( + default_factory=list, description="List of individual notifications" + ) + + +def get_agent() -> Agent: + # Create two specialized profiles: + # 1. An analyzer profile for detailed inspection tasks + analyzer_profile = AgentProfile( + name="analyzer", + llm_config=LLMConfig( + planner=LLMWithFallback( + provider="openrouter", + model="meta-llama/llama-4-scout", + fallback=LLM(provider="openrouter", model="meta-llama/llama-4-maverick"), + ), + utils=LLMConfigUtils( + outputter=LLMWithFallback( + provider="openai", + model="gpt-5-nano", + fallback=LLM(provider="openai", model="gpt-5-mini"), + ), + hopper=LLMWithFallback( + provider="openai", + model="gpt-5-nano", + fallback=LLM(provider="openai", model="gpt-5-mini"), + ), + ), + ), + # from_file="/tmp/analyzer.jsonc" # can be loaded from file + ) + + # 2. An action profile for handling easy & fast actions based on notifications + action_profile = AgentProfile( + name="note_taker", + llm_config=LLMConfig( + planner=LLMWithFallback( + provider="openai", + model="o3", + fallback=LLM(provider="openai", model="gpt-5"), + ), + utils=LLMConfigUtils( + outputter=LLMWithFallback( + provider="openai", + model="gpt-5-nano", + fallback=LLM(provider="openai", model="gpt-5-mini"), + ), + hopper=LLMWithFallback( + provider="openai", + model="gpt-5-nano", + fallback=LLM(provider="openai", model="gpt-5-mini"), + ), + ), + ), + ) + + # Configure default task settings with tracing + task_defaults = Builders.TaskDefaults.with_max_steps(200).build() + + # Configure the agent + config = ( + Builders.AgentConfig.add_profiles(profiles=[analyzer_profile, action_profile]) + .with_default_profile(profile=action_profile) + .with_default_task_config(config=task_defaults) + .build() + ) + return Agent(config=config) + + +async def main(): + # Set up traces directory with timestamp for uniqueness + timestamp = datetime.now().strftime("%Y%m%d_%H%M") + traces_dir = f"/tmp/notification_traces/{timestamp}" + agent = get_agent() + + try: + # Initialize agent (finds a device, starts required servers) + await agent.init() + + print("Checking for notifications...") + + # Task 1: Get and analyze notifications with analyzer profile + notification_task = ( + agent.new_task( + goal=( + "Open the notification panel (swipe down from top). Scroll" + " through the first 3 unread notifications. For each" + " notification, identify the app name, title, and content." + " Tag messages from messaging apps or email as high" + " priority." + ) + ) + .with_output_format(NotificationSummary) + .using_profile("analyzer") + .with_name("notification_scan") + .with_max_steps(400) + .with_trace_recording(enabled=True, path=traces_dir) + .build() + ) + + # Execute the task with proper exception handling + try: + notifications = await agent.run_task(request=notification_task) + + # Display the structured results + if notifications: + print("\n=== Notification Summary ===") + print(f"Total notifications: {notifications.total_count}") + print(f"High priority: {notifications.high_priority_count}") + + # Task 2: Create a note to store the notification summary + response = await agent.run_task( + goal=( + "Open my Notes app and create a new note summarizing" + f" the following information:\n{notifications}" + ), + name="email_action", + profile="note_taker", + ) + print(f"Action result: {response}") + + else: + print("Failed to retrieve notifications") + + except AgentError as e: + print(f"Agent error occurred: {e}") + except Exception as e: + print(f"Unexpected error: {type(e).__name__}: {e}") + raise + + finally: + # Clean up + await agent.clean() + print(f"\nTraces saved to: {traces_dir}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/artemis/sdk/examples/video_transcription_example.py b/artemis/sdk/examples/video_transcription_example.py new file mode 100644 index 0000000..5b12673 --- /dev/null +++ b/artemis/sdk/examples/video_transcription_example.py @@ -0,0 +1,110 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Video Transcription Example + +This example demonstrates how to use the video recording tools to capture +and analyze video content from a mobile device screen. + +The agent can: +1. Start a screen recording +2. Perform actions while recording +3. Stop the recording and analyze its content using Gemini models + +Use case: Recording a video playing on the screen and transcribing its content. +""" + +import asyncio + +from artemis.config import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback +from artemis.sdk.agent import Agent +from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder +from artemis.sdk.types.agent import AgentConfig +from artemis.sdk.types.task import AgentProfile, TaskRequest + + +def get_video_capable_llm_config() -> LLMConfig: + """Returns an LLM config with video_analyzer configured. + + The video_analyzer must use a video-capable Gemini model: + - gemini-3.8-flash (recommended - fast and capable) + - gemini-3.7-flash + - gemini-3-pro-preview + - gemini-2.5-flash + - gemini-2.5-pro + - gemini-2.0-flash + """ + return LLMConfig( + planner=LLMWithFallback( + provider="openai", + model="gpt-5-nano", + fallback=LLM(provider="openai", model="gpt-5-mini"), + ), + utils=LLMConfigUtils( + outputter=LLMWithFallback( + provider="openai", + model="gpt-5-nano", + fallback=LLM(provider="openai", model="gpt-5-mini"), + ), + hopper=LLMWithFallback( + provider="openai", + model="gpt-5-nano", + fallback=LLM(provider="openai", model="gpt-5-mini"), + ), + video_analyzer=LLMWithFallback( + provider="google", + model="gemini-3.8-flash", + fallback=LLM(provider="google", model="gemini-3.7-flash"), + ), + ), + ) + + +async def main(): + config: AgentConfig = ( + AgentConfigBuilder() + .add_profile( + AgentProfile( + name="VideoCapable", + llm_config=get_video_capable_llm_config(), + ) + ) + .with_video_recording_tools() + .build() + ) + + agent = Agent(config=config) + try: + await agent.init() + + result = await agent.run_task( + request=TaskRequest( + goal=""" + 1. Open YouTube app + 2. Search for "Python tutorial" + 3. Start recording the screen + 4. Play the first video + 5. Wait for the first 30 seconds of the video to play + 6. Stop recording and tell me what was said in the video + """, + profile="VideoCapable", + ) + ) + print(f"Task result: {result}") + finally: + await agent.clean() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/artemis/sdk/types/__init__.py b/artemis/sdk/types/__init__.py index 0e7585e..9c59ed1 100644 --- a/artemis/sdk/types/__init__.py +++ b/artemis/sdk/types/__init__.py @@ -16,6 +16,7 @@ from artemis.sdk.types.agent import ( AgentConfig, + ApiBaseUrl, DevicePlatform, ServerConfig, ) @@ -36,10 +37,10 @@ from artemis.sdk.types.task import ( TaskRequest, TaskRequestCommon, TaskResult, - TaskRunStatus, ) __all__ = [ + "ApiBaseUrl", "AgentConfig", "DevicePlatform", "AgentProfile", @@ -47,7 +48,6 @@ __all__ = [ "TaskRequest", "TaskResult", "TaskRequestCommon", - "TaskRunStatus", "Task", "AgentProfileNotFoundError", "AgentTaskRequestError", diff --git a/artemis/sdk/types/agent.py b/artemis/sdk/types/agent.py index 72f9b15..79a934a 100644 --- a/artemis/sdk/types/agent.py +++ b/artemis/sdk/types/agent.py @@ -11,11 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. from typing import Literal +from urllib.parse import urlparse from langchain_core.callbacks.base import Callbacks from artemis.config import ( @@ -31,8 +29,63 @@ from artemis.utils.video import detect_video_tools_enabled from pydantic import BaseModel, Field +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass + + +class ApiBaseUrl(BaseModel): + """Defines an API base URL.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + scheme: Literal["http", "https"] + host: str + port: int | None = None + + def __eq__(self, other): + if not isinstance(other, ApiBaseUrl): + return False + return self.to_url() == other.to_url() + + def to_url(self): + return ( + f"{self.scheme}://{self.host}:{self.port}" + if self.port is not None + else f"{self.scheme}://{self.host}" + ) + + @classmethod + def from_url(cls, url: str) -> "ApiBaseUrl": + parsed_url = urlparse(url) + if parsed_url.scheme not in ["http", "https"]: + raise ValueError(f"Invalid scheme: {parsed_url.scheme}") + if parsed_url.hostname is None: + raise ValueError("Invalid hostname") + return cls( + scheme=parsed_url.scheme, # type: ignore + host=parsed_url.hostname, + port=parsed_url.port, + ) + + class ServerConfig(BaseModel): """Configuration for the required servers.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + adb_host: str adb_port: int diff --git a/artemis/sdk/types/exceptions.py b/artemis/sdk/types/exceptions.py index 41c7741..63288cc 100644 --- a/artemis/sdk/types/exceptions.py +++ b/artemis/sdk/types/exceptions.py @@ -12,95 +12,94 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Hierarchical exception taxonomy for the ARTEMIS client and automation runtime. +"""Exceptions for the ARTEMIS SDK. -Establishes categorized exception types covering host device discovery, -daemon lifecycle management, agent operational faults, and toolchain dependencies. +This module defines the exception hierarchy used throughout the Artemis SDK. """ -from __future__ import annotations - from typing import Literal -_TOOLCHAIN_INSTALL_URLS: dict[str, str] = { - "adb": "https://developer.android.com/tools/adb", -} - class ArtemisError(Exception): - """Root ancestor of all exceptions originating from ARTEMIS operations.""" + """Base exception class for all ARTEMIS SDK exceptions.""" - def __init__(self, message: str = "An unexpected error occurred within the ARTEMIS runtime.") -> None: + def __init__(self, message="An error occurred in the Artemis SDK"): self.message = message super().__init__(self.message) class DeviceError(ArtemisError): - """Raised when hardware interaction or ADB communication fails.""" + """Exception raised for errors related to mobile devices.""" - def __init__(self, message: str = "A mobile device error occurred.") -> None: + def __init__(self, message="A device-related error occurred"): super().__init__(message) class DeviceNotFoundError(DeviceError): - """Raised when no authorized target Android device or emulator is accessible.""" + """Exception raised when no mobile device is found.""" - def __init__(self, message: str = "No connected and authorized Android device found.") -> None: + def __init__(self, message="No mobile device found"): super().__init__(message) class ServerError(ArtemisError): - """Raised when an internal ARTEMIS daemon or background service encounters an error.""" + """Exception raised for errors related to ARTEMIS servers.""" - def __init__(self, message: str = "An ARTEMIS daemon service failure occurred.") -> None: + def __init__(self, message="A server-related error occurred"): super().__init__(message) class ServerStartupError(ServerError): - """Raised when a background process (e.g. Scrcpy, MCP, or local server) fails to boot.""" + """Exception raised when ARTEMIS servers fail to start.""" - def __init__(self, server_name: str | None = None, message: str | None = None) -> None: - resolved_msg = message or (f"Service '{server_name}' failed to start." if server_name else "Service startup failed.") - super().__init__(resolved_msg) + def __init__(self, server_name=None, message=None): + if server_name and not message: + message = f"Failed to start {server_name}" + elif not message: + message = "Failed to start Artemis servers" + super().__init__(message) self.server_name = server_name class AgentError(ArtemisError): - """Raised when reasoning, task scheduling, or graph traversal fails.""" + """Exception raised for errors related to the ARTEMIS agent.""" - def __init__(self, message: str = "An agent operational failure occurred.") -> None: + def __init__(self, message="An agent-related error occurred"): super().__init__(message) class AgentNotInitializedError(AgentError): - """Raised when invoking automation actions before initialization.""" + """Exception raised when attempting operations on an uninitialized agent.""" - def __init__(self, message: str = "Agent runtime has not been initialized.") -> None: + def __init__(self, message="Agent is not initialized. Call init() first"): super().__init__(message) class AgentTaskRequestError(AgentError): - """Raised when task parameters violate execution bounds or constraints.""" + """Exception raised when a requested task is invalid.""" - def __init__(self, message: str = "Invalid task request configuration.") -> None: + def __init__(self, message="An agent task-related error occurred"): super().__init__(message) class AgentProfileNotFoundError(AgentTaskRequestError): - """Raised when requested execution profile is not registered in system catalog.""" + """Exception raised when an agent profile is not found.""" - def __init__(self, profile_name: str) -> None: - super().__init__(f"Agent profile '{profile_name}' is not registered.") - self.profile_name = profile_name + def __init__(self, profile_name: str): + super().__init__(f"Agent profile {profile_name} not found") + + +EXECUTABLES = Literal["adb"] class ExecutableNotFoundError(ArtemisError): - """Raised when a mandatory host CLI toolchain executable is missing from PATH.""" + """Exception raised when a required executable is not found.""" - def __init__(self, executable_name: Literal["adb"] | str) -> None: - guidance = _TOOLCHAIN_INSTALL_URLS.get(executable_name, "") - msg = f"Required tool '{executable_name}' was not detected in system PATH." - if guidance: - msg += f" Install instructions: {guidance}" - super().__init__(msg) - self.executable_name = executable_name + def __init__(self, executable_name: EXECUTABLES): + install_instructions: dict[EXECUTABLES, str] = { + "adb": "https://developer.android.com/tools/adb", + } + message = f"Required executable '{executable_name}' not found in PATH." + if executable_name in install_instructions: + message += f"\nTo install it, please visit: {install_instructions[executable_name]}" + super().__init__(message) diff --git a/artemis/sdk/types/platform.py b/artemis/sdk/types/platform.py new file mode 100644 index 0000000..5bceb4f --- /dev/null +++ b/artemis/sdk/types/platform.py @@ -0,0 +1,210 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, PlainSerializer +from pydantic.v1.utils import to_lower_camel + +TaskRunStatus = Literal["pending", "running", "completed", "failed", "cancelled"] + +IsoDatetime = Annotated[ + datetime, + PlainSerializer( + func=lambda v: v.isoformat() if v else None, + return_type=str, + when_used="unless-none", + ), +] + + +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass + + +class BaseApiModel(BaseModel): + model_config = ConfigDict( + alias_generator=to_lower_camel, + populate_by_name=True, + str_strip_whitespace=True, + ) + + +class LLMProfileResponse(BaseApiModel): + """Response model for LLM profile.""" + + id: str = Field(..., description="Profile ID") + name: str = Field(..., description="Profile name") + description: str | None = Field(None, description="Profile description") + llms: dict[str, Any] = Field(..., description="LLM configuration") + created_at: str = Field(..., description="Creation timestamp (ISO format)") + updated_at: str = Field(..., description="Last update timestamp (ISO format)") + + +class TaskOptionsResponse(BaseApiModel): + """Response model for task options.""" + + id: str = Field(..., description="Options ID") + enable_tracing: bool = Field(..., description="Whether tracing is enabled") + max_steps: int = Field(..., description="Maximum number of steps") + enable_video_tools: bool = Field(..., description="Whether video tools are enabled") + created_at: str = Field(..., description="Creation timestamp (ISO format)") + updated_at: str = Field(..., description="Last update timestamp (ISO format)") + + +class TaskResponse(BaseApiModel): + """Response model for task.""" + + id: str = Field(..., description="Task ID") + name: str = Field(..., description="Task name") + description: str | None = Field(None, description="Task description") + input_prompt: str = Field(..., description="Input prompt") + output_description: str | None = Field(None, description="Output description") + options: TaskOptionsResponse = Field(..., description="Task options") + created_at: str = Field(..., description="Creation timestamp (ISO format)") + updated_at: str = Field(..., description="Last update timestamp (ISO format)") + + +class CreateTaskRunRequest(BaseApiModel): + """Request model for creating a task run.""" + + task_id: str = Field(..., description="ID of the task to run") + llm_profile_id: str = Field(..., description="LLM profile ID to use") + virtual_mobile_id: str | None = Field(None, description="Virtual mobile ID to use") + locked_app_package: str | None = Field(None, description="App package to lock for the task run") + execution_origin: str | None = Field(None, description="Origin of the task execution") + enable_video_tools: bool = Field( + default=False, + description="Whether video tools are enabled for this task run", + ) + + +class CreateOrphanTaskRunRequest(BaseApiModel): + """Request model for creating an orphan task run (without a parent task).""" + + input_prompt: str = Field( + ..., + description="Input prompt for the task run", + ) + output_description: str | None = Field( + None, + description="Optional output description", + ) + llm_profile_id: str = Field( + ..., + description="LLM profile ID to use", + ) + virtual_mobile_id: str | None = Field( + None, + description="Optional virtual mobile ID on which it runs", + ) + locked_app_package: str | None = Field( + None, + description="Optional app package to lock execution to", + ) + execution_origin: str | None = Field( + None, + description="Origin of the task run execution", + ) + max_steps: int | None = Field( + None, + description="Maximum number of steps for the task run", + ) + enable_video_tools: bool = Field( + default=False, + description="Whether video tools are enabled for this task run", + ) + + +class UpdateTaskRunStatusRequest(BaseApiModel): + """Request model for updating task run status.""" + + status: TaskRunStatus = Field(..., description="New status of the task run") + message: str | None = Field(None, description="Message associated with the status") + output: str | None = Field(None, description="Output of the task run") + + +class TaskRunResponse(BaseApiModel): + """Response model for a single task run.""" + + id: str = Field(..., description="Unique identifier for the task run") + task: TaskResponse | None = Field( + ..., + description=("ID of the task this run is for or None if manually created"), + ) + llm_profile: LLMProfileResponse = Field(..., description="LLM profile ID used for this run") + status: TaskRunStatus = Field(..., description="Current status of the task run") + input_prompt: str = Field(..., description="Input prompt for this task run") + output_description: str | None = Field(None, description="Description of expected output") + created_at: datetime = Field(..., description="When the task run was created") + started_at: datetime | None = Field(None, description="When the task run started") + finished_at: datetime | None = Field(None, description="When the task run finished") + locked_app_package: str | None = Field( + None, description="The app package to lock for the task run" + ) + + +SubgoalState = Literal["pending", "started", "completed", "failed"] + + +class ArtemisSubgoal(BaseModel): + """Upsert ArtemisSubgoal API model.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + + name: str = Field(..., description="Name of the subgoal") + state: SubgoalState = Field(default="pending", description="Current state of the subgoal") + started_at: IsoDatetime | None = Field(default=None, description="When the subgoal started") + ended_at: IsoDatetime | None = Field(default=None, description="When the subgoal ended") + + +class UpsertTaskRunPlanRequest(BaseApiModel): + """Upsert task run plan request model.""" + + started_at: IsoDatetime = Field(..., description="When the plan started") + subgoals: list[ArtemisSubgoal] = Field(..., description="Subgoals of the plan") + ended_at: IsoDatetime | None = Field( + default=None, + description="When the plan ended (replanned or completed)", + ) + + +class TaskRunPlanResponse(UpsertTaskRunPlanRequest): + """Response model for a task run plan.""" + + id: str = Field(..., description="Unique identifier for the task run plan") + task_run_id: str = Field(..., description="ID of the task run this plan is for") + + +class TrajectoryGifUploadResponse(BaseApiModel): + """Response containing trajectory GIF signed upload URL.""" + + signed_url: str = Field(..., description="Signed upload URL for the trajectory GIF") + task_run_id: str = Field(..., description="Task run ID for which the GIF URL was generated") + file_path: str = Field(..., description="Path where the GIF will be stored") diff --git a/artemis/sdk/types/task.py b/artemis/sdk/types/task.py index 6fdce49..8a9d482 100644 --- a/artemis/sdk/types/task.py +++ b/artemis/sdk/types/task.py @@ -11,54 +11,67 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. -"""Task specifications, execution states, and outcome schemas for the ARTEMIS SDK. - -Defines the declarative configuration contract for mobile automation goals, -agent operational profiles, telemetry preferences, and asynchronous result contracts. -""" - -from __future__ import annotations +"""Task-related type definitions for the ARTEMIS SDK.""" from collections.abc import Callable, Coroutine from datetime import datetime from pathlib import Path -from typing import Any, Generic, Literal, TypeVar, overload - -from pydantic import BaseModel, ConfigDict, Field +from typing import Any, Generic, TypeVar, overload from artemis.config import LLMConfig, get_default_llm_config from artemis.constants import RECURSION_LIMIT from artemis.context import DeviceContext +from artemis.sdk.types.platform import TaskRunStatus from artemis.sdk.utils import load_llm_config_override +from pydantic import BaseModel, Field -TaskRunStatus = Literal["pending", "running", "completed", "failed", "cancelled"] + +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass class AgentProfile(BaseModel): - """Runtime configuration profile governing LLM model selection and reasoning policies.""" + """Represents a ARTEMIS agent profile. - model_config = ConfigDict(arbitrary_types_allowed=True) + Attributes: + name: Name of the agent - used to reference the agent when running + tasks. + llm_config: LLM configuration for the agent. + """ - name: str = Field(description="Identifier for referencing this operational agent profile.") + model_config = {"ignored_types": (CyFunctionDetector,)} + + name: str llm_config: LLMConfig = Field(default_factory=get_default_llm_config) @overload def __init__(self, *, name: str, llm_config: LLMConfig): ... @overload - def __init__(self, *, name: str, from_file: str | Path): ... + def __init__(self, *, name: str, from_file: str): ... def __init__( self, *, name: str, llm_config: LLMConfig | None = None, - from_file: str | Path | None = None, - **kwargs: Any, + from_file: str | None = None, + **kwargs, ): kwargs["name"] = name if from_file: @@ -66,11 +79,11 @@ class AgentProfile(BaseModel): elif llm_config: kwargs["llm_config"] = llm_config else: - raise ValueError("Either llm_config or from_file must be specified.") + raise ValueError("Either llm_config or from_file must be provided") super().__init__(**kwargs) - def __str__(self) -> str: - return f"AgentProfile({self.name}): {self.llm_config}" + def __str__(self): + return f"Profile {self.name}:\n{self.llm_config}" T = TypeVar("T", bound=BaseModel) @@ -78,73 +91,146 @@ TOutput = TypeVar("TOutput", bound=BaseModel | None) class TaskRequestBase(BaseModel): - """Baseline runtime limits and telemetry recording settings for automation runs.""" - - model_config = ConfigDict(extra="ignore", arbitrary_types_allowed=True) + """Defines common parameters of a mobile automation task request.""" + model_config = {"ignored_types": (CyFunctionDetector,)} max_steps: int = RECURSION_LIMIT record_trace: bool = True - trace_path: Path = Field(default_factory=lambda: Path("traces")) + trace_path: Path = Path("traces") llm_output_path: Path | None = None class TaskRequestCommon(TaskRequestBase): - """Common automation constraints including application isolation and pre-install packages.""" + """Defines common parameters for any task request.""" + model_config = {"ignored_types": (CyFunctionDetector,)} + max_steps: int = RECURSION_LIMIT locked_app_package: str | None = None app_path: Path | None = None + """Path to an app to install before running the task. + + For Android: Path to an APK file. + + The app will be installed automatically before the task starts. + """ class TaskRequest(TaskRequestCommon, Generic[TOutput]): - """Declarative request specifying the high-level mobile testing or automation objective.""" + """Defines the format of a mobile automation task request. - goal: str = Field(description="Natural language instruction outlining target behavior or assertions.") - profile: str | None = Field(default=None, description="Execution profile (e.g. 'flash' or 'pro').") - task_name: str | None = Field(default=None, description="Optional label identifying this task execution.") - output_description: str | None = Field(default=None, description="Formatting prompt for expected deliverables.") - output_format: type[TOutput] | None = Field(default=None, description="Target Pydantic schema for structured output.") + Attributes: + goal: Natural language description of the goal to achieve + profile: Optional agent profile to use for executing the task + task_name: Optional name for the task + output_description: Optional natural language description of expected + output format + output_format: Optional pydantic model for the output format of the task + max_steps: Maximum number of steps the agent can take (default: 20) + record_trace: Whether to record a trace (screenshots, actions) of the + execution (default: False) + trace_path: Directory path to save trace data if recording is enabled + llm_output_path: Path to save LLM output data + """ + + model_config = {"ignored_types": (CyFunctionDetector,)} + goal: str + profile: str | None = None + task_name: str | None = None + output_description: str | None = None + output_format: type[TOutput] | None = None enable_remote_tracing: bool = False class TaskResult(BaseModel): - """Immutable outcome payload produced upon automation task completion.""" + """Result of a mobile automation task. - model_config = ConfigDict(arbitrary_types_allowed=True) + Attributes: + content: Raw result content (could be text or structured data) + error: Error message if the task failed + execution_time_seconds: How long the task took to execute + steps_taken: Number of steps the agent took to complete the task + """ + model_config = {"ignored_types": (CyFunctionDetector,)} content: Any = None error: str | None = None - execution_time_seconds: float = 0.0 - steps_taken: int = 0 - - @property - def succeeded(self) -> bool: - """True if the task terminated without error.""" - return self.error is None + execution_time_seconds: float + steps_taken: int def get_as_model(self, model_class: type[T]) -> T: - """Parse structured content into the requested Pydantic model instance.""" + """Parse the content into a Pydantic model instance. + + Args: + model_class: The Pydantic model class to parse the data into + + Returns: + An instance of the specified model class + + Raises: + ValueError: If content is None or not compatible with the model + """ if self.content is None: - raise ValueError("No content available to parse into the requested schema.") + raise ValueError("No content available to parse into a model") if isinstance(self.content, model_class): return self.content return model_class.model_validate(self.content) class Task(BaseModel): - """Stateful execution container tracking lifecycle progress, status transitions, and final results.""" + """A mobile automation task to be executed. - model_config = ConfigDict(arbitrary_types_allowed=True) + Attributes: + id: Unique identifier for the task + device: Information about the target device + status: Current status of the task execution + request: User task request + created_at: ISO timestamp when the task was created + ended_at: ISO timestamp when the task ended + """ + model_config = {"ignored_types": (CyFunctionDetector,)} id: str device: DeviceContext - status: TaskRunStatus = "pending" + status: TaskRunStatus status_message: str | None = None on_status_changed: Callable[[TaskRunStatus, str | None, Any | None], Coroutine] | None = None - request: TaskRequest[Any] - created_at: datetime = Field(default_factory=datetime.now) + request: TaskRequest + created_at: datetime ended_at: datetime | None = None result: TaskResult | None = None + async def finalize( + self, + content: Any | None = None, + state: dict | None = None, + error: str | None = None, + cancelled: bool = False, + ): + new_status: TaskRunStatus = "completed" if error is None else "failed" + if new_status == "failed" and cancelled: + new_status = "cancelled" + message = "Task completed successfully" + if new_status == "failed": + message = "Task failed" + (f": {error}" if error else "") + elif new_status == "cancelled": + message = "Task cancelled" + (f": {error}" if error else "") + await self.set_status(status=new_status, message=message, output=content or error) + self.ended_at = datetime.now() + + duration = self.ended_at - self.created_at + steps_taken = -1 + if state is not None: + metadata = state.get("metadata", None) + if metadata: + steps_taken = metadata.get("step_count", -1) + + self.result = TaskResult( + content=content, + error=error, + execution_time_seconds=duration.total_seconds(), + steps_taken=steps_taken, + ) + def get_name(self) -> str: return self.request.task_name or self.id @@ -153,42 +239,8 @@ class Task(BaseModel): status: TaskRunStatus, message: str | None = None, output: Any | None = None, - ) -> None: + ): self.status = status self.status_message = message - if self.on_status_changed is not None: + if self.on_status_changed: await self.on_status_changed(status, message, output) - - async def finalize( - self, - content: Any | None = None, - state: dict[str, Any] | None = None, - error: str | None = None, - cancelled: bool = False, - ) -> None: - if cancelled: - resolved_status: TaskRunStatus = "cancelled" - default_msg = f"Task cancelled: {error}" if error else "Task cancelled." - elif error: - resolved_status = "failed" - default_msg = f"Task failed: {error}" - else: - resolved_status = "completed" - default_msg = "Task completed successfully." - - await self.set_status(status=resolved_status, message=default_msg, output=content or error) - self.ended_at = datetime.now() - - elapsed = (self.ended_at - self.created_at).total_seconds() - steps = -1 - if state is not None: - metadata = state.get("metadata") - if isinstance(metadata, dict): - steps = metadata.get("step_count", -1) - - self.result = TaskResult( - content=content, - error=error, - execution_time_seconds=max(0.0, elapsed), - steps_taken=steps, - ) diff --git a/artemis/tools/committee_tool.py b/artemis/tools/committee_tool.py index 75a19c2..5db3b34 100644 --- a/artemis/tools/committee_tool.py +++ b/artemis/tools/committee_tool.py @@ -46,6 +46,7 @@ from artemis.tools.tool_wrapper import ( invoke_tool_with_injection, tool_result_messages, ) +from artemis.tools.types import CyFunctionDetector from artemis.tools.video_tool import get_video_analyzer_tool from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path, get_notes_dir @@ -60,6 +61,8 @@ logger = get_logger(__name__) class AskCommitteeArgs(BaseModel): """Arguments schema for invoking the committee tool.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} avatar_directive: str = Field( ..., description="Directive to frame and steer the debate for your avatar.", diff --git a/artemis/tools/diagnostic_tool.py b/artemis/tools/diagnostic_tool.py index 85d735b..4262614 100644 --- a/artemis/tools/diagnostic_tool.py +++ b/artemis/tools/diagnostic_tool.py @@ -27,6 +27,7 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool from artemis.tools.tool_wrapper import ToolWrapper +from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -35,6 +36,8 @@ logger = get_logger(__name__) # pylint: disable=too-few-public-methods class AskDiagnoserArgs(BaseModel): """Arguments schema for invoking the diagnostic subagent.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} query: str = Field( ..., description=( diff --git a/artemis/tools/explorer_tool.py b/artemis/tools/explorer_tool.py index 294cd18..951108b 100644 --- a/artemis/tools/explorer_tool.py +++ b/artemis/tools/explorer_tool.py @@ -58,6 +58,7 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper +from artemis.tools.types import CyFunctionDetector from artemis.utils.element_hit_test import find_element_at_point from artemis.utils.logger import get_logger from artemis.utils.visualization import draw_dots @@ -93,6 +94,8 @@ CANDIDATE_DOT_COLOR = "magenta" class AskExplorerArgs(BaseModel): """Arguments of ``ask_explorer`` as seen by every calling agent.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} query: str = Field(..., description=ASK_EXPLORER_QUERY_DESCRIPTION) context_feedback: str = Field("", description=ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION) diff --git a/artemis/tools/image_processor_tool.py b/artemis/tools/image_processor_tool.py index 4e6e0a1..33b22c3 100644 --- a/artemis/tools/image_processor_tool.py +++ b/artemis/tools/image_processor_tool.py @@ -25,6 +25,7 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool from artemis.tools.tool_wrapper import ToolWrapper +from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -33,6 +34,8 @@ logger = get_logger(__name__) # pylint: disable=too-few-public-methods class AskVisionCoderArgs(BaseModel): """Arguments schema for invoking the Image Processor subagent.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} instruction: str = Field( ..., description=( diff --git a/artemis/tools/log_tool.py b/artemis/tools/log_tool.py index fd94495..dc61901 100644 --- a/artemis/tools/log_tool.py +++ b/artemis/tools/log_tool.py @@ -27,6 +27,7 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool from artemis.tools.tool_wrapper import ToolWrapper +from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -35,6 +36,8 @@ logger = get_logger(__name__) # pylint: disable=too-few-public-methods class AnalyzeLogsArgs(BaseModel): """Arguments schema for invoking the Log Analyzer subagent.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} specific_query: str = Field( ..., description=( diff --git a/artemis/tools/mobile/launch_app.py b/artemis/tools/mobile/launch_app.py index 3a5564e..5c0de07 100644 --- a/artemis/tools/mobile/launch_app.py +++ b/artemis/tools/mobile/launch_app.py @@ -18,10 +18,7 @@ from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool from pydantic import BaseModel, Field -from artemis.agents.entity_extractor.entity_extractor import ( - ExtractionResult, - extract_entity, -) +from artemis.agents.hopper.hopper import HopperOutput, hopper from artemis.context import ArtemisContext from artemis.controllers.platform_specific_commands_controller import ( list_packages_async, @@ -31,6 +28,7 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper +from artemis.tools.types import CyFunctionDetector from artemis.utils.app_launch_utils import launch_app_with_retries from artemis.utils.logger import get_logger @@ -39,6 +37,8 @@ logger = get_logger(__name__) class LaunchAppArgs(BaseModel): """Arguments schema for launching an application.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} app_name: str = Field( ..., description="The natural language name of the application to launch.", @@ -77,21 +77,21 @@ async def find_package(ctx: ArtemisContext, app_name: str, use_fallback: bool = package_cache[app_name] = app_name return app_name - extractor_output: ExtractionResult = await extract_entity( + hopper_output: HopperOutput = await hopper( ctx=ctx, request=(f"I'm looking for the package name of the following app: '{app_name}'"), data=all_packages, use_fallback=use_fallback, ) - if not extractor_output.found or not extractor_output.output: + if not hopper_output.found or not hopper_output.output: if isinstance(package_cache, dict): package_cache[app_name] = None return None - package_name = extractor_output.output.strip() + package_name = hopper_output.output.strip() if package_name not in package_set: logger.warning( - f"Entity extractor returned package '{package_name}' for '{app_name}', " + f"Hopper returned package '{package_name}' for '{app_name}', " "but it is NOT physically installed on the device!" ) if isinstance(package_cache, dict): diff --git a/artemis/tools/mobile/read_hierarchy.py b/artemis/tools/mobile/read_hierarchy.py index b724f74..bd8e47c 100644 --- a/artemis/tools/mobile/read_hierarchy.py +++ b/artemis/tools/mobile/read_hierarchy.py @@ -25,6 +25,7 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper +from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -32,6 +33,10 @@ logger = get_logger(__name__) class GetUiHierarchyArgs(BaseModel): """Arguments schema for getting UI hierarchy.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + + GET_UI_HIERARCHY_DOCSTRING = ( "[DIAGNOSTIC] Retrieves the current screen UI hierarchy XML using a persistent" " UI Automator client.\n\nUse this instead of running 'uiautomator dump'" diff --git a/artemis/tools/object_detection_tool.py b/artemis/tools/object_detection_tool.py index 9db1bc1..c922f90 100644 --- a/artemis/tools/object_detection_tool.py +++ b/artemis/tools/object_detection_tool.py @@ -35,6 +35,7 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper +from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -43,6 +44,8 @@ logger = get_logger(__name__) # pylint: disable=too-few-public-methods class ObjectDetectionArgs(BaseModel): """Arguments schema for object detection with explicit screenshot path.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} image_path: str = Field(..., description="Absolute path to the screenshot file.") queries: list[str] = Field(..., description="A list of single query strings to find.") @@ -50,6 +53,8 @@ class ObjectDetectionArgs(BaseModel): # pylint: disable=too-few-public-methods class OperatorObjectDetectionArgs(BaseModel): """Arguments schema for operator object detection using latest screenshot from state.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} queries: list[str] = Field(..., description="A list of single query strings to find.") diff --git a/artemis/tools/scratchpad.py b/artemis/tools/scratchpad.py index b9c3e9e..6032e43 100644 --- a/artemis/tools/scratchpad.py +++ b/artemis/tools/scratchpad.py @@ -25,6 +25,7 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool from artemis.tools.tool_wrapper import ToolWrapper +from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger from artemis.utils.notes import ( APPEND_NOTE_ARG_CONTENT_DESC, @@ -56,18 +57,24 @@ logger = get_logger(__name__) class SaveNoteArgs(BaseModel): """Arguments schema for saving notes.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} key: str = Field(..., description=SAVE_NOTE_ARG_KEY_DESC) content: str = Field(..., description=SAVE_NOTE_ARG_CONTENT_DESC) class AppendNoteArgs(BaseModel): """Arguments schema for appending notes.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} key: str = Field(..., description=APPEND_NOTE_ARG_KEY_DESC) content: str = Field(..., description=APPEND_NOTE_ARG_CONTENT_DESC) class UpdateNoteArgs(BaseModel): """Arguments schema for updating notes.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} key: str = Field(..., description=UPDATE_NOTE_ARG_KEY_DESC) target: str = Field(..., description=UPDATE_NOTE_ARG_TARGET_DESC) replacement: str = Field(..., description=UPDATE_NOTE_ARG_REPLACEMENT_DESC) @@ -75,6 +82,8 @@ class UpdateNoteArgs(BaseModel): class ReadNoteArgs(BaseModel): """Arguments schema for reading notes.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} key: str = Field(..., description=READ_NOTE_ARG_KEY_DESC) start_line: int | None = Field(None, description="Start line to read (1-indexed, inclusive)") end_line: int | None = Field(None, description="End line to read (1-indexed, inclusive)") @@ -82,6 +91,10 @@ class ReadNoteArgs(BaseModel): class ListNotesArgs(BaseModel): """Arguments schema for listing notes.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} + + class SaveNoteTool(ArtemisTool): """Universal tool for saving notes to persistent memory.""" diff --git a/artemis/tools/tool_wrapper.py b/artemis/tools/tool_wrapper.py index 4bb5498..0d97841 100644 --- a/artemis/tools/tool_wrapper.py +++ b/artemis/tools/tool_wrapper.py @@ -31,8 +31,13 @@ from artemis.context import ArtemisContext from artemis.data_engine import engine as engine_mod from artemis.data_engine.trace import CURRENT_TRACE_ID, smart_serialize from artemis.graph.state import State +from artemis.tools.types import CyFunctionDetector + + class ToolWrapper(BaseModel): """Wrapper holding a tool factory and lifecycle callbacks.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} tool_fn_getter: Callable[[ArtemisContext], BaseTool] on_success_fn: Callable[..., str] on_failure_fn: Callable[..., str] diff --git a/artemis/tools/types.py b/artemis/tools/types.py index ecf63ff..0fa8a94 100644 --- a/artemis/tools/types.py +++ b/artemis/tools/types.py @@ -12,47 +12,61 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Multi-strategy target element descriptors for ARTEMIS actions. - -Encapsulates semantic identifiers, textual queries, and spatial bounding boxes -into a prioritized target resolution contract. -""" - -from __future__ import annotations - -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, Field, model_validator from artemis.utils.ui_hierarchy import ElementBounds +class _CyFunctionDetectorMeta(type): + """Metaclass for detecting cython compiled functions.""" + + def __instancecheck__(cls, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +# pylint: disable=too-few-public-methods +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + """Detector for cython compiled functions.""" + + class Target(BaseModel): - """Composite locator descriptor resolving UI nodes across dynamic and visual channels.""" + """A comprehensive locator for a UI element, supporting a fallback mechanism.""" - model_config = ConfigDict(extra="ignore") + model_config = {"ignored_types": (CyFunctionDetector,)} - resource_id: str | None = Field(default=None, description="Android resource identifier token.") + resource_id: str | None = Field(None, description="The resource-id of the element.") resource_id_index: int | None = Field( - default=None, - description="Zero-based ordinal index when multiple nodes match resource_id.", + None, + description=("The zero-based index if multiple elements share the same resource-id."), ) text: str | None = Field( - default=None, - description="Exact or case-insensitive element label or content descriptor.", + None, + description=("The text content of the element (e.g., a label or placeholder)."), ) text_index: int | None = Field( - default=None, - description="Zero-based ordinal index when multiple nodes match text query.", + None, + description=("The zero-based index if multiple elements share the same text."), ) bounds: ElementBounds | None = Field( - default=None, - description="Spatial boundary metrics (x, y, width, height) of the element.", + None, description="The x, y, width, and height of the element." ) @model_validator(mode="after") - def _normalize_indices(self) -> Target: - """Coerce missing indices to zero if primary selector string is populated.""" - if self.resource_id and self.resource_id.strip() and self.resource_id_index is None: + def _default_indices(self): + # Treat empty strings like “not provided” + if ( + self.resource_id is not None and self.resource_id != "" + ) and self.resource_id_index is None: self.resource_id_index = 0 - if self.text and self.text.strip() and self.text_index is None: + if (self.text is not None and self.text != "") and self.text_index is None: self.text_index = 0 return self diff --git a/artemis/tools/utils.py b/artemis/tools/utils.py index 7dd1dc3..47e4f68 100644 --- a/artemis/tools/utils.py +++ b/artemis/tools/utils.py @@ -11,29 +11,25 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. - -"""Spatial positioning, focus resolution, and interaction helpers for ARTEMIS tools. - -Coordinates locator resolution across hierarchy trees, validates bounded screen areas, -and routes focus and cursor placement to target elements. -""" from __future__ import annotations -from typing import Any, Literal +from typing import Literal + from pydantic import BaseModel, ConfigDict from artemis.context import ArtemisContext -from artemis.controllers.types import CoordinatesSelectorRequest, PercentagesSelectorRequest +from artemis.controllers.types import ( + CoordinatesSelectorRequest, + PercentagesSelectorRequest, +) from artemis.controllers.unified_controller import UnifiedMobileController from artemis.graph.state import State from artemis.tools.types import Target from artemis.utils.logger import get_logger from artemis.utils.ui_hierarchy import ( ElementBounds, + Point, find_element_by_resource_id, get_bounds_for_element, get_element_text, @@ -43,54 +39,293 @@ from artemis.utils.ui_hierarchy import ( logger = get_logger(__name__) +def find_element_by_text( + ui_hierarchy: list[dict], text: str, index: int | None = None +) -> dict | None: + """Find a UI element by its text content (adapted to both flat and rich hierarchy) + + This function performs a recursive, case-insensitive search. + + Args: + ui_hierarchy: List of UI element dictionaries. + text: The text content to search for. + index: Optional index to select nth matching element. + + Returns: + The complete UI element dictionary if found, None otherwise. + """ + matches: list[dict] = [] + + def search_recursive(elements: list[dict]) -> None: + for element in elements: + if isinstance(element, dict): + src = element.get("attributes", element) + element_text = src.get("text", "") + + # Guard against non-string values + if not isinstance(element_text, str): + element_text = "" + + text_lower = text.lower() + if element_text and text_lower == element_text.lower(): + matches.append(element) + + if children := element.get("children", []): + search_recursive(children) + + search_recursive(ui_hierarchy) + + if not matches: + return None + + if index is None: + idx = 0 + elif index < 0: + return None + else: + idx = index + if idx < len(matches): + return matches[idx] + return None + + +async def tap_bottom_right_of_element(bounds: ElementBounds, ctx: ArtemisContext): + """Tap the bottom-right corner of the specified element bounds.""" + bottom_right: Point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99) + await tap( + ctx=ctx, + selector_request=SelectorRequestWithCoordinates( + coordinates=CoordinatesSelectorRequest( + x=bottom_right.x, + y=bottom_right.y, + ), + ), + ) + + +# pylint: disable=too-many-return-statements +async def move_cursor_to_end_if_bounds( + ctx: ArtemisContext, + state: State, + target: Target, + elt: dict | None = None, +) -> dict | None: + """Best-effort move of the text cursor near the end of the input by tapping the + + bottom-right area of the focused element (if bounds are available). + """ + if target.resource_id: + if not elt: + elt = find_element_by_resource_id( + ui_hierarchy=state.latest_ui_hierarchy or [], + resource_id=target.resource_id, + index=target.resource_id_index, + ) + if not elt: + return None + + bounds = get_bounds_for_element(elt) + if not bounds: + return elt + + logger.debug("Tapping near the end of the input to move the cursor") + await tap_bottom_right_of_element(bounds=bounds, ctx=ctx) + logger.debug(f"Tapped end of input {target.resource_id}") + return elt + + if target.bounds: + await tap_bottom_right_of_element(target.bounds, ctx=ctx) + logger.debug("Tapped end of input by coordinates") + return elt + + if target.text: + text_elt = find_element_by_text( + state.latest_ui_hierarchy or [], + target.text, + index=target.text_index, + ) + if text_elt: + bounds = get_bounds_for_element(text_elt) + if bounds: + await tap_bottom_right_of_element(bounds=bounds, ctx=ctx) + logger.debug(f"Tapped end of input that had text'{target.text}'") + return text_elt + return None + + return None + + +async def focus_element_if_needed( + ctx: ArtemisContext, target: Target +) -> Literal["resource_id", "coordinates", "text", "already_focused"] | None: + """Ensures the element is focused, with a sanity check to prevent trusting misleading IDs. + + If no target locator is provided (empty target), assumes the field is + already focused + and returns "already_focused" to allow typing without re-focusing. + """ + # If no locator is provided, assume already focused (e.g., keyboard is visible) + if not target.resource_id and not target.bounds and not target.text: + logger.debug("No target locator provided, assuming element is already focused") + return "already_focused" + + controller = UnifiedMobileController(ctx) + rich_hierarchy = await controller.get_ui_elements() + elt_from_id = None + if target.resource_id: + elt_from_id = find_element_by_resource_id( + ui_hierarchy=rich_hierarchy, + resource_id=target.resource_id, + index=target.resource_id_index, + is_rich_hierarchy=False, + ) + + if elt_from_id and target.text: + text_from_id_elt = get_element_text(elt_from_id) + if not text_from_id_elt or target.text.lower() != text_from_id_elt.lower(): + logger.warning( + f"ID '{target.resource_id}' and text '{target.text}' seem to be" + " on different elements. Ignoring the resource_id and falling" + " back to other locators." + ) + elt_from_id = None + + if elt_from_id: + if not is_element_focused(elt_from_id): + await tap( + ctx=ctx, + selector_request=IdSelectorRequest(id=target.resource_id), # type: ignore + index=target.resource_id_index, + ) + logger.debug(f"Focused (tap) on resource_id={target.resource_id}") + rich_hierarchy = await controller.get_ui_elements() + elt_from_id = find_element_by_resource_id( + ui_hierarchy=rich_hierarchy, + resource_id=target.resource_id, # type: ignore + index=target.resource_id_index, + is_rich_hierarchy=False, + ) + if elt_from_id and is_element_focused(elt_from_id): + logger.debug(f"Text input is focused: {target.resource_id}") + return "resource_id" + logger.warning(f"Failed to focus using resource_id='{target.resource_id}'. Fallback...") + + if target.bounds: + relative_point = target.bounds.get_center() + await tap( + ctx=ctx, + selector_request=SelectorRequestWithCoordinates( + coordinates=CoordinatesSelectorRequest(x=relative_point.x, y=relative_point.y) + ), + ) + logger.debug(f"Tapped on coordinates ({relative_point.x}, {relative_point.y}) to focus.") + return "coordinates" + + if target.text: + text_elt = find_element_by_text(rich_hierarchy, target.text, index=target.text_index) + if text_elt: + bounds = get_bounds_for_element(text_elt) + if bounds: + relative_point = bounds.get_center() + await tap( + ctx=ctx, + selector_request=SelectorRequestWithCoordinates( + coordinates=CoordinatesSelectorRequest( + x=relative_point.x, y=relative_point.y + ) + ), + ) + logger.debug(f"Tapped on text element '{target.text}' to focus.") + return "text" + + logger.error( + "Failed to focus element. No valid locator (resource_id, coordinates, or text) succeeded." + ) + return None + + +def validate_coordinates_bounds( + target: Target, screen_width: int, screen_height: int +) -> str | None: + """Validate that coordinates are within screen bounds. + + Returns error message if invalid, None if valid. + """ + if not target.bounds: + return None + + center = target.bounds.get_center() + errors = [] + + if center.x < 0 or center.x >= screen_width: + errors.append(f"x={center.x} is outside screen width (0-{screen_width})") + if center.y < 0 or center.y >= screen_height: + errors.append(f"y={center.y} is outside screen height (0-{screen_height})") + + return "; ".join(errors) if errors else None + + +def has_valid_selectors(target: Target) -> bool: + """Check if target has at least one valid selector.""" + has_coordinates = target.bounds is not None + has_resource_id = target.resource_id is not None and target.resource_id != "" + has_text = target.text is not None and target.text != "" + return has_coordinates or has_resource_id or has_text + + class IdSelectorRequest(BaseModel): - """Target selector driven by Android resource ID.""" + """Selector request using resource id.""" model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) id: str def to_dict(self) -> dict[str, str | int]: + """Convert selector to dictionary.""" return {"id": self.id} class TextSelectorRequest(BaseModel): - """Target selector driven by exact or case-insensitive element label.""" + """Selector request using element text.""" model_config = ConfigDict(extra="forbid") text: str def to_dict(self) -> dict[str, str | int]: + """Convert selector to dictionary.""" return {"text": self.text} class SelectorRequestWithCoordinates(BaseModel): - """Target selector pointing to an absolute pixel coordinate.""" + """Selector request using absolute coordinates.""" model_config = ConfigDict(extra="forbid") coordinates: CoordinatesSelectorRequest def to_dict(self) -> dict[str, str | int]: + """Convert selector to dictionary.""" return {"point": self.coordinates.to_str()} class SelectorRequestWithPercentages(BaseModel): - """Target selector pointing to normalized fractional [0.0, 1.0] coordinates.""" + """Selector request using percentage coordinates.""" model_config = ConfigDict(extra="forbid") percentages: PercentagesSelectorRequest def to_dict(self) -> dict[str, str | int]: + """Convert selector to dictionary.""" return {"point": self.percentages.to_str()} class IdWithTextSelectorRequest(BaseModel): - """Composite selector pairing resource ID with expected text.""" + """Selector request using both resource id and text.""" model_config = ConfigDict(extra="forbid") id: str text: str def to_dict(self) -> dict[str, str | int]: + """Convert selector to dictionary.""" return {"id": self.id, "text": self.text} @@ -106,30 +341,37 @@ SelectorRequest = ( def _extract_resource_id_and_text_from_selector( selector: SelectorRequest, ) -> tuple[str | None, str | None]: - """Unpack identifier and textual query from any supported SelectorRequest.""" + """Extract resource_id and text from a selector.""" + resource_id = None + text = None + if isinstance(selector, IdSelectorRequest): - return selector.id, None - if isinstance(selector, TextSelectorRequest): - return None, selector.text - if isinstance(selector, IdWithTextSelectorRequest): - return selector.id, selector.text - return None, None + resource_id = selector.id + elif isinstance(selector, TextSelectorRequest): + text = selector.text + elif isinstance(selector, IdWithTextSelectorRequest): + resource_id = selector.id + text = selector.text + + return resource_id, text async def tap( ctx: ArtemisContext, selector_request: SelectorRequest, index: int | None = None, -) -> Any: - """Dispatch an on-screen tap to an element or coordinate selector.""" +): + """Tap on a selector. + + Index is optional and is used when you have multiple views matching the same + selector. ui_hierarchy is optional and used for ADB taps to find elements. + """ controller = UnifiedMobileController(ctx) if isinstance(selector_request, SelectorRequestWithCoordinates): - res = await controller.tap_at( - x=selector_request.coordinates.x, - y=selector_request.coordinates.y, + result = await controller.tap_at( + x=selector_request.coordinates.x, y=selector_request.coordinates.y ) - return res.error if res.error else None - + return result.error if result.error else None if isinstance(selector_request, SelectorRequestWithPercentages): coords = selector_request.percentages.to_coords( width=ctx.device.device_width, @@ -137,183 +379,11 @@ async def tap( ) return await controller.tap_at(coords.x, coords.y) - res_id, txt = _extract_resource_id_and_text_from_selector(selector_request) + # For other selectors, we need the UI hierarchy + resource_id, text = _extract_resource_id_and_text_from_selector(selector_request) + return await controller.tap_element( - resource_id=res_id, - text=txt, + resource_id=resource_id, + text=text, index=index if index is not None else 0, ) - - -def find_element_by_text( - ui_hierarchy: list[dict[str, Any]], text: str, index: int | None = None -) -> dict[str, Any] | None: - """Search for elements matching label text across nested hierarchy tree nodes.""" - target_text = (text or "").strip().lower() - if not target_text: - return None - - matched_nodes: list[dict[str, Any]] = [] - - def _traverse(nodes: list[dict[str, Any]]) -> None: - for node in nodes: - if not isinstance(node, dict): - continue - attrs = node.get("attributes", node) - val = attrs.get("text") - if isinstance(val, str) and val.strip().lower() == target_text: - matched_nodes.append(attrs if "attributes" in node else node) - - children = node.get("children") - if isinstance(children, list) and children: - _traverse(children) - - _traverse(ui_hierarchy) - if not matched_nodes: - return None - - target_idx = max(0, index or 0) - if target_idx < len(matched_nodes): - return matched_nodes[target_idx] - return None - - -async def tap_bottom_right_of_element(bounds: ElementBounds, ctx: ArtemisContext) -> None: - """Position pointer near the trailing boundary of an input element.""" - corner_point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99) - await tap( - ctx=ctx, - selector_request=SelectorRequestWithCoordinates( - coordinates=CoordinatesSelectorRequest(x=corner_point.x, y=corner_point.y) - ), - ) - - -async def move_cursor_to_end_if_bounds( - ctx: ArtemisContext, - state: State, - target: Target, - elt: dict[str, Any] | None = None, -) -> dict[str, Any] | None: - """Position insertion point at the end of an input element boundary.""" - current_tree = state.latest_ui_hierarchy or [] - - if target.resource_id: - element_node = elt or find_element_by_resource_id( - ui_hierarchy=current_tree, - resource_id=target.resource_id, - index=target.resource_id_index, - ) - if not element_node: - return None - box = get_bounds_for_element(element_node) - if not box: - return element_node - await tap_bottom_right_of_element(bounds=box, ctx=ctx) - return element_node - - if target.bounds: - await tap_bottom_right_of_element(bounds=target.bounds, ctx=ctx) - return elt - - if target.text: - matched = find_element_by_text(current_tree, target.text, index=target.text_index) - if matched: - box = get_bounds_for_element(matched) - if box: - await tap_bottom_right_of_element(bounds=box, ctx=ctx) - return matched - return None - - return None - - -async def focus_element_if_needed( - ctx: ArtemisContext, target: Target -) -> Literal["resource_id", "coordinates", "text", "already_focused"] | None: - """Ensure element possesses active input focus, falling back through available selectors.""" - if not target.resource_id and not target.bounds and not target.text: - return "already_focused" - - controller = UnifiedMobileController(ctx) - hierarchy = await controller.get_ui_elements() - - element_by_id = None - if target.resource_id: - element_by_id = find_element_by_resource_id( - ui_hierarchy=hierarchy, - resource_id=target.resource_id, - index=target.resource_id_index, - is_rich_hierarchy=False, - ) - - if element_by_id and target.text: - observed_label = get_element_text(element_by_id) - if not observed_label or target.text.strip().lower() != observed_label.strip().lower(): - logger.warning( - f"ID '{target.resource_id}' and text '{target.text}' do not align to the same node." - " Falling back to secondary selectors." - ) - element_by_id = None - - if element_by_id: - if not is_element_focused(element_by_id): - await tap( - ctx=ctx, - selector_request=IdSelectorRequest(id=target.resource_id), # type: ignore[arg-type] - index=target.resource_id_index, - ) - hierarchy = await controller.get_ui_elements() - element_by_id = find_element_by_resource_id( - ui_hierarchy=hierarchy, - resource_id=target.resource_id, # type: ignore[arg-type] - index=target.resource_id_index, - is_rich_hierarchy=False, - ) - if element_by_id and is_element_focused(element_by_id): - return "resource_id" - - if target.bounds: - centroid = target.bounds.get_center() - await tap( - ctx=ctx, - selector_request=SelectorRequestWithCoordinates( - coordinates=CoordinatesSelectorRequest(x=centroid.x, y=centroid.y) - ), - ) - return "coordinates" - - if target.text: - matched_text_node = find_element_by_text(hierarchy, target.text, index=target.text_index) - if matched_text_node: - text_box = get_bounds_for_element(matched_text_node) - if text_box: - centroid = text_box.get_center() - await tap( - ctx=ctx, - selector_request=SelectorRequestWithCoordinates( - coordinates=CoordinatesSelectorRequest(x=centroid.x, y=centroid.y) - ), - ) - return "text" - - logger.error("Failed to focus element. No valid locator (resource_id, coordinates, or text) succeeded.") - return None - - -def validate_coordinates_bounds(target: Target, screen_width: int, screen_height: int) -> str | None: - """Verify target centroid resides within active viewport boundaries.""" - if not target.bounds: - return None - c = target.bounds.get_center() - errors: list[str] = [] - if not (0 <= c.x < screen_width): - errors.append(f"x={c.x} is outside screen width (0-{screen_width})") - if not (0 <= c.y < screen_height): - errors.append(f"y={c.y} is outside screen height (0-{screen_height})") - return "; ".join(errors) if errors else None - - -def has_valid_selectors(target: Target) -> bool: - """Verify target defines at least one valid locator criterion.""" - return bool(target.bounds is not None or target.resource_id or target.text) diff --git a/artemis/tools/video_tool.py b/artemis/tools/video_tool.py index 9120a0f..da4209f 100644 --- a/artemis/tools/video_tool.py +++ b/artemis/tools/video_tool.py @@ -34,6 +34,7 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper +from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -41,6 +42,8 @@ logger = get_logger(__name__) class VideoAnalyzerArgs(BaseModel): """Arguments schema for video analyzer tool.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} time_description: str = Field( ..., description=( diff --git a/artemis/tools/wait_tool.py b/artemis/tools/wait_tool.py index be33105..96371d6 100644 --- a/artemis/tools/wait_tool.py +++ b/artemis/tools/wait_tool.py @@ -23,6 +23,7 @@ from artemis.data_engine.trace import trace_langchain_tool from artemis.drivers.base import BaseDeviceDriver from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper +from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -30,6 +31,8 @@ logger = get_logger(__name__) class WaitArgs(BaseModel): """Arguments schema for wait tool.""" + + model_config = {"ignored_types": (CyFunctionDetector,)} seconds: int = Field( ..., description=("The duration to wait/sleep in seconds. Minimum is 1, maximum is 60."), diff --git a/artemis/utils/app_launch_utils.py b/artemis/utils/app_launch_utils.py index 770efbe..a7b3468 100644 --- a/artemis/utils/app_launch_utils.py +++ b/artemis/utils/app_launch_utils.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. """Utilities for handling app locking and initial app launch logic.""" @@ -219,7 +216,7 @@ async def _poll_for_app_ready( Only fails if we get a different (non-null) package or timeout. Args: - ctx: ArtemisContext + ctx: Mobile use context app_package: Expected package name max_poll_seconds: Maximum time to poll (default: 15s) poll_interval: Time between polls (default: 1s) @@ -275,7 +272,7 @@ async def launch_app_with_retries( """Launch an app with retry logic and smart polling. Args: - ctx: ArtemisContext + ctx: Mobile use context app_package: Package name (Android) to launch max_retries: Maximum number of launch attempts (default: 3) max_poll_seconds: Maximum time to wait for app to load per attempt @@ -348,7 +345,7 @@ async def _handle_initial_app_launch( 3. Return status with success/error information Args: - ctx: ArtemisContext + ctx: Mobile use context locked_app_package: Package name (Android) to lock to Returns: diff --git a/artemis/utils/cli_helpers.py b/artemis/utils/cli_helpers.py new file mode 100644 index 0000000..f7056b2 --- /dev/null +++ b/artemis/utils/cli_helpers.py @@ -0,0 +1,38 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from adbutils import AdbClient +from rich.console import Console + + +def display_device_status(console: Console, adb_client: AdbClient | None = None): + """Checks for connected devices and displays the status.""" + console.print("\n[bold]📱 Device Status[/bold]") + devices = None + if adb_client is not None: + devices = adb_client.device_list() + if devices: + console.print("✅ [bold green]Android device(s) connected:[/bold green]") + for device in devices: + console.print(f" - {device.serial}") + else: + console.print("❌ [bold red]No Android device found.[/bold red]") + command = "emulator -avd " + if sys.platform not in ["win32", "darwin"]: + command = f"./{command}" + console.print( + f"You can start an emulator using a command like: [bold]'{command}'[/bold]" + ) diff --git a/artemis/utils/cli_selection.py b/artemis/utils/cli_selection.py new file mode 100644 index 0000000..8b288c5 --- /dev/null +++ b/artemis/utils/cli_selection.py @@ -0,0 +1,154 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +import inquirer +from rich.console import Console +from rich.prompt import Prompt +from rich.text import Text + + +def select_provider_and_model( + console: Console, + available_providers: list[str], + available_models: dict, + default_provider: str, + default_model: str, + provider: str | None = None, + model: str | None = None, +) -> tuple[str, str]: + """Interactive selection of LLM provider and model with arrow-key dropdowns when available. + + Args: + console: Rich console for output + available_providers: List of available provider names + available_models: Dict mapping providers to their available models + default_provider: Default provider to use + default_model: Default model to use + provider: Pre-selected provider (optional) + model: Pre-selected model (optional) + + Returns: + Tuple of (selected_provider, selected_model) + """ + final_provider = provider + final_model = model + + # Interactive provider selection + if not final_provider: + console.print("\n🤖 [bold cyan]LLM Provider Selection[/bold cyan]") + final_provider = _select_from_list( + console=console, + item_type="provider", + choices=available_providers, + default=default_provider, + message="Select LLM provider", + ) + + # Interactive model selection + if not final_model: + console.print(f"\n🎯 [bold green]Model Selection for {final_provider}[/bold green]") + available_model_list = ( + available_models[final_provider] + if final_provider + else available_models[default_provider] + ) + + default_model_for_provider = ( + default_model if default_model in available_model_list else available_model_list[0] + ) + + final_model = _select_from_list( + console=console, + item_type="model", + choices=available_model_list, + default=default_model_for_provider, + message=f"Select model for {final_provider}", + ) + + return final_provider, final_model + + +def _select_from_list( + console: Console, + item_type: str, + choices: list[str], + default: str, + message: str, +) -> str: + """Select an item from a list using arrow keys when available, fallback to numbered selection. + + Args: + console: Rich console for output + item_type: Type of item being selected (for error messages) + choices: List of choices to select from + default: Default choice + message: Message to display in dropdown + + Returns: + Selected choice + """ + # Try arrow-key dropdown if TTY is available, fallback to numbered selection + if sys.stdin.isatty(): + try: + questions = [ + inquirer.List( + "selection", + message=f"{message} (use arrow keys)", + choices=choices, + default=default, + ), + ] + answers = inquirer.prompt(questions) + return answers["selection"] if answers else default + except Exception: + # Fallback to numbered selection + return _numbered_selection(console, item_type, choices, default) + else: + return _numbered_selection(console, item_type, choices, default) + + +def _numbered_selection(console: Console, item_type: str, choices: list[str], default: str) -> str: + """Fallback numbered selection when arrow keys aren't available.""" + choices_text = "\n".join([f" {i + 1}. {choice}" for i, choice in enumerate(choices)]) + console.print(f"Available {item_type}s:\n{choices_text}") + + default_idx = choices.index(default) + 1 if default in choices else 1 + + while True: + choice = Prompt.ask( + f"Select {item_type} (1-{len(choices)}) or press Enter for default", + default=str(default_idx), + ) + try: + choice_idx = int(choice) - 1 + if 0 <= choice_idx < len(choices): + return choices[choice_idx] + else: + console.print("[red]Invalid choice. Please try again.[/red]") + except ValueError: + console.print("[red]Please enter a number.[/red]") + + +def display_llm_config(console: Console, provider: str, model: str) -> None: + """Display the selected LLM configuration with colors.""" + config_text = Text() + config_text.append("🤖 LLM Configuration: ", style="bold white") + config_text.append("Provider: ", style="white") + config_text.append(f"{provider}", style="bold cyan") + config_text.append(" | Model: ", style="white") + config_text.append(f"{model}", style="bold green") + + console.print(config_text) diff --git a/artemis/utils/conversations.py b/artemis/utils/conversations.py new file mode 100644 index 0000000..c7a2058 --- /dev/null +++ b/artemis/utils/conversations.py @@ -0,0 +1,45 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TypeGuard + +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage + + +def is_ai_message(message: BaseMessage) -> TypeGuard[AIMessage]: + return isinstance(message, AIMessage) + + +def is_human_message(message: BaseMessage) -> TypeGuard[HumanMessage]: + return isinstance(message, HumanMessage) + + +def is_tool_message(message: BaseMessage) -> TypeGuard[ToolMessage]: + return isinstance(message, ToolMessage) + + +def is_tool_for_name(tool_message: ToolMessage, name: str) -> bool: + return tool_message.name == name + + +def get_screenshot_message_for_llm(screenshot_base64: str): + prefix = "" if screenshot_base64.startswith("data:image") else "data:image/jpeg;base64," + return HumanMessage( + content=[ + { + "type": "image_url", + "image_url": {"url": f"{prefix}{screenshot_base64}"}, + } + ] + ) diff --git a/artemis/utils/decorators.py b/artemis/utils/decorators.py index e50eb88..4ef07b9 100644 --- a/artemis/utils/decorators.py +++ b/artemis/utils/decorators.py @@ -12,89 +12,128 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Lifecycle callback and execution telemetry decorators for ARTEMIS. - -Provides function interception hooks for pre-execution logging, success verification, -and error capture across synchronous and asynchronous agent invocations. -""" - -from __future__ import annotations - +import asyncio +from collections.abc import Awaitable, Callable from functools import wraps import inspect -from typing import Any, Callable +from typing import Any, TypeVar, cast, overload + +R = TypeVar("R") + + +def wrap_with_callbacks_sync( + fn: Callable[..., R], + *, + before: Callable[..., None] | None = None, + on_success: Callable[[R], None] | None = None, + on_failure: Callable[[Exception], None] | None = None, + suppress_exceptions: bool = False, +) -> Callable[..., R]: + @wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> R: + if before: + before() + try: + result = fn(*args, **kwargs) + if on_success: + on_success(result) + return result + except Exception as e: + if on_failure: + on_failure(e) + if suppress_exceptions: + return None # type: ignore + raise + + return wrapper + + +def wrap_with_callbacks_async( + fn: Callable[..., Awaitable[R]], + *, + before: Callable[..., None] | None = None, + on_success: Callable[[R], None] | None = None, + on_failure: Callable[[Exception], None] | None = None, + suppress_exceptions: bool = False, +) -> Callable[..., Awaitable[R]]: + @wraps(fn) + async def wrapper(*args: Any, **kwargs: Any) -> R: + if before: + before() + try: + result = await fn(*args, **kwargs) + if on_success: + on_success(result) + return result + except Exception as e: + if on_failure: + on_failure(e) + if suppress_exceptions: + return None # type: ignore + raise + + return wrapper + + +@overload +def wrap_with_callbacks( + fn: Callable[..., Awaitable[R]], + *, + before: Callable[[], None] | None = ..., + on_success: Callable[[R], None] | None = ..., + on_failure: Callable[[Exception], None] | None = ..., + suppress_exceptions: bool = ..., +) -> Callable[..., Awaitable[R]]: ... + + +@overload +def wrap_with_callbacks( + *, + before: Callable[..., None] | None = ..., + on_success: Callable[[Any], None] | None = ..., + on_failure: Callable[[Exception], None] | None = ..., + suppress_exceptions: bool = ..., +) -> Callable[[Callable[..., R]], Callable[..., R]]: ... + + +@overload +def wrap_with_callbacks( + fn: Callable[..., R], + *, + before: Callable[[], None] | None = ..., + on_success: Callable[[R], None] | None = ..., + on_failure: Callable[[Exception], None] | None = ..., + suppress_exceptions: bool = ..., +) -> Callable[..., R]: ... def wrap_with_callbacks( fn: Callable[..., Any] | None = None, *, - before: Callable[..., None] | None = None, + before: Callable[[], None] | None = None, on_success: Callable[[Any], None] | None = None, on_failure: Callable[[Exception], None] | None = None, suppress_exceptions: bool = False, ) -> Any: - """Wrap a callable with pre-invocation, success, and error callback handlers. + def decorator(func: Callable[..., Any]) -> Any: + if inspect.iscoroutinefunction(func): + return wrap_with_callbacks_async( + cast(Callable[..., Awaitable[Any]], func), + before=before, + on_success=on_success, + on_failure=on_failure, + suppress_exceptions=suppress_exceptions, + ) + else: + return wrap_with_callbacks_sync( + cast(Callable[..., Any], func), + before=before, + on_success=on_success, + on_failure=on_failure, + suppress_exceptions=suppress_exceptions, + ) - Supports decorating both coroutines and regular synchronous functions. - """ - - def _decorator(target_fn: Callable[..., Any]) -> Callable[..., Any]: - if inspect.iscoroutinefunction(target_fn): - - @wraps(target_fn) - async def _async_wrapped(*args: Any, **kwargs: Any) -> Any: - if before is not None: - try: - before() - except Exception: - pass - try: - result = await target_fn(*args, **kwargs) - if on_success is not None: - try: - on_success(result) - except Exception: - pass - return result - except Exception as exc: - if on_failure is not None: - try: - on_failure(exc) - except Exception: - pass - if suppress_exceptions: - return None - raise - - return _async_wrapped - - @wraps(target_fn) - def _sync_wrapped(*args: Any, **kwargs: Any) -> Any: - if before is not None: - try: - before() - except Exception: - pass - try: - result = target_fn(*args, **kwargs) - if on_success is not None: - try: - on_success(result) - except Exception: - pass - return result - except Exception as exc: - if on_failure is not None: - try: - on_failure(exc) - except Exception: - pass - if suppress_exceptions: - return None - raise - - return _sync_wrapped - - if fn is not None: - return _decorator(fn) - return _decorator + if fn is None: + return decorator + else: + return decorator(fn) diff --git a/artemis/utils/file.py b/artemis/utils/file.py index 365b1f7..7b8d97d 100644 --- a/artemis/utils/file.py +++ b/artemis/utils/file.py @@ -12,43 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""File and configuration loading utilities for ARTEMIS.""" - -from __future__ import annotations - import json -from pathlib import Path import re -from typing import IO, Any +from typing import IO def strip_json_comments(text: str) -> str: - """Removes single-line (//) and multi-line (/* ... */) comments from JSONC text. - - Args: - text: Raw JSON with comments string. - - Returns: - Clean JSON string with comments stripped. - """ - pattern = r"//.*?$|/\*.*?\*/" - return re.sub(pattern, "", text, flags=re.MULTILINE | re.DOTALL) + text = re.sub(r"//.*?$", "", text, flags=re.MULTILINE) + text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL) + return text -def load_jsonc(source: str | Path | IO[str] | IO[bytes]) -> dict[str, Any]: - """Loads and parses a JSONC (JSON with Comments) document. - - Args: - source: File-like object, Path, or string path to parse. - - Returns: - Parsed dictionary. - """ - if isinstance(source, (str, Path)): - raw_content = Path(source).read_text(encoding="utf-8") - else: - content = source.read() - raw_content = content.decode("utf-8") if isinstance(content, bytes) else content - - cleaned = strip_json_comments(raw_content) - return json.loads(cleaned) +def load_jsonc(file: IO) -> dict: + return json.loads(strip_json_comments(file.read())) diff --git a/artemis/utils/logger.py b/artemis/utils/logger.py index 321d278..4d863f6 100644 --- a/artemis/utils/logger.py +++ b/artemis/utils/logger.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. from enum import Enum import logging diff --git a/artemis/utils/media.py b/artemis/utils/media.py index bf91f58..4b5680a 100644 --- a/artemis/utils/media.py +++ b/artemis/utils/media.py @@ -11,33 +11,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. - -"""Media synthesis and trace asset management for ARTEMIS. - -Handles chronological trace GIF generation, frame quantization, and step snapshot -compilation while preserving active session recording manifests. -""" - -from __future__ import annotations import json import os from pathlib import Path -import shutil import subprocess import tempfile -from typing import Any - -from PIL import Image from artemis.utils.logger import get_logger +from PIL import Image logger = get_logger(__name__) -_PREFER_FFMPEG = os.environ.get("USE_FFMPEG_GIF", "").strip().lower() in ("1", "true", "yes") +USE_FFMPEG_GIF = os.environ.get("USE_FFMPEG_GIF", "").lower() in ( + "1", + "true", + "yes", +) def quantize_and_save_gif_from_paths( @@ -46,68 +36,81 @@ def quantize_and_save_gif_from_paths( colors: int = 128, duration: int = 100, ) -> None: - """Compile ordered image snapshots into a quantized animation asset.""" + """Create an optimized GIF from image file paths. + + By default uses PIL (loads all frames into memory). + Set USE_FFMPEG_GIF=1 env var to use ffmpeg for memory-efficient streaming. + + Args: + image_paths: List of paths to image files (must be sorted in desired + order) + output_path: Path where the GIF will be saved + colors: Number of colors to use in quantization (lower = smaller file) + duration: Duration of each frame in milliseconds + + Raises: + ValueError: If image_paths list is empty + RuntimeError: If ffmpeg fails (when USE_FFMPEG_GIF is enabled) + """ if not image_paths: - raise ValueError("Cannot assemble animation from an empty image sequence.") + raise ValueError("image_paths must not be empty") - if _PREFER_FFMPEG and shutil.which("ffmpeg"): - try: - _render_gif_with_ffmpeg(image_paths, output_path, colors=colors, frame_delay_ms=duration) - return - except Exception as exc: - logger.warning(f"FFmpeg GIF rendering encountered an error ({exc}); falling back to Pillow.") - - _render_gif_with_pillow(image_paths, output_path, colors=colors, frame_delay_ms=duration) + if USE_FFMPEG_GIF: + _save_gif_ffmpeg(image_paths, output_path, colors, duration) + else: + _save_gif_pillow(image_paths, output_path, colors, duration) -def _render_gif_with_pillow( - paths: list[Path], +def _save_gif_pillow( + image_paths: list[Path], output_path: Path, colors: int, - frame_delay_ms: int, + duration: int, ) -> None: - """Internal Pillow quantization loop.""" - frames: list[Image.Image] = [] - for p in paths: - try: - with Image.open(p) as img: - rgb_img = img.convert("RGB") if img.mode != "RGB" else img.copy() - frames.append(rgb_img.quantize(colors=colors, method=Image.Quantize.MEDIANCUT)) - except Exception as exc: - logger.debug(f"Skipping damaged frame {p}: {exc}") + """Create GIF using PIL (loads all frames into memory).""" - if not frames: - return + def frame_generator(): + for path in image_paths: + with Image.open(path) as img: + if img.mode != "RGB": + img = img.convert("RGB") + yield img.quantize(colors=colors, method=2) - first_frame = frames[0] + frames = frame_generator() + first_frame = next(frames) first_frame.save( output_path, save_all=True, - append_images=frames[1:], + append_images=frames, loop=0, optimize=True, - duration=frame_delay_ms, + duration=duration, ) -def _render_gif_with_ffmpeg( - paths: list[Path], +def _save_gif_ffmpeg( + image_paths: list[Path], output_path: Path, colors: int, - frame_delay_ms: int, + duration: int, ) -> None: - """Streamlined two-pass palette-generated GIF rendering via FFmpeg.""" - fps = max(1.0, 1000.0 / max(1, frame_delay_ms)) - with tempfile.TemporaryDirectory() as tmpdir: - list_file = Path(tmpdir) / "frames.txt" - palette = Path(tmpdir) / "palette.png" + """Create GIF using ffmpeg (memory-efficient streaming).""" + fps = 1000 / duration - lines = [f"file '{p.resolve()}'\nduration {frame_delay_ms / 1000.0}" for p in paths] - if paths: - lines.append(f"file '{paths[-1].resolve()}'") - list_file.write_text("\n".join(lines), encoding="utf-8") + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: + for i, path in enumerate(image_paths): + f.write(f"file '{path.absolute()}'\n") + # Last file needs no duration (ffmpeg concat demuxer uses it as the final frame) + if i < len(image_paths) - 1: + f.write(f"duration {duration / 1000}\n") + # Repeat last file to ensure it's included (ffmpeg concat demuxer quirk) + f.write(f"file '{image_paths[-1].absolute()}'\n") + concat_file = Path(f.name) - subprocess.run( + palette_path = output_path.with_suffix(".palette.png") + + try: + result = subprocess.run( [ "ffmpeg", "-y", @@ -116,16 +119,18 @@ def _render_gif_with_ffmpeg( "-safe", "0", "-i", - str(list_file), + str(concat_file), "-vf", - f"palettegen=max_colors={min(colors, 256)}", - str(palette), + f"palettegen=max_colors={min(colors, 256)}:stats_mode=diff", + str(palette_path), ], capture_output=True, - check=True, + text=True, ) + if result.returncode != 0: + raise RuntimeError(f"ffmpeg palette generation failed: {result.stderr}") - subprocess.run( + result = subprocess.run( [ "ffmpeg", "-y", @@ -134,78 +139,71 @@ def _render_gif_with_ffmpeg( "-safe", "0", "-i", - str(list_file), + str(concat_file), "-i", - str(palette), + str(palette_path), "-lavfi", - f"fps={fps},paletteuse", + f"fps={fps},paletteuse=dither=bayer:bayer_scale=5", + "-loop", + "0", str(output_path), ], capture_output=True, - check=True, + text=True, ) + if result.returncode != 0: + raise RuntimeError(f"ffmpeg GIF creation failed: {result.stderr}") + + finally: + concat_file.unlink(missing_ok=True) + if palette_path.exists(): + palette_path.unlink() -def create_gif_from_trace_folder(trace_folder_path: Path) -> None: - """Scan and synthesize sequential JPEG frames in a trace directory to trace.gif.""" - if not trace_folder_path.is_dir(): +def create_gif_from_trace_folder(trace_folder_path: Path): + image_files: list[Path] = [] + + for file in trace_folder_path.iterdir(): + if file.suffix == ".jpeg": + image_files.append(file) + + image_files.sort(key=lambda f: int(f.stem)) + + logger.info(f"Found {len(image_files)} images to compile") + + if not image_files: return - frames = sorted( - [f for f in trace_folder_path.iterdir() if f.suffix.lower() in (".jpeg", ".jpg") and f.stem.isdigit()], - key=lambda x: int(x.stem), - ) - if not frames: - return - - target_gif = trace_folder_path / "trace.gif" - quantize_and_save_gif_from_paths(frames, target_gif) - logger.info(f"Assembled trace animation: {target_gif}") + gif_path = trace_folder_path / "trace.gif" + quantize_and_save_gif_from_paths(image_files, gif_path) + logger.info(f"GIF created at {gif_path}") -def remove_images_from_trace_folder(trace_folder_path: Path) -> None: - """Purge temporary individual JPEG frame captures once compiled.""" - if not trace_folder_path.is_dir(): - return - for item in trace_folder_path.iterdir(): - if item.suffix.lower() in (".jpeg", ".jpg") and item.stem.isdigit(): - try: - item.unlink(missing_ok=True) - except OSError: - pass +def remove_images_from_trace_folder(trace_folder_path: Path): + for file in trace_folder_path.iterdir(): + if file.suffix == ".jpeg": + file.unlink() -def create_steps_json_from_trace_folder(trace_folder_path: Path) -> None: - """Collate discrete numeric step payload dumps into steps.json preserving metadata manifests.""" - if not trace_folder_path.is_dir(): - return +def create_steps_json_from_trace_folder(trace_folder_path: Path): + """Compile legacy timestamp-named step files without consuming manifests.""" + steps = [] + for file in trace_folder_path.iterdir(): + if file.suffix == ".json" and file.stem.isdigit(): + with open(file, encoding="utf-8", errors="ignore") as f: + json_content = f.read() + steps.append({"timestamp": int(file.stem), "data": json_content}) - numeric_step_files = sorted( - [f for f in trace_folder_path.iterdir() if f.suffix == ".json" and f.stem.isdigit()], - key=lambda x: int(x.stem), - ) + steps.sort(key=lambda f: f["timestamp"]) - records: list[dict[str, Any]] = [] - for step_file in numeric_step_files: - try: - records.append({ - "timestamp": int(step_file.stem), - "data": step_file.read_text(encoding="utf-8", errors="ignore"), - }) - except Exception as exc: - logger.debug(f"Unable to read step file {step_file}: {exc}") + logger.info(f"Found {len(steps)} steps to compile") - output_file = trace_folder_path / "steps.json" - output_file.write_text(json.dumps(records, ensure_ascii=False), encoding="utf-8") + with open(trace_folder_path / "steps.json", "w", encoding="utf-8", errors="ignore") as f: + f.write(json.dumps(steps)) -def remove_steps_json_from_trace_folder(trace_folder_path: Path) -> None: - """Clean up standalone numeric step files, leaving steps.json and recording manifests untouched.""" - if not trace_folder_path.is_dir(): - return - for item in trace_folder_path.iterdir(): - if item.suffix == ".json" and item.stem.isdigit(): - try: - item.unlink(missing_ok=True) - except OSError: - pass +def remove_steps_json_from_trace_folder(trace_folder_path: Path): + """Remove only legacy timestamp-named step files, preserving manifests.""" + for file in trace_folder_path.iterdir(): + if file.suffix == ".json" and file.stem.isdigit(): + file.unlink() diff --git a/artemis/utils/recorder.py b/artemis/utils/recorder.py new file mode 100644 index 0000000..70dbee1 --- /dev/null +++ b/artemis/utils/recorder.py @@ -0,0 +1,60 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 +import time + +from langchain_core.messages import BaseMessage +from artemis.context import ArtemisContext +from artemis.controllers.controller_factory import create_device_controller +from artemis.utils.logger import get_logger + +logger = get_logger(__name__) + + +async def record_interaction(ctx: ArtemisContext, response: BaseMessage): + if not ctx.execution_setup: + raise ValueError("No execution setup found") + if not ctx.execution_setup.traces_path or not ctx.execution_setup.trace_name: + raise ValueError("No traces path or trace name found") + + logger.info("Recording interaction") + controller = create_device_controller(ctx) + screenshot_base64 = await controller.screenshot() + logger.info("Screenshot taken") + try: + controller = create_device_controller(ctx) + compressed_screenshot_base64 = controller.get_compressed_b64_screenshot(screenshot_base64) + except Exception as e: + logger.error(f"Error compressing screenshot: {e}") + return "Could not record this interaction" + timestamp = time.time() + folder = ctx.execution_setup.traces_path.joinpath(ctx.execution_setup.trace_name).resolve() + folder.mkdir(parents=True, exist_ok=True) + try: + with open( + folder.joinpath(f"{int(timestamp)}.jpeg").resolve(), + "wb", + ) as f: + f.write(base64.b64decode(compressed_screenshot_base64)) + + with open( + folder.joinpath(f"{int(timestamp)}.json").resolve(), + "w", + encoding="utf-8", + ) as f: + f.write(response.model_dump_json()) + except Exception as e: + logger.error(f"Error recording interaction: {e}") + return "Screenshot recorded successfully" diff --git a/artemis/utils/requests_utils.py b/artemis/utils/requests_utils.py new file mode 100644 index 0000000..aa56194 --- /dev/null +++ b/artemis/utils/requests_utils.py @@ -0,0 +1,51 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from artemis.utils.logger import get_logger +import requests + +logger = get_logger(__name__) + + +def curl_from_request(req: requests.PreparedRequest) -> str: + """Converts a requests.PreparedRequest object to a valid cURL command string.""" + command = ["curl", f"-X {req.method}"] + + for key, value in req.headers.items(): + command.append(f'-H "{key}: {value}"') + + if req.body: + body = req.body + if isinstance(body, bytes): + body = body.decode("utf-8") + # Escape single quotes in the body for shell safety + body = body.replace("'", "'\\''") + command.append(f"-d '{body}'") + + command.append(f"'{req.url}'") + + return " ".join(command) + + +def logging_hook(response, *args, **kwargs): + """Hook to log the request as a cURL command.""" + curl_command = curl_from_request(response.request) + logger.debug(f"\n--- cURL Command ---\n{curl_command}\n--------------------") + + +def get_session_with_curl_logging() -> requests.Session: + """Returns a requests.Session with cURL logging enabled.""" + session = requests.Session() + session.hooks["response"] = [logging_hook] + return session diff --git a/artemis/utils/shell_utils.py b/artemis/utils/shell_utils.py new file mode 100644 index 0000000..1972a73 --- /dev/null +++ b/artemis/utils/shell_utils.py @@ -0,0 +1,35 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess + + +def run_shell_command_on_host(command: str) -> str: + """Helper to run a shell command on the host and return the output.""" + try: + result = subprocess.run( + command, + stdin=subprocess.DEVNULL, + shell=True, + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + # Log the error and stderr for better debugging + error_message = ( + f"Command '{command}' failed with exit code {e.returncode}.\nStderr: {e.stderr.strip()}" + ) + raise RuntimeError(error_message) from e diff --git a/artemis/utils/task_tree.py b/artemis/utils/task_tree.py index d7bd739..ee9d1f2 100644 --- a/artemis/utils/task_tree.py +++ b/artemis/utils/task_tree.py @@ -739,7 +739,7 @@ def _render_step_detailed( INTERNAL_SYSTEM_TOOLS = { "safety_net_validation", "safety_net_pixel_validation", - "entity_extractor", + "hopper", } if interleaved: diff --git a/artemis/agents/entity_extractor/__init__.py b/artemis/utils/time.py similarity index 71% rename from artemis/agents/entity_extractor/__init__.py rename to artemis/utils/time.py index c40cae6..5ec443a 100644 --- a/artemis/agents/entity_extractor/__init__.py +++ b/artemis/utils/time.py @@ -12,14 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Entity extractor and package resolver agent for ARTEMIS.""" +from datetime import datetime -from artemis.agents.entity_extractor.entity_extractor import ( - ExtractionResult, - extract_entity, -) -__all__ = [ - "ExtractionResult", - "extract_entity", -] +def convert_timestamp_to_str(ts: float) -> str: + dt = datetime.fromtimestamp(ts) + return dt.strftime("%Y-%m-%dT%H-%M-%S") diff --git a/artemis/utils/ui_hierarchy.py b/artemis/utils/ui_hierarchy.py index a90089d..6114497 100644 --- a/artemis/utils/ui_hierarchy.py +++ b/artemis/utils/ui_hierarchy.py @@ -12,151 +12,155 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""UI hierarchy traversal and element geometry primitives for ARTEMIS. - -Provides structured spatial coordinates, bounding box metrics, and recursive -node querying across Android accessibility hierarchies and UIAutomator tree dumps. -""" - -from __future__ import annotations - -from typing import Any -from pydantic import BaseModel, ConfigDict, Field - from artemis.utils.logger import get_logger +from pydantic import BaseModel, Field logger = get_logger(__name__) +def __find_element_by_resource_id_in_rich_hierarchy( + hierarchy: list[dict], resource_id: str +) -> dict | None: + """Retrieves all the sibling elements for a given resource ID from a nested dictionary. + + Args: + hierarchy (dict): The nested dictionary representing the UI hierarchy. + resource_id (str): The resource-id to find. + + Returns: + list: A list of the sibling elements, or None if the resource_id is not + found. + """ + if not hierarchy: + return None + + for child in hierarchy: + if child.get("attributes", {}).get("resource-id") == resource_id: + return child.get("attributes", {}) + + for child in hierarchy: + result = __find_element_by_resource_id_in_rich_hierarchy( + child.get("children", []), resource_id + ) + if result is not None: + return result + + return None + + +def text_input_is_empty(text: str | None, hint_text: str | None) -> bool: + return not text or text == hint_text + + +def find_element_by_resource_id( + ui_hierarchy: list[dict], + resource_id: str, + index: int | None = None, + is_rich_hierarchy: bool = False, +) -> dict | None: + """Find a UI element by its resource-id in the UI hierarchy. + + Args: + ui_hierarchy: List of UI element dictionaries + resource_id: The resource-id to search for (e.g., + "com.google.android.settings.intelligence:id/open_search_view_edit_text") + + Returns: + The complete UI element dictionary if found, None otherwise + """ + if is_rich_hierarchy: + return __find_element_by_resource_id_in_rich_hierarchy(ui_hierarchy, resource_id) + + target_idx = index or 0 + + def search_recursive(elements: list[dict]) -> dict | None: + nonlocal target_idx + for element in elements: + if isinstance(element, dict): + if element.get("resourceId") == resource_id: + if target_idx == 0: + return element + target_idx -= 1 + continue + + children = element.get("children", []) + if children: + result = search_recursive(children) + if result: + return result + return None + + return search_recursive(ui_hierarchy) + + +def is_element_focused(element: dict) -> bool: + return element.get("focused", None) == "true" + + +def get_element_text(element: dict, hint_text: bool = False) -> str | None: + if hint_text: + return element.get("hintText", None) + return element.get("text", None) + + +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass + + class Point(BaseModel): - """Discrete pixel coordinate on screen.""" - - model_config = ConfigDict(frozen=True) - + model_config = {"ignored_types": (CyFunctionDetector,)} x: int y: int class ElementBounds(BaseModel): - """Axis-aligned rectangular boundary defining an on-screen element.""" - - model_config = ConfigDict(extra="ignore") - - x: int = Field(description="Horizontal offset of the top-left corner in pixels.") - y: int = Field(description="Vertical offset of the top-left corner in pixels.") - width: int = Field(description="Horizontal extent of the bounding box.") - height: int = Field(description="Vertical extent of the bounding box.") + model_config = {"ignored_types": (CyFunctionDetector,)} + x: int = Field(description="The x coordinate of the top-left corner of the element.") + y: int = Field(description="The y coordinate of the top-left corner of the element.") + width: int = Field(description="The width of the element.") + height: int = Field(description="The height of the element.") def get_center(self) -> Point: - """Compute the centroid of the element boundary.""" - return Point(x=self.x + (self.width // 2), y=self.y + (self.height // 2)) + return Point(x=self.x + self.width // 2, y=self.y + self.height // 2) def get_relative_point(self, x_percent: float, y_percent: float) -> Point: - """Compute an internal pixel coordinate by fractional offsets [0.0, 1.0].""" - clamped_x = max(0.0, min(1.0, float(x_percent))) - clamped_y = max(0.0, min(1.0, float(y_percent))) + """Returns the coordinates of the point at x_percent of the width and y_percent + + of the height of the element. + + Ex if x_percent = 0.95 and y_percent = 0.95, + the point is at the bottom right of the element: + <------> + | | + | x| + <------> + """ return Point( - x=int(self.x + (self.width * clamped_x)), - y=int(self.y + (self.height * clamped_y)), + x=int(self.x + self.width * x_percent), + y=int(self.y + self.height * y_percent), ) -def get_bounds_for_element(element: dict[str, Any]) -> ElementBounds | None: - """Extract and validate the bounding box from a node descriptor.""" - raw_bounds = element.get("bounds") - if not isinstance(raw_bounds, dict): - return None - - try: - return ElementBounds( - x=int(raw_bounds["x"]), - y=int(raw_bounds["y"]), - width=int(raw_bounds["width"]), - height=int(raw_bounds["height"]), - ) - except (KeyError, TypeError, ValueError) as exc: - logger.error(f"Failed to validate element bounds ({raw_bounds}): {exc}") - return None - - -def text_input_is_empty(text: str | None, hint_text: str | None) -> bool: - """Determine whether an editable field contains user-entered text.""" - if not text: - return True - return text == hint_text - - -def is_element_focused(element: dict[str, Any]) -> bool: - """Check if the given node currently holds system focus.""" - focused_val = element.get("focused") - if isinstance(focused_val, bool): - return focused_val - return str(focused_val).strip().lower() == "true" - - -def get_element_text(element: dict[str, Any], hint_text: bool = False) -> str | None: - """Retrieve textual label or placeholder from an element node.""" - if hint_text: - return element.get("hintText") - return element.get("text") - - -def find_element_by_resource_id( - ui_hierarchy: list[dict[str, Any]], - resource_id: str, - index: int | None = None, - is_rich_hierarchy: bool = False, -) -> dict[str, Any] | None: - """Locate an element in the hierarchy tree matching the specified resource ID. - - Supports both flat attribute mappings and rich accessibility tree hierarchies. - """ - if not ui_hierarchy or not resource_id: - return None - - if is_rich_hierarchy: - return _search_rich_tree(ui_hierarchy, resource_id) - - target_index = max(0, index or 0) - current_match = 0 - - def _walk(nodes: list[dict[str, Any]]) -> dict[str, Any] | None: - nonlocal current_match - for node in nodes: - if not isinstance(node, dict): - continue - - if node.get("resourceId") == resource_id: - if current_match == target_index: - return node - current_match += 1 - - children = node.get("children") - if isinstance(children, list) and children: - found = _walk(children) - if found is not None: - return found - return None - - return _walk(ui_hierarchy) - - -def _search_rich_tree( - nodes: list[dict[str, Any]], resource_id: str -) -> dict[str, Any] | None: - """Recursively traverse a rich XML attribute tree for resource ID match.""" - for node in nodes: - if not isinstance(node, dict): - continue - - attributes = node.get("attributes") - if isinstance(attributes, dict) and attributes.get("resource-id") == resource_id: - return attributes - - children = node.get("children") - if isinstance(children, list) and children: - sub = _search_rich_tree(children, resource_id) - if sub is not None: - return sub - +def get_bounds_for_element(element: dict) -> ElementBounds | None: + bounds = element.get("bounds", None) + if bounds: + try: + return ElementBounds(**bounds) + except Exception as e: + logger.error(f"Failed to validate bounds: {e}") + return None return None diff --git a/artemis/utils/video.py b/artemis/utils/video.py index 630bc55..a81bedd 100644 --- a/artemis/utils/video.py +++ b/artemis/utils/video.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. """Video recording utilities for mobile devices. @@ -126,6 +123,24 @@ async def await_scrcpy_first_frame( return max(spawned_at, time.time()) +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass + + class RecordingSession(BaseModel): """Tracks an active video recording session.""" diff --git a/config/artemis.jsonc b/config/artemis.jsonc index 98ce44b..517d738 100644 --- a/config/artemis.jsonc +++ b/config/artemis.jsonc @@ -93,8 +93,8 @@ // 📋 Summarizer: Aggregates step history and generates structured outcome "summarizer": {}, - // 🔍 Entity Extractor: Fast lightweight package name locator and entity extractor - "entity_extractor": { + // 🔽 Hopper: Fast lightweight package name locator + "hopper": { "model": "gemini-3.5-flash-lite", "fallback": { "provider": "google", diff --git a/tests/tools/entity_extractor/test_entity_extractor_tools.py b/tests/tools/hopper/test_hopper_tools.py similarity index 89% rename from tests/tools/entity_extractor/test_entity_extractor_tools.py rename to tests/tools/hopper/test_hopper_tools.py index d9f2bf7..deb7fb7 100644 --- a/tests/tools/entity_extractor/test_entity_extractor_tools.py +++ b/tests/tools/hopper/test_hopper_tools.py @@ -14,6 +14,6 @@ """This file is intentionally empty. -The entity_extractor agent does not expose any tools directly. It is invoked +The hopper agent does not expose any tools directly. It is invoked programmatically by other tools (e.g., launch_app_tool) as an LLM utility. """ diff --git a/tests/unit/agents/test_entity_extractor.py b/tests/unit/agents/test_entity_extractor.py deleted file mode 100644 index 5cb8785..0000000 --- a/tests/unit/agents/test_entity_extractor.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit tests for Artemis entity extraction and package resolver.""" - -from unittest.mock import AsyncMock, MagicMock, patch -import pytest - -from artemis.agents.entity_extractor.entity_extractor import ( - ExtractionResult, - extract_entity, -) - - -def test_extraction_result_model(): - """Verify ExtractionResult attributes and validation.""" - res = ExtractionResult(found=True, output="com.google.android.apps.nexuslauncher", reason="Exact match") - assert res.found is True - assert res.output == "com.google.android.apps.nexuslauncher" - assert res.reason == "Exact match" - - -@pytest.mark.asyncio -async def test_extract_entity_success(): - """Verify successful LLM entity extraction invocation.""" - mock_ctx = MagicMock() - mock_llm = MagicMock() - expected_result = ExtractionResult( - found=True, - output="com.google.android.calculator", - reason="Found calculator package in list", - ) - mock_llm.ainvoke = AsyncMock(return_value=expected_result) - - with patch("artemis.agents.entity_extractor.entity_extractor.get_llm") as mock_get_llm: - mock_get_llm.return_value.with_structured_output.return_value = mock_llm - result = await extract_entity( - request="Calculator", - data="com.google.android.calculator\ncom.google.android.calendar", - ctx=mock_ctx, - ) - - assert result.found is True - assert result.output == "com.google.android.calculator" - assert result.reason == "Found calculator package in list" - - -@pytest.mark.asyncio -async def test_extract_entity_handles_exception(): - """Verify extraction returns a graceful failure result when LLM throws an exception.""" - mock_ctx = MagicMock() - mock_llm = MagicMock() - mock_llm.ainvoke = AsyncMock(side_effect=RuntimeError("LLM connection timed out")) - - with patch("artemis.agents.entity_extractor.entity_extractor.get_llm") as mock_get_llm: - mock_get_llm.return_value.with_structured_output.return_value = mock_llm - result = await extract_entity( - request="Settings", - data="com.android.settings", - ctx=mock_ctx, - ) - - assert result.found is False - assert result.output is None - assert "LLM extraction error" in result.reason diff --git a/tests/unit/agents/test_outputter.py b/tests/unit/agents/test_outputter.py index ab05f5d..668e5b3 100644 --- a/tests/unit/agents/test_outputter.py +++ b/tests/unit/agents/test_outputter.py @@ -26,7 +26,26 @@ from artemis.utils.logger import get_logger # noqa: E402 logger = get_logger(__name__) +class _CyFunctionDetectorMeta(type): + def __instancecheck__(self, instance): + name = type(instance).__name__ + return ( + name + in ( + "cyfunction", + "cython_function_or_method", + "builtin_function_or_method", + ) + or "cyfunction" in name.lower() + ) + + +class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): + pass + + class MockPydanticSchema(BaseModel): + model_config = {"ignored_types": (CyFunctionDetector,)} color: str price: float currency_symbol: str diff --git a/tests/unit/mcp/test_device_utils.py b/tests/unit/mcp/test_device_utils.py index adf4a78..02fa97f 100644 --- a/tests/unit/mcp/test_device_utils.py +++ b/tests/unit/mcp/test_device_utils.py @@ -9,8 +9,6 @@ from mcp_server.utils import device_utils def test_ensure_emulator_uses_windows_creation_flags(monkeypatch) -> None: popen = MagicMock() monkeypatch.setattr(device_utils.sys, "platform", "win32") - monkeypatch.setattr(device_utils.subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200, raising=False) - monkeypatch.setattr(device_utils.subprocess, "DETACHED_PROCESS", 0x00000008, raising=False) monkeypatch.setattr(device_utils, "is_emulator_running", lambda _adb: False) monkeypatch.setattr(device_utils.os.path, "exists", lambda _path: True) monkeypatch.setattr(device_utils.subprocess, "Popen", popen) @@ -22,7 +20,7 @@ def test_ensure_emulator_uses_windows_creation_flags(monkeypatch) -> None: monkeypatch.setattr(device_utils.time, "sleep", lambda _seconds: None) assert device_utils.ensure_emulator( - adb_path="adb.exe", emulator_path="emulator.exe", timeout_seconds=10 + adb_path="adb.exe", emulator_path="emulator.exe", timeout_seconds=1 ) kwargs = popen.call_args.kwargs @@ -45,7 +43,7 @@ def test_ensure_emulator_starts_new_session_on_posix(monkeypatch) -> None: ) monkeypatch.setattr(device_utils.time, "sleep", lambda _seconds: None) - assert device_utils.ensure_emulator(adb_path="adb", emulator_path="emulator", timeout_seconds=10) + assert device_utils.ensure_emulator(adb_path="adb", emulator_path="emulator", timeout_seconds=1) kwargs = popen.call_args.kwargs assert kwargs["start_new_session"] is True diff --git a/tests/unit/mcp/test_mcp_stdio_lifecycle.py b/tests/unit/mcp/test_mcp_stdio_lifecycle.py index 472bc01..7861f7a 100644 --- a/tests/unit/mcp/test_mcp_stdio_lifecycle.py +++ b/tests/unit/mcp/test_mcp_stdio_lifecycle.py @@ -24,8 +24,11 @@ import queue import subprocess import sys import threading +import time from unittest.mock import MagicMock, patch +import pytest + from artemis.clients import ui_automator_client from artemis.runtime.awake_lease import ScreenAwakeLease from artemis.runtime.awake_service import _run_awake_adb_command @@ -201,12 +204,16 @@ def test_device_utils_isolates_stdin(): def test_ui_automator_client_isolates_stdin(): - """Verify ui_automator_client screencap commands isolate stdin.""" - client = ui_automator_client.UIAutomatorClient("dev-1") + """Verify ui_automator_client helper commands isolate stdin.""" with patch("artemis.clients.ui_automator_client.subprocess.run") as mock_run: - mock_run.return_value = MagicMock(returncode=0, stdout=b"", stderr="") - with patch("artemis.clients.ui_automator_client.Image.open"): - client.get_screenshot() + mock_run.return_value = MagicMock(returncode=0, stdout="package:com.test\n", stderr="") + ui_automator_client._is_package_installed("dev-1", "com.test") + assert mock_run.called + assert mock_run.call_args.kwargs.get("stdin") == subprocess.DEVNULL + + with patch("artemis.clients.ui_automator_client.subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") + ui_automator_client._uninstall_package("dev-1", "com.test") assert mock_run.called assert mock_run.call_args.kwargs.get("stdin") == subprocess.DEVNULL diff --git a/tests/unit/sdk/test_exceptions.py b/tests/unit/sdk/test_exceptions.py deleted file mode 100644 index 5df5415..0000000 --- a/tests/unit/sdk/test_exceptions.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit tests validating Artemis SDK exception hierarchy and diagnostics.""" - -from artemis.sdk.types.exceptions import ( - AgentError, - AgentNotInitializedError, - AgentProfileNotFoundError, - AgentTaskRequestError, - ArtemisError, - DeviceError, - DeviceNotFoundError, - ExecutableNotFoundError, - ServerError, - ServerStartupError, -) - - -def test_exception_hierarchy(): - """Verify all custom exceptions inherit properly from ArtemisError.""" - assert issubclass(DeviceError, ArtemisError) - assert issubclass(DeviceNotFoundError, DeviceError) - assert issubclass(ServerError, ArtemisError) - assert issubclass(ServerStartupError, ServerError) - assert issubclass(AgentError, ArtemisError) - assert issubclass(AgentNotInitializedError, AgentError) - assert issubclass(AgentTaskRequestError, AgentError) - assert issubclass(AgentProfileNotFoundError, AgentTaskRequestError) - assert issubclass(ExecutableNotFoundError, ArtemisError) - - -def test_executable_not_found_error_adb_guidance(): - """Verify executable error formats download instructions for known tools.""" - err = ExecutableNotFoundError("adb") - assert "adb" in str(err) - assert "developer.android.com" in str(err) - assert err.executable_name == "adb" - - -def test_server_startup_error_formatting(): - """Verify server startup error encapsulates service identifier.""" - err = ServerStartupError(server_name="mcp_daemon") - assert "mcp_daemon" in str(err) - assert err.server_name == "mcp_daemon" - - -def test_agent_profile_not_found_error(): - """Verify profile error retains profile name.""" - err = AgentProfileNotFoundError("nonexistent_profile") - assert "nonexistent_profile" in str(err) - assert err.profile_name == "nonexistent_profile" diff --git a/tests/unit/sdk/test_task_request_builder.py b/tests/unit/sdk/test_task_request_builder.py deleted file mode 100644 index e90a4b2..0000000 --- a/tests/unit/sdk/test_task_request_builder.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit tests for Artemis SDK TaskRequestBuilder.""" - -from pathlib import Path -from pydantic import BaseModel -import pytest - -from artemis.sdk.builders.task_request_builder import TaskRequestBuilder, TaskRequestCommonBuilder -from artemis.sdk.types.task import TaskRequest - - -class DummyOutputSchema(BaseModel): - summary: str - item_count: int - - -def test_task_request_builder_fluent_chaining(): - """Verify basic fluent configuration and build.""" - req = ( - TaskRequestBuilder(goal="Open Settings and verify battery level") - .with_max_steps(25) - .with_trace_recording(True, path="custom_traces") - .with_locked_app_package("com.android.settings") - .with_app_path("build/app.apk") - .using_profile("flash") - .with_name("Battery Check") - .build() - ) - - assert isinstance(req, TaskRequest) - assert req.goal == "Open Settings and verify battery level" - assert req.max_steps == 25 - assert req.record_trace is True - assert req.trace_path == Path("custom_traces") - assert req.locked_app_package == "com.android.settings" - assert req.app_path == Path("build/app.apk") - assert req.profile == "flash" - assert req.task_name == "Battery Check" - - -def test_task_request_builder_with_output_format(): - """Verify builder binds structured schema properly.""" - req = ( - TaskRequestBuilder(goal="Extract items") - .with_output_format(DummyOutputSchema) - .build() - ) - - assert req.output_format is DummyOutputSchema - assert req.output_description is None - - -def test_task_request_builder_with_output_description(): - """Verify builder accepts unformatted string deliverables.""" - req = ( - TaskRequestBuilder(goal="Extract items") - .with_output_description("Return a numbered list") - .build() - ) - - assert req.output_description == "Return a numbered list" - assert req.output_format is None - - -def test_task_request_builder_conflicting_outputs_raises(): - """Verify that specifying both output_format and output_description raises ValueError.""" - builder = ( - TaskRequestBuilder(goal="Extract items") - .with_output_format(DummyOutputSchema) - .with_output_description("Description text") - ) - with pytest.raises(ValueError, match="Cannot specify both output_format and output_description"): - builder.build() - - -def test_task_request_builder_empty_goal_raises(): - """Verify empty or whitespace-only goal raises ValueError.""" - with pytest.raises(ValueError, match="Task goal must be a non-empty"): - TaskRequestBuilder(goal=" ").build() - - -def test_task_request_builder_from_common(): - """Verify task request initialization derived from common base settings.""" - common_settings = ( - TaskRequestCommonBuilder() - .with_max_steps(42) - .with_trace_recording(False) - .with_locked_app_package("org.artemis.target") - .with_app_path(Path("/tmp/target.apk")) - .build() - ) - - derived = TaskRequestBuilder.from_common("Derived goal", common_settings).build() - assert derived.goal == "Derived goal" - assert derived.max_steps == 42 - assert derived.record_trace is False - assert derived.locked_app_package == "org.artemis.target" - assert derived.app_path == Path("/tmp/target.apk") diff --git a/tests/unit/test_config_package.py b/tests/unit/test_config_package.py index 313d322..5b8c56a 100644 --- a/tests/unit/test_config_package.py +++ b/tests/unit/test_config_package.py @@ -154,7 +154,7 @@ def test_llm_config_parsing_and_merging(): assert isinstance(llm_cfg, LLMConfig) assert llm_cfg.planner.provider in ("google", "openai", "openrouter", "xai", "vertexai") assert llm_cfg.get_agent("planner") is not None - assert llm_cfg.get_utils("entity_extractor") is not None + assert llm_cfg.get_utils("hopper") is not None # Deep merge overrides overrides = { diff --git a/tests/unit/test_ui_automator_client_awake.py b/tests/unit/test_ui_automator_client_awake.py index 19152a2..12e3a82 100644 --- a/tests/unit/test_ui_automator_client_awake.py +++ b/tests/unit/test_ui_automator_client_awake.py @@ -11,14 +11,16 @@ from artemis.drivers.android.adb_driver import AndroidAdbDriver "artemis.clients.ui_automator_client.ensure_device_awake", return_value="host_heartbeat", ) +@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed") def test_new_ui_connection_enrolls_device_in_shared_awake_strategy( - mock_ensure_awake, mock_connect + mock_remove_maestro, mock_ensure_awake, mock_connect ): client = UIAutomatorClient("device-123") client.connect() client.connect() + mock_remove_maestro.assert_called_once_with("device-123") mock_ensure_awake.assert_called_once_with("device-123") mock_connect.assert_called_once_with("device-123") @@ -28,8 +30,9 @@ def test_new_ui_connection_enrolls_device_in_shared_awake_strategy( "artemis.clients.ui_automator_client.ensure_device_awake", return_value="host_heartbeat", ) +@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed") def test_failed_ui_connection_does_not_stop_process_awake_service( - mock_ensure_awake, _mock_connect + _mock_remove_maestro, mock_ensure_awake, _mock_connect ): client = UIAutomatorClient("device-123") @@ -45,8 +48,9 @@ def test_failed_ui_connection_does_not_stop_process_awake_service( "artemis.clients.ui_automator_client.ensure_device_awake", return_value="host_heartbeat", ) +@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed") def test_client_disconnect_does_not_send_power_cleanup_commands( - mock_ensure_awake, _mock_connect + _mock_remove_maestro, mock_ensure_awake, _mock_connect ): client = UIAutomatorClient("device-123") client.connect() diff --git a/tests/unit/tools/test_launch_app.py b/tests/unit/tools/test_launch_app.py index c2c07bb..bc45611 100644 --- a/tests/unit/tools/test_launch_app.py +++ b/tests/unit/tools/test_launch_app.py @@ -13,11 +13,10 @@ # limitations under the License. from unittest.mock import MagicMock, patch -import pytest - -from artemis.agents.entity_extractor.entity_extractor import ExtractionResult +from artemis.agents.hopper.hopper import HopperOutput from artemis.context import ArtemisContext from artemis.tools.mobile.launch_app import find_package +import pytest @pytest.fixture @@ -29,63 +28,63 @@ def mock_context(): @pytest.mark.asyncio @patch("artemis.tools.mobile.launch_app.list_packages_async") -@patch("artemis.tools.mobile.launch_app.extract_entity") -async def test_find_package_cache_hit(mock_extract_entity, mock_list_packages, mock_context): +@patch("artemis.tools.mobile.launch_app.hopper") +async def test_find_package_cache_hit(mock_hopper, mock_list_packages, mock_context): mock_list_packages.return_value = "com.example.app1\ncom.example.app2" - # First call: not in cache, calls extract_entity - mock_extract_entity.return_value = ExtractionResult( + # First call: not in cache, calls hopper + mock_hopper.return_value = HopperOutput( found=True, output="com.example.app1", reason="Found it" ) pkg1 = await find_package(mock_context, "App 1") assert pkg1 == "com.example.app1" - assert mock_extract_entity.call_count == 1 + assert mock_hopper.call_count == 1 assert mock_context.package_cache["App 1"] == "com.example.app1" - # Second call: cache hit, does not call extract_entity or list_packages again - mock_extract_entity.reset_mock() + # Second call: cache hit, does not call hopper or list_packages again + mock_hopper.reset_mock() mock_list_packages.reset_mock() pkg2 = await find_package(mock_context, "App 1") assert pkg2 == "com.example.app1" - mock_extract_entity.assert_not_called() + mock_hopper.assert_not_called() mock_list_packages.assert_not_called() @pytest.mark.asyncio @patch("artemis.tools.mobile.launch_app.list_packages_async") -@patch("artemis.tools.mobile.launch_app.extract_entity") -async def test_find_package_not_found_caches_none(mock_extract_entity, mock_list_packages, mock_context): +@patch("artemis.tools.mobile.launch_app.hopper") +async def test_find_package_not_found_caches_none(mock_hopper, mock_list_packages, mock_context): mock_list_packages.return_value = "com.example.app1\ncom.example.app2" - # Extractor doesn't find it - mock_extract_entity.return_value = ExtractionResult(found=False, output=None, reason="Not found") + # Hopper doesn't find it + mock_hopper.return_value = HopperOutput(found=False, output=None, reason="Not found") pkg1 = await find_package(mock_context, "App 3") assert pkg1 is None - assert mock_extract_entity.call_count == 1 + assert mock_hopper.call_count == 1 assert mock_context.package_cache["App 3"] is None - # Second call: cache hit (None), does not call extract_entity again - mock_extract_entity.reset_mock() + # Second call: cache hit (None), does not call hopper again + mock_hopper.reset_mock() pkg2 = await find_package(mock_context, "App 3") assert pkg2 is None - mock_extract_entity.assert_not_called() + mock_hopper.assert_not_called() @pytest.mark.asyncio @patch("artemis.tools.mobile.launch_app.list_packages_async") -@patch("artemis.tools.mobile.launch_app.extract_entity") -async def test_find_package_passes_use_fallback(mock_extract_entity, mock_list_packages, mock_context): +@patch("artemis.tools.mobile.launch_app.hopper") +async def test_find_package_passes_use_fallback(mock_hopper, mock_list_packages, mock_context): mock_list_packages.return_value = "com.example.app1\ncom.example.app2" - mock_extract_entity.return_value = ExtractionResult( + mock_hopper.return_value = HopperOutput( found=True, output="com.example.app1", reason="Found it" ) # Pass use_fallback=False await find_package(mock_context, "App 1", use_fallback=False) - mock_extract_entity.assert_called_once_with( + mock_hopper.assert_called_once_with( ctx=mock_context, request=("I'm looking for the package name of the following app: 'App 1'"), data="com.example.app1\ncom.example.app2", diff --git a/tests/unit/tools/test_utils.py b/tests/unit/tools/test_utils.py index 7f067a7..63b9fd9 100644 --- a/tests/unit/tools/test_utils.py +++ b/tests/unit/tools/test_utils.py @@ -12,340 +12,446 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Unit tests for ARTEMIS tool utility functions and element interaction helpers.""" - -from __future__ import annotations - -from unittest.mock import AsyncMock, MagicMock, patch +import asyncio +import sys +from unittest.mock import Mock, patch import pytest -from artemis.context import ArtemisContext, DeviceContext, DevicePlatform -from artemis.controllers.types import CoordinatesSelectorRequest -from artemis.tools.types import Target -from artemis.tools.utils import ( +# Mock the problematic langgraph import at module level +_orig_chat_agent_executor = sys.modules.get("langgraph.prebuilt.chat_agent_executor") +_orig_state = sys.modules.get("artemis.graph.state") + +sys.modules["langgraph.prebuilt.chat_agent_executor"] = Mock() +sys.modules["artemis.graph.state"] = Mock() + +from artemis.context import DeviceContext, DevicePlatform, ArtemisContext # noqa: E402 +from artemis.tools.types import Target # noqa: E402 +from artemis.tools.utils import ( # noqa: E402 IdSelectorRequest, - IdWithTextSelectorRequest, SelectorRequestWithCoordinates, - SelectorRequestWithPercentages, - TextSelectorRequest, - _extract_resource_id_and_text_from_selector, - find_element_by_text, focus_element_if_needed, - has_valid_selectors, move_cursor_to_end_if_bounds, - tap_bottom_right_of_element, - validate_coordinates_bounds, ) -from artemis.utils.ui_hierarchy import ElementBounds +from artemis.utils.ui_hierarchy import ElementBounds # noqa: E402 + +# Restore original modules to prevent contaminating other tests +if _orig_chat_agent_executor is not None: + sys.modules["langgraph.prebuilt.chat_agent_executor"] = _orig_chat_agent_executor +else: + del sys.modules["langgraph.prebuilt.chat_agent_executor"] + +if _orig_state is not None: + sys.modules["artemis.graph.state"] = _orig_state +else: + del sys.modules["artemis.graph.state"] @pytest.fixture -def dummy_context() -> MagicMock: - """Fixture providing an ARTEMIS runtime context configured for Android.""" - ctx = MagicMock(spec=ArtemisContext) - ctx.device = MagicMock(spec=DeviceContext) +def mock_context(): + """Create a mock ArtemisContext for testing.""" + ctx = Mock(spec=ArtemisContext) + + # Create device context with necessary attributes + ctx.device = Mock(spec=DeviceContext) ctx.device.mobile_platform = DevicePlatform.ANDROID - ctx.device.device_id = "test-device-pixel8" + ctx.device.device_id = "test_device_123" ctx.device.device_width = 1080 - ctx.device.device_height = 2400 + ctx.device.device_height = 2340 + ctx.device.host_platform = "LINUX" + + ctx.ui_adb_client = Mock() + + # Mock the ADB client for Android + ctx.adb_client = Mock() + mock_device = Mock() + mock_device.shell = Mock(return_value="") + ctx.adb_client.device = Mock(return_value=mock_device) + + # Mock the ADB client for Android + mock_response = Mock() + mock_response.json.return_value = {"elements": []} + ctx.ui_adb_client.get_screen_data = Mock(return_value=mock_response) + ctx.ui_adb_client.get_hierarchy = Mock( + return_value=( + b'' + ) + ) + return ctx @pytest.fixture -def mock_graph_state() -> MagicMock: - """Fixture providing a mock LangGraph state container.""" - state = MagicMock() +def mock_state(): + """Create a mock State for testing.""" + state = Mock() state.latest_ui_hierarchy = [] return state @pytest.fixture -def sample_hierarchy_node() -> dict: - """Fixture providing a standard flat hierarchy node.""" +def sample_element(): + """Create a sample UI element for testing.""" return { - "resourceId": "com.google.android.apps:id/search_query", - "text": "Search Google", - "bounds": {"x": 120, "y": 250, "width": 400, "height": 80}, + "resourceId": "com.example:id/text_input", + "text": "Sample text", + "bounds": {"x": 100, "y": 200, "width": 300, "height": 50}, "focused": "false", } @pytest.fixture -def sample_nested_node() -> dict: - """Fixture providing a rich nested hierarchy node.""" +def sample_rich_element(): + """Create a sample rich UI element for testing.""" return { "attributes": { - "resource-id": "com.google.android.apps:id/search_query", - "text": "Search Google", - "bounds": {"x": 120, "y": 250, "width": 400, "height": 80}, + "resource-id": "com.example:id/text_input", "focused": "false", + "text": "Sample text", + "bounds": {"x": 100, "y": 200, "width": 300, "height": 50}, }, "children": [], } -# ============================================================================== -# Cursor Placement & Boundary Alignment Tests -# ============================================================================== +class TestMoveCursorToEndIfBounds: + """Test cases for move_cursor_to_end_if_bounds function.""" - -class TestCursorAlignment: - """Validates precise cursor placement against interactive targets.""" - - @pytest.mark.asyncio - @patch("artemis.tools.utils.tap", new_callable=AsyncMock) + @patch("artemis.tools.utils.tap") @patch("artemis.tools.utils.find_element_by_resource_id") - async def test_cursor_routes_to_end_via_resource_id( + def test_move_cursor_with_resource_id( self, - mock_find_elem: MagicMock, - mock_tap_fn: AsyncMock, - dummy_context: MagicMock, - mock_graph_state: MagicMock, - sample_hierarchy_node: dict, + mock_find_element, + mock_tap, + mock_context, + mock_state, + sample_element, ): - """Resource ID locator should calculate end-boundary tap (99% width/height offset).""" - mock_graph_state.latest_ui_hierarchy = [sample_hierarchy_node] - mock_find_elem.return_value = sample_hierarchy_node + """Test moving cursor using resource_id (highest priority).""" + mock_state.latest_ui_hierarchy = [sample_element] + mock_find_element.return_value = sample_element - target = Target(resource_id="com.google.android.apps:id/search_query") - elem = await move_cursor_to_end_if_bounds( - ctx=dummy_context, state=mock_graph_state, target=target + target = Target( + resource_id="com.example:id/text_input", + resource_id_index=None, + text=None, + text_index=None, + bounds=None, + ) + result = asyncio.run( + move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) ) - mock_find_elem.assert_called_once_with( - ui_hierarchy=[sample_hierarchy_node], - resource_id="com.google.android.apps:id/search_query", + mock_find_element.assert_called_once_with( + ui_hierarchy=[sample_element], + resource_id="com.example:id/text_input", index=0, ) - mock_tap_fn.assert_awaited_once() - kwargs = mock_tap_fn.await_args.kwargs - selector_req = kwargs["selector_request"] - assert isinstance(selector_req, SelectorRequestWithCoordinates) - assert selector_req.coordinates.x == 516 - assert selector_req.coordinates.y == 329 - assert elem == sample_hierarchy_node + mock_tap.assert_called_once() + call_args = mock_tap.call_args[1] + selector_request = call_args["selector_request"] + assert isinstance(selector_request, SelectorRequestWithCoordinates) + coords = selector_request.coordinates + assert coords.x == 397 # 100 + 300 * 0.99 + assert coords.y == 249 # 200 + 50 * 0.99 + assert result == sample_element - @pytest.mark.asyncio - @patch("artemis.tools.utils.tap", new_callable=AsyncMock) + @patch("artemis.tools.utils.tap") @patch("artemis.tools.utils.find_element_by_resource_id") - async def test_cursor_routes_using_direct_bounds( - self, - mock_find_elem: MagicMock, - mock_tap_fn: AsyncMock, - dummy_context: MagicMock, - mock_graph_state: MagicMock, + def test_move_cursor_with_coordinates_only( + self, mock_find_element, mock_tap, mock_context, mock_state ): - """Direct bounding box coordinates should bypass hierarchy inspection.""" - bounds = ElementBounds(x=80, y=100, width=200, height=60) - target = Target(bounds=bounds) - - result = await move_cursor_to_end_if_bounds( - ctx=dummy_context, state=mock_graph_state, target=target + """Test moving cursor when only coordinates are provided.""" + bounds = ElementBounds(x=50, y=150, width=200, height=40) + target = Target( + resource_id=None, + resource_id_index=None, + text=None, + text_index=None, + bounds=bounds, ) - mock_find_elem.assert_not_called() - mock_tap_fn.assert_awaited_once() - kwargs = mock_tap_fn.await_args.kwargs - selector_req = kwargs["selector_request"] - assert selector_req.coordinates.x == 278 - assert selector_req.coordinates.y == 159 - assert result is None - - @pytest.mark.asyncio - @patch("artemis.tools.utils.tap", new_callable=AsyncMock) - @patch("artemis.tools.utils.find_element_by_text") - async def test_cursor_routes_by_matched_text( - self, - mock_find_text: MagicMock, - mock_tap_fn: AsyncMock, - dummy_context: MagicMock, - mock_graph_state: MagicMock, - sample_hierarchy_node: dict, - ): - """Visible text locator should successfully resolve bounds and trigger cursor placement.""" - mock_graph_state.latest_ui_hierarchy = [sample_hierarchy_node] - mock_find_text.return_value = sample_hierarchy_node - - target = Target(text="Search Google", text_index=0) - result = await move_cursor_to_end_if_bounds( - ctx=dummy_context, state=mock_graph_state, target=target + result = asyncio.run( + move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) ) - mock_find_text.assert_called_once_with([sample_hierarchy_node], "Search Google", index=0) - mock_tap_fn.assert_awaited_once() - assert result == sample_hierarchy_node + mock_find_element.assert_not_called() + mock_tap.assert_called_once() + call_args = mock_tap.call_args[1] + selector_request = call_args["selector_request"] + coords = selector_request.coordinates + assert coords.x == 248 # 50 + 200 * 0.99 + assert coords.y == 189 # 150 + 40 * 0.99 + assert result is None # No element is returned when using coords directly - @pytest.mark.asyncio - @patch("artemis.tools.utils.tap", new_callable=AsyncMock) + @patch("artemis.tools.utils.tap") @patch("artemis.tools.utils.find_element_by_text") - async def test_cursor_skips_when_text_unmatched( - self, - mock_find_text: MagicMock, - mock_tap_fn: AsyncMock, - dummy_context: MagicMock, - mock_graph_state: MagicMock, + def test_move_cursor_with_text_only_success( + self, mock_find_text, mock_tap, mock_context, mock_state, sample_element ): - """Unresolved text target returns None without tapping.""" + """Test moving cursor when only text is provided and succeeds.""" + mock_state.latest_ui_hierarchy = [sample_element] + mock_find_text.return_value = sample_element + + target = Target( + resource_id=None, + resource_id_index=None, + text="Sample text", + text_index=0, + bounds=None, + ) + result = asyncio.run( + move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) + ) + + mock_find_text.assert_called_once_with([sample_element], "Sample text", index=0) + mock_tap.assert_called_once() + assert result == sample_element + + @patch("artemis.tools.utils.tap") + @patch("artemis.tools.utils.find_element_by_text") + def test_move_cursor_with_text_only_element_not_found( + self, mock_find_text, mock_tap, mock_context, mock_state + ): + """Test when searching by text finds no element.""" + mock_state.latest_ui_hierarchy = [] mock_find_text.return_value = None - target = Target(text="Unmatched Query") - result = await move_cursor_to_end_if_bounds( - ctx=dummy_context, state=mock_graph_state, target=target + target = Target( + resource_id=None, + resource_id_index=None, + text="Nonexistent text", + text_index=None, + bounds=None, ) - mock_tap_fn.assert_not_awaited() + result = asyncio.run( + move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) + ) + + mock_tap.assert_not_called() + assert result is None + + @patch("artemis.tools.utils.tap") + @patch("artemis.tools.utils.find_element_by_text") + def test_move_cursor_with_text_only_no_bounds( + self, mock_find_text, mock_tap, mock_context, mock_state + ): + """Test when element is found by text but has no bounds.""" + element_no_bounds = {"text": "Text without bounds"} + mock_state.latest_ui_hierarchy = [element_no_bounds] + mock_find_text.return_value = element_no_bounds + + target = Target( + resource_id=None, + resource_id_index=None, + text="Text without bounds", + text_index=None, + bounds=None, + ) + result = asyncio.run( + move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) + ) + + mock_tap.assert_not_called() + assert result is None # Should return None as no action was taken + + @patch("artemis.tools.utils.find_element_by_resource_id") + def test_move_cursor_element_not_found_by_id(self, mock_find_element, mock_context, mock_state): + """Test when element is not found by resource_id.""" + mock_find_element.return_value = None + + target = Target( + resource_id="com.example:id/nonexistent", + resource_id_index=None, + text=None, + text_index=None, + bounds=None, + ) + result = asyncio.run( + move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) + ) + assert result is None -# ============================================================================== -# Element Focus Resolution Tests -# ============================================================================== +class TestFocusElementIfNeeded: + """Test cases for focus_element_if_needed function.""" - -class TestElementFocusing: - """Validates conditional focusing behavior prior to keyboard inputs.""" - - @pytest.mark.asyncio - @patch("artemis.tools.utils.tap", new_callable=AsyncMock) - @patch("artemis.tools.utils.UnifiedMobileController") - async def test_skips_tap_when_already_focused( - self, - mock_controller_cls: MagicMock, - mock_tap_fn: AsyncMock, - dummy_context: MagicMock, - sample_hierarchy_node: dict, + @patch("artemis.tools.utils.tap") + @patch("artemis.tools.utils.find_element_by_resource_id") + def test_focus_element_already_focused( + self, mock_find_element, mock_tap, mock_context, sample_rich_element ): - """Elements reporting focused state true should avoid redundant tap interactions.""" - focused_elem = sample_hierarchy_node.copy() - focused_elem["focused"] = "true" + """Test when element is already focused.""" + focused_element = sample_rich_element.copy() + focused_element["attributes"]["focused"] = "true" - mock_ctrl = MagicMock() - mock_ctrl.get_ui_elements = AsyncMock(return_value=[focused_elem]) - mock_controller_cls.return_value = mock_ctrl + mock_response = Mock() + mock_response.json.return_value = {"elements": [focused_element]} + mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response) + mock_find_element.return_value = focused_element["attributes"] - target = Target(resource_id="com.google.android.apps:id/search_query") - focus_strategy = await focus_element_if_needed(ctx=dummy_context, target=target) + target = Target( + resource_id="com.example:id/text_input", + resource_id_index=None, + text=None, + text_index=None, + bounds=None, + ) + result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) - mock_tap_fn.assert_not_awaited() - assert focus_strategy == "resource_id" + mock_tap.assert_not_called() + assert result == "resource_id" + mock_context.ui_adb_client.get_hierarchy.assert_called_once() - @pytest.mark.asyncio - @patch("artemis.tools.utils.tap", new_callable=AsyncMock) - @patch("artemis.tools.utils.UnifiedMobileController") - async def test_focus_executes_tap_for_unfocused_element( - self, - mock_controller_cls: MagicMock, - mock_tap_fn: AsyncMock, - dummy_context: MagicMock, - sample_hierarchy_node: dict, + @patch("artemis.tools.utils.tap") + @patch("artemis.tools.utils.find_element_by_resource_id") + def test_focus_element_needs_focus_success( + self, mock_find_element, mock_tap, mock_context, sample_rich_element ): - """Unfocused element triggers focus tap and confirms state change.""" - unfocused = sample_hierarchy_node.copy() - unfocused["focused"] = "false" - focused = sample_hierarchy_node.copy() - focused["focused"] = "true" + """Test when element needs focus and focusing succeeds.""" + unfocused_element = sample_rich_element + focused_element = { + "attributes": { + "resource-id": "com.example:id/text_input", + "focused": "true", + }, + "children": [], + } - mock_ctrl = MagicMock() - mock_ctrl.get_ui_elements = AsyncMock(side_effect=[[unfocused], [focused]]) - mock_controller_cls.return_value = mock_ctrl + mock_find_element.side_effect = [ + unfocused_element["attributes"], + focused_element["attributes"], + ] - target = Target(resource_id="com.google.android.apps:id/search_query") - strategy = await focus_element_if_needed(ctx=dummy_context, target=target) + target = Target( + resource_id="com.example:id/text_input", + resource_id_index=None, + text=None, + text_index=None, + bounds=None, + ) + result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) - mock_tap_fn.assert_awaited_once_with( - ctx=dummy_context, - selector_request=IdSelectorRequest(id="com.google.android.apps:id/search_query"), + mock_tap.assert_called_once_with( + ctx=mock_context, + selector_request=IdSelectorRequest(id="com.example:id/text_input"), index=0, ) - assert strategy == "resource_id" + assert mock_context.ui_adb_client.get_hierarchy.call_count == 2 + assert result == "resource_id" - @pytest.mark.asyncio - @patch("artemis.tools.utils.tap", new_callable=AsyncMock) - @patch("artemis.tools.utils.UnifiedMobileController") - async def test_focus_falls_back_to_text_coordinates( - self, - mock_controller_cls: MagicMock, - mock_tap_fn: AsyncMock, - dummy_context: MagicMock, - sample_hierarchy_node: dict, - ): - """When ID is unavailable, focusing falls back to text center tap coordinates.""" - elem = sample_hierarchy_node.copy() - elem["bounds"] = {"x": 20, "y": 40, "width": 200, "height": 60} - - mock_ctrl = MagicMock() - mock_ctrl.get_ui_elements = AsyncMock(return_value=[elem]) - mock_controller_cls.return_value = mock_ctrl - - target = Target(text="Search Google") - strategy = await focus_element_if_needed(ctx=dummy_context, target=target) - - mock_tap_fn.assert_awaited_once() - req = mock_tap_fn.await_args.kwargs["selector_request"] - assert isinstance(req, SelectorRequestWithCoordinates) - assert req.coordinates.x == 120 # 20 + 200/2 - assert req.coordinates.y == 70 # 40 + 60/2 - assert strategy == "text" - - @pytest.mark.asyncio + @patch("artemis.tools.utils.tap") @patch("artemis.tools.utils.logger") - @patch("artemis.tools.utils.UnifiedMobileController") - async def test_focus_aborts_when_no_locators_resolve( + @patch("artemis.tools.utils.find_element_by_resource_id") + def test_focus_id_and_text_mismatch_fallback_to_text( self, - mock_controller_cls: MagicMock, - mock_logger: MagicMock, - dummy_context: MagicMock, + mock_find_id, + mock_logger, + mock_tap, + mock_context, + sample_rich_element, ): - """When none of the declared locators match an active element, returns None and logs error.""" - mock_ctrl = MagicMock() - mock_ctrl.get_ui_elements = AsyncMock(return_value=[]) - mock_controller_cls.return_value = mock_ctrl + """Test fallback when resource_id and text point to different elements.""" + element_from_id = sample_rich_element["attributes"].copy() + element_from_id["text"] = "Different text" - target = Target(resource_id="unknown_id", text="unknown_text") - strategy = await focus_element_if_needed(ctx=dummy_context, target=target) + element_from_text = sample_rich_element.copy() + element_from_text["attributes"]["bounds"] = { + "x": 10, + "y": 20, + "width": 100, + "height": 30, + } - mock_logger.error.assert_called_once() - assert strategy is None + mock_response = Mock() + mock_response.json.return_value = {"elements": [element_from_text]} + mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response) + mock_find_id.return_value = element_from_id + with patch("artemis.tools.utils.find_element_by_text") as mock_find_text: + mock_find_text.return_value = element_from_text["attributes"] -# ============================================================================== -# Helper Verification & Selector Validation Tests -# ============================================================================== + target = Target( + resource_id="com.example:id/text_input", + resource_id_index=None, + text="Sample text", + text_index=None, + bounds=None, + ) + result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) + mock_logger.warning.assert_called_once() + mock_tap.assert_called_once() + assert result == "text" -class TestSelectorValidation: - """Verifies target completeness and boundary constraint checks.""" + @patch("artemis.tools.utils.tap") + @patch("artemis.tools.utils.find_element_by_text") + def test_focus_fallback_to_text( + self, mock_find_text, mock_tap, mock_context, sample_rich_element + ): + """Test fallback to focusing using text.""" + element_with_bounds = sample_rich_element.copy() + element_with_bounds["attributes"]["bounds"] = { + "x": 10, + "y": 20, + "width": 100, + "height": 30, + } - def test_has_valid_selectors_matrix(self): - """Verifies selector detection across valid and invalid combinations.""" - assert has_valid_selectors(Target(resource_id="elem_id")) - assert has_valid_selectors(Target(text="Label")) - assert has_valid_selectors(Target(bounds=ElementBounds(x=0, y=0, width=10, height=10))) - assert not has_valid_selectors(Target()) + mock_response = Mock() + mock_response.json.return_value = {"elements": [element_with_bounds]} + mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response) + mock_find_text.return_value = element_with_bounds["attributes"] - def test_validate_coordinates_bounds(self): - """Checks bounds containment against physical device display limits.""" - valid_target = Target(bounds=ElementBounds(x=100, y=200, width=100, height=100)) - assert validate_coordinates_bounds(valid_target, 1080, 2400) is None - - out_width = Target(bounds=ElementBounds(x=1050, y=200, width=100, height=100)) - assert validate_coordinates_bounds(out_width, 1080, 2400) is not None - - out_height = Target(bounds=ElementBounds(x=100, y=2380, width=100, height=100)) - assert validate_coordinates_bounds(out_height, 1080, 2400) is not None - - def test_extract_selector_attributes(self): - """Extracts resource ID and text from heterogeneous selector requests.""" - req_id = IdSelectorRequest(id="view_item") - assert _extract_resource_id_and_text_from_selector(req_id) == ("view_item", None) - - req_text = TextSelectorRequest(text="Confirm") - assert _extract_resource_id_and_text_from_selector(req_text) == (None, "Confirm") - - req_both = IdWithTextSelectorRequest(id="submit_btn", text="Submit") - assert _extract_resource_id_and_text_from_selector(req_both) == ("submit_btn", "Submit") - - req_coords = SelectorRequestWithCoordinates( - coordinates=CoordinatesSelectorRequest(x=50, y=50) + target = Target( + resource_id=None, + resource_id_index=None, + text="Sample text", + text_index=None, + bounds=None, ) - assert _extract_resource_id_and_text_from_selector(req_coords) == (None, None) + result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) + + mock_find_text.assert_called_once() + mock_tap.assert_called_once() + call_args = mock_tap.call_args[1] + selector = call_args["selector_request"] + assert isinstance(selector, SelectorRequestWithCoordinates) + assert selector.coordinates.x == 60 # 10 + 100/2 + assert selector.coordinates.y == 35 # 20 + 30/2 + assert result == "text" + + @patch("artemis.tools.utils.logger") + def test_focus_all_locators_fail(self, mock_logger, mock_context): + """Test failure when no locator can find an element.""" + + mock_response = Mock() + mock_response.json.return_value = {"elements": []} + mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response) + with ( + patch("artemis.tools.utils.find_element_by_resource_id") as mock_find_id, + patch("artemis.tools.utils.find_element_by_text") as mock_find_text, + ): + mock_find_id.return_value = None + mock_find_text.return_value = None + + target = Target( + resource_id="nonexistent", + resource_id_index=None, + text="nonexistent", + text_index=None, + bounds=None, + ) + result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) + + mock_logger.error.assert_called_once_with( + "Failed to focus element." + + " No valid locator (resource_id, coordinates, or text) succeeded." + ) + assert result is None + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/tests/unit/utils/test_ui_hierarchy.py b/tests/unit/utils/test_ui_hierarchy.py index 54c1dc9..9a8642a 100644 --- a/tests/unit/utils/test_ui_hierarchy.py +++ b/tests/unit/utils/test_ui_hierarchy.py @@ -11,11 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) -# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. - -"""Unit tests validating Artemis UI hierarchy parsing and geometry computation.""" from unittest.mock import patch @@ -34,160 +29,215 @@ def test_text_input_is_empty(): assert text_input_is_empty(text=None, hint_text=None) assert text_input_is_empty(text="", hint_text=None) assert text_input_is_empty(text="", hint_text="") - assert text_input_is_empty(text="search", hint_text="search") + assert text_input_is_empty(text="text", hint_text="text") - assert not text_input_is_empty(text="search", hint_text=None) - assert not text_input_is_empty(text="search", hint_text="") + assert not text_input_is_empty(text="text", hint_text=None) + assert not text_input_is_empty(text="text", hint_text="") def test_find_element_by_resource_id(): - tree = [ + ui_hierarchy = [ { - "resourceId": "com.google.android.settings:id/search_bar", - "text": "Search Settings", + "resourceId": "com.example:id/button1", + "text": "Button 1", "children": [], }, { - "resourceId": "com.google.android.settings:id/content_frame", + "resourceId": "com.example:id/container", "children": [ { - "resourceId": "com.google.android.settings:id/network_item", - "text": "Network & Internet", + "resourceId": "com.example:id/nested_button", + "text": "Nested Button", "children": [], } ], }, ] - target = find_element_by_resource_id(tree, "com.google.android.settings:id/search_bar") - assert target is not None - assert target["resourceId"] == "com.google.android.settings:id/search_bar" - assert target["text"] == "Search Settings" + result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button1") + assert result is not None + assert result["resourceId"] == "com.example:id/button1" + assert result["text"] == "Button 1" - nested = find_element_by_resource_id(tree, "com.google.android.settings:id/network_item") - assert nested is not None - assert nested["resourceId"] == "com.google.android.settings:id/network_item" + result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nested_button") + assert result is not None + assert result["resourceId"] == "com.example:id/nested_button" + assert result["text"] == "Nested Button" - assert find_element_by_resource_id(tree, "com.google.android.settings:id/missing") is None - assert find_element_by_resource_id([], "com.google.android.settings:id/search_bar") is None + result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nonexistent") + assert result is None + + result = find_element_by_resource_id([], "com.example:id/button1") + assert result is None def test_find_element_by_resource_id_with_index(): - duplicate_items = [ - {"resourceId": "com.artemis.ui:id/row_item", "text": "Item Alpha", "children": []}, - {"resourceId": "com.artemis.ui:id/row_item", "text": "Item Beta", "children": []}, + ui_hierarchy = [ { - "resourceId": "com.artemis.ui:id/section_group", - "children": [ - {"resourceId": "com.artemis.ui:id/row_item", "text": "Item Gamma", "children": []} - ], - }, - ] - - first = find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=0) - assert first is not None - assert first["text"] == "Item Alpha" - - second = find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=1) - assert second is not None - assert second["text"] == "Item Beta" - - third = find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=2) - assert third is not None - assert third["text"] == "Item Gamma" - - assert find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=3) is None - - -def test_find_element_by_resource_id_rich_hierarchy(): - rich_nodes = [ - { - "attributes": {"resource-id": "com.android.systemui:id/clock"}, + "resourceId": "com.example:id/button", + "text": "Button 1", "children": [], }, { - "attributes": {"resource-id": "com.android.systemui:id/status_bar"}, + "resourceId": "com.example:id/button", + "text": "Button 2", + "children": [], + }, + { + "resourceId": "com.example:id/container", "children": [ { - "attributes": {"resource-id": "com.android.systemui:id/battery_meter"}, + "resourceId": "com.example:id/button", + "text": "Button 3", "children": [], } ], }, ] - match = find_element_by_resource_id( - rich_nodes, "com.android.systemui:id/clock", is_rich_hierarchy=True - ) - assert match is not None - assert match["resource-id"] == "com.android.systemui:id/clock" + result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=0) + assert result is not None + assert result["text"] == "Button 1" - nested_match = find_element_by_resource_id( - rich_nodes, "com.android.systemui:id/battery_meter", is_rich_hierarchy=True - ) - assert nested_match is not None - assert nested_match["resource-id"] == "com.android.systemui:id/battery_meter" + result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=1) + assert result is not None + assert result["text"] == "Button 2" - assert find_element_by_resource_id( - rich_nodes, "com.android.systemui:id/unknown", is_rich_hierarchy=True - ) is None + result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=2) + assert result is not None + assert result["text"] == "Button 3" + + result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=3) + assert result is None + + +def test_find_element_by_resource_id_rich_hierarchy(): + rich_hierarchy = [ + { + "attributes": {"resource-id": "com.example:id/button1"}, + "children": [], + }, + { + "attributes": {"resource-id": "com.example:id/container"}, + "children": [ + { + "attributes": {"resource-id": "com.example:id/nested_button"}, + "children": [], + } + ], + }, + ] + + result = find_element_by_resource_id( + rich_hierarchy, "com.example:id/button1", is_rich_hierarchy=True + ) + assert result is not None + assert result["resource-id"] == "com.example:id/button1" + + result = find_element_by_resource_id( + rich_hierarchy, "com.example:id/nested_button", is_rich_hierarchy=True + ) + assert result is not None + assert result["resource-id"] == "com.example:id/nested_button" + + result = find_element_by_resource_id( + rich_hierarchy, "com.example:id/nonexistent", is_rich_hierarchy=True + ) + assert result is None def test_is_element_focused(): - assert is_element_focused({"focused": "true"}) - assert is_element_focused({"focused": True}) - assert not is_element_focused({"focused": "false"}) - assert not is_element_focused({"focused": False}) - assert not is_element_focused({"text": "sample"}) - assert not is_element_focused({"focused": None}) + focused_element = {"focused": "true"} + assert is_element_focused(focused_element) + + non_focused_element = {"focused": "false"} + assert not is_element_focused(non_focused_element) + + no_focused_element = {"text": "some text"} + assert not is_element_focused(no_focused_element) + + none_focused_element = {"focused": None} + assert not is_element_focused(none_focused_element) def test_get_element_text(): - node = {"text": "Battery", "hintText": "Search settings"} - assert get_element_text(node) == "Battery" - assert get_element_text(node, hint_text=False) == "Battery" - assert get_element_text(node, hint_text=True) == "Search settings" + element = {"text": "Button Text", "hintText": "Hint Text"} + assert get_element_text(element) == "Button Text" + assert get_element_text(element, hint_text=False) == "Button Text" + assert get_element_text(element, hint_text=True) == "Hint Text" - node_no_text = {"hintText": "Placeholder"} - assert get_element_text(node_no_text) is None - assert get_element_text(node_no_text, hint_text=True) == "Placeholder" + element_no_text = {"hintText": "Hint Text"} + assert get_element_text(element_no_text) is None + assert get_element_text(element_no_text, hint_text=True) == "Hint Text" + element_no_hint = {"text": "Button Text"} + assert get_element_text(element_no_hint) == "Button Text" + assert get_element_text(element_no_hint, hint_text=True) is None - assert get_element_text({}) is None + empty_element = {} + assert get_element_text(empty_element) is None + assert get_element_text(empty_element, hint_text=True) is None def test_get_bounds_for_element(): - node_with_box = {"bounds": {"x": 24, "y": 48, "width": 400, "height": 80}} - box = get_bounds_for_element(node_with_box) - assert box is not None - assert isinstance(box, ElementBounds) - assert box.x == 24 - assert box.y == 48 - assert box.width == 400 - assert box.height == 80 + element_with_bounds = {"bounds": {"x": 10, "y": 20, "width": 100, "height": 50}} + bounds = get_bounds_for_element(element_with_bounds) + assert bounds is not None + assert isinstance(bounds, ElementBounds) + assert bounds.x == 10 + assert bounds.y == 20 + assert bounds.width == 100 + assert bounds.height == 50 - assert get_bounds_for_element({"text": "Click"}) is None + element_no_bounds = {"text": "Button"} + bounds = get_bounds_for_element(element_no_bounds) + assert bounds is None + # Suppress logger output for the invalid bounds test case with patch("artemis.utils.ui_hierarchy.logger.error"): - bad_box = {"bounds": {"x": "corrupted", "y": 48, "width": 400, "height": 80}} - assert get_bounds_for_element(bad_box) is None + element_invalid_bounds = { + "bounds": { + "x": "invalid", # Should be int + "y": 20, + "width": 100, + "height": 50, + } + } + bounds = get_bounds_for_element(element_invalid_bounds) + assert bounds is None -def test_element_bounds_geometry(): - box = ElementBounds(x=100, y=200, width=500, height=100) +def test_element_bounds(): + bounds = ElementBounds(x=10, y=20, width=100, height=50) - center = box.get_center() + center = bounds.get_center() assert isinstance(center, Point) - assert center.x == 350 - assert center.y == 250 + assert center.x == 60 + assert center.y == 45 - mid = box.get_relative_point(0.5, 0.5) - assert mid.x == 350 - assert mid.y == 250 + center_point = bounds.get_relative_point(0.5, 0.5) + assert isinstance(center_point, Point) + assert center_point.x == 60 + assert center_point.y == 45 - top_left = box.get_relative_point(0.0, 0.0) - assert top_left.x == 100 - assert top_left.y == 200 + top_left = bounds.get_relative_point(0.0, 0.0) + assert top_left.x == 10 + assert top_left.y == 20 - bottom_right = box.get_relative_point(1.0, 1.0) - assert bottom_right.x == 600 - assert bottom_right.y == 300 + bottom_right = bounds.get_relative_point(1.0, 1.0) + assert bottom_right.x == 110 + assert bottom_right.y == 70 + custom_point = bounds.get_relative_point(0.95, 0.95) + assert custom_point.x == 105 + assert custom_point.y == 67 + + +if __name__ == "__main__": + test_text_input_is_empty() + test_find_element_by_resource_id() + test_find_element_by_resource_id_with_index() + test_find_element_by_resource_id_rich_hierarchy() + test_is_element_focused() + test_get_element_text() + test_get_bounds_for_element() + test_element_bounds() + print("All tests passed")