From 54fcef9887fbea193911acddca33e3b0d822862c Mon Sep 17 00:00:00 2001 From: somew1nd Date: Fri, 11 Sep 2026 16:19:42 -0700 Subject: [PATCH] refactor: clean up legacy code, rename hopper to entity_extractor, and add third-party notices - Add NOTICE file and README license references for mobile-use and finalrun-agent - Add derivative attribution headers to 17 files derived from mobile-use - Rename hopper agent to entity_extractor across prompts, tools, and configs - Remove obsolete adb_tunnel, ApiBaseUrl, and legacy CLI utility scripts - Replace legacy examples with native Artemis diagnostic and workflow examples - Update unit and tool tests to validate entity_extractor and SDK builders --- .env.example | 2 +- NOTICE | 27 + README.md | 4 + README_CN.md | 11 + .../entity_extractor/__init__.py} | 13 +- .../entity_extractor/entity_extractor.md | 30 + .../entity_extractor/entity_extractor.py | 90 +++ artemis/agents/hopper/hopper.md | 15 - artemis/agents/hopper/hopper.py | 93 --- artemis/agents/log_analyzer/log_analyzer.py | 20 - artemis/agents/outputter/outputter.py | 3 + artemis/agents/planner/planner.py | 21 +- artemis/clients/adb_tunnel.py | 327 ---------- artemis/clients/ui_automator_client.py | 116 +--- artemis/config/__init__.py | 4 +- artemis/config/constants.py | 3 +- artemis/config/llm.py | 37 +- artemis/config/output.py | 3 - artemis/config/settings.py | 24 +- artemis/context.py | 3 + artemis/controllers/device_controller.py | 212 ++---- .../platform_specific_commands_controller.py | 18 +- artemis/controllers/unified_controller.py | 3 + artemis/graph/graph.py | 20 - artemis/interfaces/cli/commands/run.py | 20 +- artemis/resources/config/artemis.jsonc | 4 +- artemis/sdk/agent.py | 4 +- artemis/sdk/builders/agent_config_builder.py | 10 +- artemis/sdk/builders/task_request_builder.py | 221 ++----- .../accessibility_helper_diagnostics.py | 53 ++ .../examples/android_settings_inspection.py | 69 ++ artemis/sdk/examples/app_lock_messaging.py | 87 --- .../sdk/examples/app_workflow_automation.py | 53 ++ .../sdk/examples/multimodal_vision_testing.py | 68 ++ .../sdk/examples/simple_photo_organizer.py | 111 ---- .../examples/smart_notification_assistant.py | 235 ------- .../examples/video_transcription_example.py | 110 ---- artemis/sdk/types/__init__.py | 4 +- artemis/sdk/types/agent.py | 59 +- artemis/sdk/types/exceptions.py | 77 +-- artemis/sdk/types/platform.py | 210 ------ artemis/sdk/types/task.py | 224 +++---- artemis/tools/committee_tool.py | 3 - artemis/tools/diagnostic_tool.py | 3 - artemis/tools/explorer_tool.py | 3 - artemis/tools/image_processor_tool.py | 3 - artemis/tools/log_tool.py | 3 - artemis/tools/mobile/launch_app.py | 16 +- artemis/tools/mobile/read_hierarchy.py | 5 - artemis/tools/object_detection_tool.py | 5 - artemis/tools/scratchpad.py | 13 - artemis/tools/tool_wrapper.py | 5 - artemis/tools/types.py | 62 +- artemis/tools/utils.py | 484 ++++++-------- artemis/tools/video_tool.py | 3 - artemis/tools/wait_tool.py | 3 - artemis/utils/app_launch_utils.py | 9 +- artemis/utils/cli_helpers.py | 38 -- artemis/utils/cli_selection.py | 154 ----- artemis/utils/conversations.py | 45 -- artemis/utils/decorators.py | 187 +++--- artemis/utils/file.py | 39 +- artemis/utils/logger.py | 3 + artemis/utils/media.py | 232 +++---- artemis/utils/recorder.py | 60 -- artemis/utils/requests_utils.py | 51 -- artemis/utils/shell_utils.py | 35 - artemis/utils/task_tree.py | 2 +- artemis/utils/ui_hierarchy.py | 260 ++++---- artemis/utils/video.py | 21 +- config/artemis.jsonc | 4 +- .../test_entity_extractor_tools.py} | 2 +- tests/unit/agents/test_entity_extractor.py | 76 +++ tests/unit/agents/test_outputter.py | 19 - tests/unit/mcp/test_device_utils.py | 6 +- tests/unit/mcp/test_mcp_stdio_lifecycle.py | 17 +- tests/unit/sdk/test_exceptions.py | 63 ++ tests/unit/sdk/test_task_request_builder.py | 111 ++++ tests/unit/test_config_package.py | 2 +- tests/unit/test_ui_automator_client_awake.py | 10 +- tests/unit/tools/test_launch_app.py | 45 +- tests/unit/tools/test_utils.py | 602 ++++++++---------- tests/unit/utils/test_ui_hierarchy.py | 234 +++---- 83 files changed, 1996 insertions(+), 3660 deletions(-) create mode 100644 NOTICE rename artemis/{utils/time.py => agents/entity_extractor/__init__.py} (71%) create mode 100644 artemis/agents/entity_extractor/entity_extractor.md create mode 100644 artemis/agents/entity_extractor/entity_extractor.py delete mode 100644 artemis/agents/hopper/hopper.md delete mode 100644 artemis/agents/hopper/hopper.py delete mode 100644 artemis/clients/adb_tunnel.py create mode 100644 artemis/sdk/examples/accessibility_helper_diagnostics.py create mode 100644 artemis/sdk/examples/android_settings_inspection.py delete mode 100644 artemis/sdk/examples/app_lock_messaging.py create mode 100644 artemis/sdk/examples/app_workflow_automation.py create mode 100644 artemis/sdk/examples/multimodal_vision_testing.py delete mode 100644 artemis/sdk/examples/simple_photo_organizer.py delete mode 100644 artemis/sdk/examples/smart_notification_assistant.py delete mode 100644 artemis/sdk/examples/video_transcription_example.py delete mode 100644 artemis/sdk/types/platform.py delete mode 100644 artemis/utils/cli_helpers.py delete mode 100644 artemis/utils/cli_selection.py delete mode 100644 artemis/utils/conversations.py delete mode 100644 artemis/utils/recorder.py delete mode 100644 artemis/utils/requests_utils.py delete mode 100644 artemis/utils/shell_utils.py rename tests/tools/{hopper/test_hopper_tools.py => entity_extractor/test_entity_extractor_tools.py} (89%) create mode 100644 tests/unit/agents/test_entity_extractor.py create mode 100644 tests/unit/sdk/test_exceptions.py create mode 100644 tests/unit/sdk/test_task_request_builder.py diff --git a/.env.example b/.env.example index 8b23d0a..f8b492e 100644 --- a/.env.example +++ b/.env.example @@ -9,7 +9,7 @@ GEMINI_API_KEY= GOOGLE_API_KEY= OPENAI_API_KEY= ANTHROPIC_API_KEY= -OPEN_ROUTER_API_KEY= +OPENROUTER_API_KEY= XAI_API_KEY= # Google Cloud Vision OCR (Optional - for advanced OCR processing) diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..e7cd397 --- /dev/null +++ b/NOTICE @@ -0,0 +1,27 @@ +ARTEMIS Android Agent & Automation Platform +Copyright 2026 Google LLC + +This product includes software developed by Google LLC. + +================================================================================ +Third-Party Software Attribution +================================================================================ + +Portions of this software are derived from and incorporate code originally +developed by the following open-source projects under the Apache License, Version 2.0: + +1. mobile-use + Repository: https://github.com/minitap-ai/mobile-use + Copyright 2025-2026 Minitap, Inc. + Original authors & contributors: + Pierre-Louis Favreau, Jean-Pierre Lo, Nicolas Dehandschoewercker, + Clément Guiguet, Karun Agarwal, and the mobile-use open-source community. + License: Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +2. finalrun-agent + Repository: https://github.com/final-run/finalrun-agent + Copyright 2025 FinalRun Inc. + Original authors & contributors: + Ashish Yadav, Arnold Laishram, Srinidhi G S, + and the finalrun-agent open-source community. + License: Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) diff --git a/README.md b/README.md index 0c6e2c6..2e81c30 100644 --- a/README.md +++ b/README.md @@ -314,3 +314,7 @@ Contributions are warmly welcomed! ## License This project is licensed under the [Apache License 2.0](LICENSE). + +This project includes source code developed by [Minitap, Inc.](https://github.com/minitap-ai/mobile-use). + + diff --git a/README_CN.md b/README_CN.md index 9b1995a..7a2e342 100644 --- a/README_CN.md +++ b/README_CN.md @@ -312,3 +312,14 @@ ARTEMIS 提供两种运行模式以适应不同的自动化需求： ## 开源许可证 本项目基于 [Apache License 2.0](LICENSE) 协议开源。 + +## 致谢 + +本项目包含由 [Minitap, Inc.](https://github.com/minitap-ai/mobile-use)（`mobile-use`）团队开发的部分源代码与工具组件，遵循 Apache License 2.0 开源许可。我们感谢其团队及开源社区在移动端自动化领域的贡献。 + +同时感谢 [Finalrun](https://github.com/final-run/finalrun-agent) 在智能滑动与自适应手势机制理念上的启发。 + +关于完整的第三方开源软件版权与归属声明，请参阅根目录的 [NOTICE](NOTICE) 文件。 + + + diff --git a/artemis/utils/time.py b/artemis/agents/entity_extractor/__init__.py similarity index 71% rename from artemis/utils/time.py rename to artemis/agents/entity_extractor/__init__.py index 5ec443a..c40cae6 100644 --- a/artemis/utils/time.py +++ b/artemis/agents/entity_extractor/__init__.py @@ -12,9 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime +"""Entity extractor and package resolver agent for ARTEMIS.""" +from artemis.agents.entity_extractor.entity_extractor import ( + ExtractionResult, + extract_entity, +) -def convert_timestamp_to_str(ts: float) -> str: - dt = datetime.fromtimestamp(ts) - return dt.strftime("%Y-%m-%dT%H-%M-%S") +__all__ = [ + "ExtractionResult", + "extract_entity", +] diff --git a/artemis/agents/entity_extractor/entity_extractor.md b/artemis/agents/entity_extractor/entity_extractor.md new file mode 100644 index 0000000..6bf1325 --- /dev/null +++ b/artemis/agents/entity_extractor/entity_extractor.md @@ -0,0 +1,30 @@ +# Artemis Entity Extractor & Package Resolver + +You are an autonomous, high-precision structured data extractor in the ARTEMIS mobile testing runtime. +Your responsibility is to extract the exact requested identifier, package name, or configuration token from raw device dump outputs (such as `pm list packages`, `dumpsys`, logcat records, or accessibility tree dumps). + +## Operating Directives + +1. **Verbatim Extraction**: + - Return the extracted entity exactly as it appears in the source data. + - Do NOT modify capitalization, strip domain segments, translate, or normalize the matched identifier. + +2. **Package Resolution Rules**: + - Match target application titles, codenames, or brand identifiers against standard Android package formats (`com..`, `org..`, etc.). + - Prioritize exact matches over partial or prefix matches. + +3. **Strict Disambiguation & Null-Safety**: + - If the requested entity is absent from the input dataset, return `found: false` with `output: null`. + - If multiple candidates conflict and cannot be deterministically verified from the query, do NOT guess. Set `found: false` and `output: null`. + +## Structured Deliverable Schema + +You must produce your final answer conforming to the following JSON structure: + +```json +{ + "found": true, + "output": "", + "reason": "" +} +``` diff --git a/artemis/agents/entity_extractor/entity_extractor.py b/artemis/agents/entity_extractor/entity_extractor.py new file mode 100644 index 0000000..bac4aff --- /dev/null +++ b/artemis/agents/entity_extractor/entity_extractor.py @@ -0,0 +1,90 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Lightweight entity extraction and package resolver for ARTEMIS. + +Dispatches bounded LLM extraction turns to identify application package identifiers +and configuration attributes from raw device listings without manual parsing heuristics. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import cast + +from jinja2 import Template +from langchain_core.messages import HumanMessage, SystemMessage +from pydantic import BaseModel, Field + +from artemis.context import ArtemisContext +from artemis.data_engine.trace import trace +from artemis.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback +from artemis.utils.logger import get_logger + +logger = get_logger(__name__) + + +class ExtractionResult(BaseModel): + """Structured extraction outcome for entity resolution.""" + + found: bool = Field(description="True if the requested entity was definitively identified.") + output: str | None = Field(default=None, description="Exact extracted identifier or string value.") + reason: str = Field(description="Brief justification of the match criteria applied.") + + +@trace(type="agent", name="entity_extractor") +async def extract_entity( + ctx: ArtemisContext, + request: str, + data: str, + use_fallback: bool = True, +) -> ExtractionResult: + """Extract a specific identifier or entity from raw batch device data.""" + logger.info(f"Invoking Artemis Entity Extractor (fallback={use_fallback})") + template_path = Path(__file__).parent / "entity_extractor.md" + system_prompt = Template(template_path.read_text(encoding="utf-8")).render() + + messages = [ + SystemMessage(content=system_prompt), + HumanMessage(content=f"Target Query:\n{request}\n\nCandidate Raw Dataset:\n{data}"), + ] + + primary_llm = get_llm(ctx=ctx, name="entity_extractor", is_utils=True).with_structured_output( + ExtractionResult + ) + + try: + if use_fallback: + fallback_llm = get_llm( + ctx=ctx, name="entity_extractor", is_utils=True, use_fallback=True + ).with_structured_output(ExtractionResult) + raw = await with_fallback( + main_call=lambda: invoke_llm_with_timeout_message(primary_llm.ainvoke(messages)), + fallback_call=lambda: invoke_llm_with_timeout_message(fallback_llm.ainvoke(messages)), + ) + else: + raw = await invoke_llm_with_timeout_message(primary_llm.ainvoke(messages)) + + if isinstance(raw, ExtractionResult): + return raw + if isinstance(raw, dict): + return ExtractionResult.model_validate(raw) + return cast(ExtractionResult, raw) + except Exception as exc: + logger.error(f"Entity extraction turn failed: {exc}") + return ExtractionResult( + found=False, + output=None, + reason=f"LLM extraction error: {str(exc)[:400]}", + ) diff --git a/artemis/agents/hopper/hopper.md b/artemis/agents/hopper/hopper.md deleted file mode 100644 index 27c9d1a..0000000 --- a/artemis/agents/hopper/hopper.md +++ /dev/null @@ -1,15 +0,0 @@ -## Hopper - -Extract relevant information from batch data. **Keep extracted data exactly as-is** - no reformatting. - -## Output -- **found**: `true` if data was found, `false` otherwise -- **output**: Extracted information if found, `null` otherwise -- **reason**: Brief explanation of search logic - -## Rules -1. **Search entire input** - may contain hundreds of entries -2. **For app package lookup**: Match app name (or variations) in package identifier - - Common patterns: lowercase app name, company+app, brand name, codenames -3. **Prefer direct matches** over partial matches -4. **Return `null`** if not found or if multiple ambiguous matches exist - don't guess diff --git a/artemis/agents/hopper/hopper.py b/artemis/agents/hopper/hopper.py deleted file mode 100644 index 0c1cb9f..0000000 --- a/artemis/agents/hopper/hopper.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pathlib import Path - -from jinja2 import Template -from langchain_core.messages import HumanMessage, SystemMessage -from artemis.context import ArtemisContext -from artemis.data_engine.trace import trace -from artemis.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback -from artemis.utils.logger import get_logger -from pydantic import BaseModel, Field - -logger = get_logger(__name__) - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class HopperOutput(BaseModel): - model_config = {"ignored_types": (CyFunctionDetector,)} - found: bool = Field(description="True if the requested data was found, False otherwise.") - output: str | None = Field(description="The extracted data if found, null otherwise.") - reason: str = Field( - description="A short explanation of what you looked for" - + " and how you decided what to extract." - ) - - -@trace(type="agent", name="hopper") -async def hopper( - ctx: ArtemisContext, - request: str, - data: str, - use_fallback: bool = True, -) -> HopperOutput: - logger.info(f"Starting Hopper Agent (use_fallback={use_fallback})") - system_message = Template( - Path(__file__).parent.joinpath("hopper.md").read_text(encoding="utf-8") - ).render() - messages = [ - SystemMessage(content=system_message), - HumanMessage(content=f"{request}\nHere is the data you must dig:\n{data}"), - ] - - llm = get_llm(ctx=ctx, name="hopper", is_utils=True).with_structured_output(HopperOutput) - try: - if use_fallback: - llm_fallback = get_llm( - ctx=ctx, name="hopper", is_utils=True, use_fallback=True - ).with_structured_output(HopperOutput) - response: HopperOutput = await with_fallback( - main_call=lambda: invoke_llm_with_timeout_message(llm.ainvoke(messages)), - fallback_call=lambda: invoke_llm_with_timeout_message( - llm_fallback.ainvoke(messages) - ), - ) # type: ignore - else: - response: HopperOutput = await invoke_llm_with_timeout_message(llm.ainvoke(messages)) - return response - except Exception as e: - logger.error(f"Hopper LLM invocation failed: {e}") - return HopperOutput( - found=False, - output=None, - reason=f"Failed due to LLM error: {str(e)[:500]}", - ) diff --git a/artemis/agents/log_analyzer/log_analyzer.py b/artemis/agents/log_analyzer/log_analyzer.py index 126b770..76e4947 100644 --- a/artemis/agents/log_analyzer/log_analyzer.py +++ b/artemis/agents/log_analyzer/log_analyzer.py @@ -21,26 +21,7 @@ from langchain_core.tools import BaseTool, StructuredTool from pydantic import BaseModel, Field -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - class SpawnLogReaderArgs(BaseModel): - model_config = {"ignored_types": (CyFunctionDetector,)} specific_query: str = Field( ..., description=( @@ -54,7 +35,6 @@ from langgraph.prebuilt import InjectedState from artemis.core.tool_failure import ToolFailure, is_tool_failure from artemis.context import ArtemisContext from artemis.data_engine.trace import ( - CURRENT_TRACE_ID, TraceSpan, trace, ) diff --git a/artemis/agents/outputter/outputter.py b/artemis/agents/outputter/outputter.py index b84f429..aed600d 100644 --- a/artemis/agents/outputter/outputter.py +++ b/artemis/agents/outputter/outputter.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. import json from pathlib import Path diff --git a/artemis/agents/planner/planner.py b/artemis/agents/planner/planner.py index 9a46038..df23d56 100644 --- a/artemis/agents/planner/planner.py +++ b/artemis/agents/planner/planner.py @@ -23,7 +23,7 @@ from pydantic import BaseModel, Field from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController -from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace +from artemis.data_engine.trace import TraceSpan, trace from artemis.graph.state import State from artemis.graph.visibility import strict_state from artemis.services.llm import ( @@ -108,26 +108,7 @@ def build_planner_system_blocks(prompts_data: dict, mode: str, include_checks: b return blocks -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - class ValidationResult(BaseModel): - model_config = {"ignored_types": (CyFunctionDetector,)} is_approved: bool = Field( description=( "True if the new plan still serves the initial goal and is coherent." diff --git a/artemis/clients/adb_tunnel.py b/artemis/clients/adb_tunnel.py deleted file mode 100644 index 2cd79b6..0000000 --- a/artemis/clients/adb_tunnel.py +++ /dev/null @@ -1,327 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""ADB WebSocket tunnel implementation. - -Bridges local TCP connections to a remote ADB server via WebSocket. -This replaces the need for the `lim` CLI for ADB tunneling. - -The tunnel runs in a dedicated thread with its own event loop to ensure -it cannot be blocked by synchronous operations in the main thread. - -Based on the Go SDK implementation which: -1. Creates a TCP listener -2. Accepts TCP connections from ADB -3. Dials the WebSocket -4. Forwards traffic bidirectionally -""" - -import asyncio -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager -import socket -import threading - -from artemis.utils.logger import get_logger -import websockets -from websockets.asyncio.client import ClientConnection - -logger = get_logger(__name__) - -BUFFER_SIZE = 32 * 1024 # 32KB, same as Go SDK -PING_INTERVAL = 30 # seconds - - -class AdbTunnel: - """ADB tunnel that bridges local TCP connections to a remote WebSocket endpoint. - - Runs in a dedicated thread with its own event loop to ensure the tunnel - cannot be blocked by synchronous operations in the caller's thread. - - Usage: - tunnel = AdbTunnel(remote_url, token) - addr = await tunnel.start() # Returns "127.0.0.1:PORT" - # Use addr with ADB: adb connect 127.0.0.1:PORT - await tunnel.stop() - """ - - def __init__(self, remote_url: str, token: str): - """Initialize the ADB tunnel. - - Args: - remote_url: WebSocket URL for the remote ADB endpoint - token: Bearer token for authentication - """ - self.remote_url = remote_url - self.token = token - - self._local_port: int | None = None - self._thread: threading.Thread | None = None - self._loop: asyncio.AbstractEventLoop | None = None - self._started = threading.Event() - self._stop_event: asyncio.Event | None = None - - @property - def local_addr(self) -> str: - """Get the local address to connect ADB to.""" - if self._local_port is None: - raise RuntimeError("Tunnel not started") - return f"127.0.0.1:{self._local_port}" - - @property - def local_port(self) -> int: - """Get the local port number.""" - if self._local_port is None: - raise RuntimeError("Tunnel not started") - return self._local_port - - async def start(self) -> str: - """Start the ADB tunnel in a dedicated thread. - - Returns: - The local address (host:port) to connect ADB to. - """ - if self._thread is not None and self._thread.is_alive(): - return self.local_addr - - # Create listener in main thread to get the port immediately - listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - listener.bind(("127.0.0.1", 0)) - listener.listen(5) - listener.setblocking(False) - self._local_port = listener.getsockname()[1] - - logger.info(f"ADB tunnel listening on {self.local_addr}") - - # Start tunnel thread - self._thread = threading.Thread( - target=self._run_in_thread, - args=(listener,), - name=f"adb-tunnel-{self._local_port}", - daemon=True, - ) - self._thread.start() - - # Wait for thread to initialize its event loop - self._started.wait(timeout=5.0) - if not self._started.is_set(): - await self.stop() - raise RuntimeError("Tunnel thread failed to start") - - return self.local_addr - - async def stop(self) -> None: - """Stop the ADB tunnel and cleanup resources.""" - if self._loop and self._stop_event: - # Signal the tunnel loop to stop - self._loop.call_soon_threadsafe(self._stop_event.set) - - if self._thread and self._thread.is_alive(): - await asyncio.to_thread(self._thread.join, 5.0) - if self._thread.is_alive(): - logger.warning("ADB tunnel thread did not stop within 5s") - - self._thread = None - self._loop = None - self._stop_event = None - logger.info("ADB tunnel stopped") - - def _run_in_thread(self, listener: socket.socket) -> None: - """Run the tunnel event loop in a dedicated thread.""" - try: - self._loop = asyncio.new_event_loop() - asyncio.set_event_loop(self._loop) - self._stop_event = asyncio.Event() - self._started.set() - - self._loop.run_until_complete(self._run_tunnel(listener)) - except Exception as e: - logger.error(f"Tunnel thread error: {e}") - finally: - listener.close() - if self._loop: - self._loop.close() - - async def _run_tunnel(self, listener: socket.socket) -> None: - """Run the tunnel - accept TCP connections and bridge to WebSocket.""" - loop = asyncio.get_event_loop() - connection_tasks: list[asyncio.Task] = [] - - stop_event = self._stop_event - assert stop_event is not None - - logger.debug("Tunnel: ready to accept connections") - - while not stop_event.is_set(): - try: - # Use wait_for to periodically check stop_event - try: - tcp_conn, addr = await asyncio.wait_for( - loop.sock_accept(listener), - timeout=1.0, - ) - except TimeoutError: - # Check if we should stop - continue - - logger.debug(f"Tunnel: new TCP connection from {addr}") - - task = asyncio.create_task( - self._handle_connection(tcp_conn, loop), - name=f"conn-{addr[1]}", - ) - task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None) - connection_tasks.append(task) - connection_tasks = [t for t in connection_tasks if not t.done()] - - except Exception as e: - if not stop_event.is_set(): - logger.error(f"Tunnel accept error: {e}") - break - - # Cleanup pending connections - for task in connection_tasks: - task.cancel() - try: - await task - except asyncio.CancelledError: - pass - - async def _handle_connection( - self, - tcp_conn: socket.socket, - loop: asyncio.AbstractEventLoop, - ) -> None: - """Handle a single TCP connection.""" - tcp_conn.setblocking(False) - - ws: ClientConnection | None = None - try: - logger.debug(f"Connecting to WebSocket: {self.remote_url[:80]}...") - ws = await websockets.connect( - self.remote_url, - additional_headers={"Authorization": f"Bearer {self.token}"}, - ping_interval=PING_INTERVAL, - ping_timeout=10, - ) - logger.debug("WebSocket connected to remote ADB") - await self._bridge(tcp_conn, ws, loop) - except Exception as e: - logger.error(f"Connection error: {e}") - finally: - if ws is not None: - await ws.close() - tcp_conn.close() - logger.debug("Connection closed") - - async def _bridge( - self, - tcp_conn: socket.socket, - ws: ClientConnection, - loop: asyncio.AbstractEventLoop, - ) -> None: - """Bridge TCP and WebSocket bidirectionally.""" - - stop_event = self._stop_event - assert stop_event is not None - - async def tcp_to_ws() -> None: - """Forward data from TCP to WebSocket.""" - msg_count = 0 - try: - while not stop_event.is_set(): - try: - data = await asyncio.wait_for( - loop.sock_recv(tcp_conn, BUFFER_SIZE), - timeout=1.0, - ) - except TimeoutError: - continue - except (OSError, ConnectionError) as e: - logger.debug(f"tcp->ws: recv error after {msg_count} msgs: {e}") - break - if not data: - logger.debug(f"tcp->ws: closed by client after {msg_count} msgs") - break - msg_count += 1 - logger.debug(f"tcp->ws: [{msg_count}] sending {len(data)} bytes") - await ws.send(data) - logger.debug(f"tcp->ws: loop ended after {msg_count} messages") - except asyncio.CancelledError: - logger.debug(f"tcp->ws: cancelled after {msg_count} messages") - raise - except Exception as e: - logger.error(f"tcp->ws: error after {msg_count} messages: {e}") - - async def ws_to_tcp() -> None: - """Forward data from WebSocket to TCP.""" - msg_count = 0 - try: - logger.debug("ws->tcp: starting message loop") - async for message in ws: - if stop_event.is_set(): - break - if isinstance(message, bytes) and message: - msg_count += 1 - logger.debug(f"ws->tcp: [{msg_count}] received {len(message)} bytes") - await loop.sock_sendall(tcp_conn, message) - elif isinstance(message, str): - logger.debug(f"ws->tcp: received string: {message[:100]}") - logger.debug(f"ws->tcp: loop ended after {msg_count} messages") - except asyncio.CancelledError: - logger.debug(f"ws->tcp: cancelled after {msg_count} messages") - raise - except Exception as e: - logger.error(f"ws->tcp: error after {msg_count} messages: {e}") - - tcp_task = asyncio.create_task(tcp_to_ws()) - ws_task = asyncio.create_task(ws_to_tcp()) - - done, pending = await asyncio.wait( - [tcp_task, ws_task], - return_when=asyncio.FIRST_COMPLETED, - ) - - for task in pending: - task.cancel() - try: - await task - except asyncio.CancelledError: - pass - - -@asynccontextmanager -async def adb_tunnel(remote_url: str, token: str) -> AsyncGenerator[AdbTunnel, None]: - """Context manager for creating and managing an ADB tunnel. - - Args: - remote_url: WebSocket URL for the remote ADB endpoint - token: Bearer token for authentication - - Yields: - AdbTunnel instance with the tunnel running - - Example: - async with adb_tunnel(ws_url, token) as tunnel: - # Connect ADB to tunnel.local_addr - adb_client = AdbClient() - adb_client.connect(tunnel.local_addr) - """ - tunnel = AdbTunnel(remote_url, token) - try: - await tunnel.start() - yield tunnel - finally: - await tunnel.stop() diff --git a/artemis/clients/ui_automator_client.py b/artemis/clients/ui_automator_client.py index 34c1d21..b6b5b41 100644 --- a/artemis/clients/ui_automator_client.py +++ b/artemis/clients/ui_automator_client.py @@ -11,12 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. """UIAutomator2 client for Android device screen data retrieval. -Provides an alternative to the Maestro-based screen API with direct device -access. -Handles Maestro blocker detection and removal before connecting. +Provides direct device access for hierarchy and screenshot retrieval. """ import base64 @@ -41,32 +42,9 @@ if TYPE_CHECKING: logger = get_logger(__name__) -MAESTRO_PACKAGE = "dev.mobile.maestro" - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - class UIAutomatorScreenData(BaseModel): """Screen data response from UIAutomator2.""" - model_config = {"ignored_types": (CyFunctionDetector,)} - base64: str hierarchy_xml: str elements: list[dict] @@ -170,88 +148,11 @@ def _pil_to_base64(img: Image.Image, format: str = "JPEG", quality: int = 80) -> return base64.b64encode(buffer.getvalue()).decode("utf-8") -def _is_package_installed(device_id: str, pkg: str) -> bool: - """Check if a package is installed on the device.""" - try: - result = subprocess.run( - adb_command(["-s", device_id, "shell", "pm", "list", "packages"]), - stdin=subprocess.DEVNULL, - capture_output=True, - text=True, - timeout=10, - ) - if result.returncode != 0: - logger.warning(f"Failed to list packages: {result.stderr}") - return False - - lines = result.stdout.splitlines() - target = f"package:{pkg}" - return target in lines - except subprocess.TimeoutExpired: - logger.warning("Timeout checking installed packages") - return False - except (OSError, subprocess.SubprocessError) as e: - logger.warning(f"Error checking installed packages: {e}") - return False - - -def _uninstall_package(device_id: str, pkg: str) -> bool: - """Uninstall a package from the device for the current user.""" - try: - result = subprocess.run( - adb_command( - [ - "-s", - device_id, - "shell", - "pm", - "uninstall", - "--user", - "0", - pkg, - ] - ), - stdin=subprocess.DEVNULL, - capture_output=True, - text=True, - timeout=30, - ) - if result.returncode == 0: - logger.info(f"Successfully uninstalled {pkg}") - return True - else: - logger.warning(f"Failed to uninstall {pkg}: {result.stderr}") - return False - except subprocess.TimeoutExpired: - logger.warning(f"Timeout uninstalling {pkg}") - return False - except Exception as e: - logger.warning(f"Error uninstalling {pkg}: {e}") - return False - - -def _ensure_maestro_not_installed(device_id: str) -> None: - """Check if Maestro is installed and uninstall it. - - Maestro conflicts with uiautomator2 - if Maestro's package is installed, - u2.connect() will fail. This function ensures Maestro is removed before - attempting to connect. - """ - if _is_package_installed(device_id, MAESTRO_PACKAGE): - logger.warning( - f"Maestro ({MAESTRO_PACKAGE}) detected - uninstalling to enable UIAutomator2..." - ) - _uninstall_package(device_id, MAESTRO_PACKAGE) - - class UIAutomatorClient: """UIAutomator2 client for Android screen data retrieval. - This client uses uiautomator2 library for direct device communication, - providing faster hierarchy and screenshot retrieval compared to Maestro. - - Important: Maestro must not be installed on the device as it conflicts - with uiautomator2. This client automatically handles Maestro removal. + This client uses the uiautomator2 library for direct device communication, + providing high-performance hierarchy and screenshot retrieval. """ def __init__(self, device_id: str): @@ -265,7 +166,7 @@ class UIAutomatorClient: self._awake_strategy: str | None = None def _ensure_connected(self) -> "Device": - """Ensure connection to the device, handling Maestro blocker. + """Ensure connection to the device. Returns: Connected uiautomator2 Device instance @@ -279,9 +180,6 @@ class UIAutomatorClient: logger.warning("UIAutomator2 connection lost, reconnecting...") self._device = None - # Ensure Maestro is not blocking us - _ensure_maestro_not_installed(self._device_id) - # Enroll the device in the containing Artemis service lifetime before # UIAutomator2 reads the screen. Client disconnects do not release it. if self._awake_strategy is None: diff --git a/artemis/config/__init__.py b/artemis/config/__init__.py index e539f93..d28428a 100644 --- a/artemis/config/__init__.py +++ b/artemis/config/__init__.py @@ -83,6 +83,7 @@ from artemis.config.constants import ( ENV_ARTEMIS_TRACES_DIR, ENV_ARTEMIS_USE_FILE_API, ENV_ARTEMIS_USE_USER_DIR, + ENV_OPENROUTER_API_KEY, ENV_OPEN_ROUTER_API_KEY, ENV_OPENAI_API_KEY, ENV_OPENAI_BASE_URL, @@ -100,7 +101,6 @@ from artemis.config.constants import ( LLMUtilsNodeWithFallback, ) from artemis.config.llm import ( - CyFunctionDetector, LLM, LLMConfig, LLMConfigUtils, @@ -196,7 +196,6 @@ __all__ = [ "get_ls_address_file", "is_frozen_bundle", # LLM Config & Models - "CyFunctionDetector", "LLM", "LLMWithFallback", "LLMConfigUtils", @@ -296,6 +295,7 @@ __all__ = [ "ENV_OPENAI_API_KEY", "ENV_OPENAI_BASE_URL", "ENV_ANTHROPIC_API_KEY", + "ENV_OPENROUTER_API_KEY", "ENV_OPEN_ROUTER_API_KEY", "ENV_XAI_API_KEY", "ENV_ARTEMIS_EXPLORER_VERSION", diff --git a/artemis/config/constants.py b/artemis/config/constants.py index 972bc6d..369c43e 100644 --- a/artemis/config/constants.py +++ b/artemis/config/constants.py @@ -62,6 +62,7 @@ ENV_API_KEY = "API_KEY" ENV_OPENAI_API_KEY = "OPENAI_API_KEY" ENV_OPENAI_BASE_URL = "OPENAI_BASE_URL" ENV_ANTHROPIC_API_KEY = "ANTHROPIC_API_KEY" +ENV_OPENROUTER_API_KEY = "OPENROUTER_API_KEY" ENV_OPEN_ROUTER_API_KEY = "OPEN_ROUTER_API_KEY" ENV_XAI_API_KEY = "XAI_API_KEY" @@ -125,7 +126,7 @@ ExplorerVersion = Literal["flash", "pro", "ultra"] LLMUtilsNode = Literal[ "outputter", - "hopper", + "entity_extractor", "video_analyzer", "object_detector", ] diff --git a/artemis/config/llm.py b/artemis/config/llm.py index 36915dd..5fc6062 100644 --- a/artemis/config/llm.py +++ b/artemis/config/llm.py @@ -48,30 +48,11 @@ def validate_vertex_ai_credentials() -> None: ) -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - """Utility type detector for Cython and C-extension functions in Pydantic.""" - pass class LLM(BaseModel): """Base model representing an LLM model provider and runtime parameters.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} provider: LLMProvider model: str temperature: float | None = None @@ -95,8 +76,8 @@ class LLM(BaseModel): if not (settings.ANTHROPIC_API_KEY or os.environ.get("ANTHROPIC_API_KEY")): raise Exception(f"{name} requires ANTHROPIC_API_KEY in .env") elif self.provider == "openrouter": - if not settings.OPEN_ROUTER_API_KEY: - raise Exception(f"{name} requires OPEN_ROUTER_API_KEY in .env") + if not (settings.OPENROUTER_API_KEY or settings.OPEN_ROUTER_API_KEY): + raise Exception(f"{name} requires OPENROUTER_API_KEY in .env") elif self.provider == "xai": if not settings.XAI_API_KEY: raise Exception(f"{name} requires XAI_API_KEY in .env") @@ -133,18 +114,14 @@ def lightweight_judge_default() -> "LLMWithFallback": class LLMConfigUtils(BaseModel): """Configuration container for auxiliary utility agents/nodes.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} outputter: LLMWithFallback - hopper: LLMWithFallback + entity_extractor: LLMWithFallback video_analyzer: LLMWithFallback | None = None object_detector: LLMWithFallback | None = None class LLMConfig(BaseModel): """Comprehensive LLM configuration mapping every node to primary/fallback models.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} planner: LLMWithFallback utils: LLMConfigUtils summarizer: LLMWithFallback @@ -167,7 +144,7 @@ class LLMConfig(BaseModel): """Validate credentials across all configured agent nodes.""" self.planner.validate_provider("Planner") self.utils.outputter.validate_provider("Outputter") - self.utils.hopper.validate_provider("Hopper") + self.utils.entity_extractor.validate_provider("EntityExtractor") if self.utils.video_analyzer: self.utils.video_analyzer.validate_provider("VideoAnalyzer") if self.utils.object_detector: @@ -196,7 +173,7 @@ class LLMConfig(BaseModel): return f""" 📃 Planner: {self.planner} 🧩 Utils: - 🔽 Hopper: {self.utils.hopper} + 🔍 Entity Extractor: {self.utils.entity_extractor} 📝 Outputter: {self.utils.outputter} 🎬 Video Analyzer: {self.utils.video_analyzer or "Not configured"} 👁️ Object Detector: {self.utils.object_detector or "Not configured"} @@ -219,7 +196,7 @@ class LLMConfig(BaseModel): def get_utils(self, item: LLMUtilsNode) -> LLMWithFallback: """Retrieve model configuration for a specific utility node.""" - value = getattr(self.utils, item) + value = getattr(self.utils, item, None) if value is None: raise ValueError( f"Utils '{item}' is not configured. Please add it to your LLM " @@ -264,7 +241,7 @@ def _expand_default_into_nodes(config_dict: dict) -> dict: all_utils_nodes = [ "outputter", - "hopper", + "entity_extractor", "video_analyzer", "object_detector", ] diff --git a/artemis/config/output.py b/artemis/config/output.py index 1955b9a..02967ae 100644 --- a/artemis/config/output.py +++ b/artemis/config/output.py @@ -26,7 +26,6 @@ from artemis.config.constants import ( ENV_EVENTS_OUTPUT_PATH, ENV_RESULTS_OUTPUT_PATH, ) -from artemis.config.llm import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -34,8 +33,6 @@ logger = get_logger(__name__) class OutputConfig(BaseModel): """Configuration for structured output formatting and schemas.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} structured_output: Annotated[ type[BaseModel] | dict | None, Field( diff --git a/artemis/config/settings.py b/artemis/config/settings.py index 7129dd8..0856a06 100644 --- a/artemis/config/settings.py +++ b/artemis/config/settings.py @@ -18,7 +18,7 @@ import os from pathlib import Path from dotenv import load_dotenv -from pydantic import Field, SecretStr, model_validator +from pydantic import AliasChoices, Field, SecretStr, model_validator from pydantic_settings import BaseSettings from artemis.config.constants import ( @@ -32,6 +32,7 @@ from artemis.config.constants import ( ENV_GEMINI_API_KEY, ENV_GOOGLE_API_KEY, ENV_OCR_API_KEY, + ENV_OPENROUTER_API_KEY, ENV_OPEN_ROUTER_API_KEY, ENV_OPENAI_API_KEY, ENV_VISION_API_KEY, @@ -99,6 +100,10 @@ class Settings(BaseSettings): GCP_API_KEY: SecretStr | None = None ANTHROPIC_API_KEY: SecretStr | None = None XAI_API_KEY: SecretStr | None = None + OPENROUTER_API_KEY: SecretStr | None = Field( + default=None, + validation_alias=AliasChoices("OPENROUTER_API_KEY", "OPEN_ROUTER_API_KEY"), + ) OPEN_ROUTER_API_KEY: SecretStr | None = None # Google Cloud Vision OCR Authentication @@ -163,6 +168,7 @@ class Settings(BaseSettings): "GCP_API_KEY", "ANTHROPIC_API_KEY", "XAI_API_KEY", + "OPENROUTER_API_KEY", "OPEN_ROUTER_API_KEY", "OCR_API_KEY", "VISION_API_KEY", @@ -172,6 +178,12 @@ class Settings(BaseSettings): if val and is_placeholder_key(val): setattr(self, attr, None) + # Synchronize OpenRouter keys + if not self.OPENROUTER_API_KEY and self.OPEN_ROUTER_API_KEY: + self.OPENROUTER_API_KEY = self.OPEN_ROUTER_API_KEY + elif not self.OPEN_ROUTER_API_KEY and self.OPENROUTER_API_KEY: + self.OPEN_ROUTER_API_KEY = self.OPENROUTER_API_KEY + if not self.GOOGLE_API_KEY: if self.GEMINI_API_KEY: self.GOOGLE_API_KEY = self.GEMINI_API_KEY @@ -204,8 +216,8 @@ class Settings(BaseSettings): key = self.OPENAI_API_KEY elif provider_lower in ("anthropic", "claude"): key = self.ANTHROPIC_API_KEY - elif provider_lower == "openrouter": - key = self.OPEN_ROUTER_API_KEY + elif provider_lower in ("openrouter", "open_router"): + key = self.OPENROUTER_API_KEY or self.OPEN_ROUTER_API_KEY elif provider_lower in ("xai", "grok"): key = self.XAI_API_KEY @@ -243,9 +255,11 @@ class Settings(BaseSettings): self.ANTHROPIC_API_KEY = secret env_key_name = ENV_ANTHROPIC_API_KEY os.environ[ENV_ANTHROPIC_API_KEY] = key - elif provider_lower == "openrouter": + elif provider_lower in ("openrouter", "open_router"): + self.OPENROUTER_API_KEY = secret self.OPEN_ROUTER_API_KEY = secret - env_key_name = ENV_OPEN_ROUTER_API_KEY + env_key_name = ENV_OPENROUTER_API_KEY + os.environ[ENV_OPENROUTER_API_KEY] = key os.environ[ENV_OPEN_ROUTER_API_KEY] = key elif provider_lower == "xai": self.XAI_API_KEY = secret diff --git a/artemis/context.py b/artemis/context.py index 9e51ed7..fa698db 100644 --- a/artemis/context.py +++ b/artemis/context.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. """Context variables for global state management. diff --git a/artemis/controllers/device_controller.py b/artemis/controllers/device_controller.py index d5c2457..32723da 100644 --- a/artemis/controllers/device_controller.py +++ b/artemis/controllers/device_controller.py @@ -11,45 +11,42 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. + +"""Abstract device controller contract and telemetry interfaces for ARTEMIS. + +Defines the hardware interaction lifecycle including touch gestures, text entry, +process lifecycle operations, display capture, and multi-segment recording. +""" + +from __future__ import annotations from abc import ABC, abstractmethod from typing import Any +from pydantic import BaseModel, ConfigDict, Field + from artemis.controllers.types import Bounds, CoordinatesSelectorRequest, TapOutput from artemis.utils.video import VideoRecordingResult -from pydantic import BaseModel, Field - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass class ScreenDataResponse(BaseModel): - model_config = {"ignored_types": (CyFunctionDetector,)} - base64: str = Field(default="", description="Base64 encoded screenshot string") - elements: list[dict[str, Any]] = Field( - default_factory=list, description="Parsed UI hierarchy elements" - ) - width: int = Field(default=1080, description="Device screen width in pixels") - height: int = Field(default=2400, description="Device screen height in pixels") - platform: str = Field(default="android", description="Platform identifier") + """Aggregate snapshot containing live display frames and parsed node hierarchies.""" + + model_config = ConfigDict(extra="ignore") + + base64: str = Field(default="", description="Base64 encoded JPEG/PNG frame capture.") + elements: list[dict[str, Any]] = Field(default_factory=list, description="Parsed hierarchy node elements.") + width: int = Field(default=1080, description="Active viewport width in physical pixels.") + height: int = Field(default=2400, description="Active viewport height in physical pixels.") + platform: str = Field(default="android", description="Target platform identifier.") class MobileDeviceController(ABC): + """Abstract driver interface establishing device actuation and telemetry operations.""" + @abstractmethod async def tap( self, @@ -59,7 +56,8 @@ class MobileDeviceController(ABC): times: int = 1, delay_ms: int = 100, ) -> TapOutput: - raise NotImplementedError("Subclasses must implement this method") + """Dispatch single, repeated, or long-press tap events to the target coordinates.""" + raise NotImplementedError @abstractmethod async def swipe( @@ -68,161 +66,102 @@ class MobileDeviceController(ABC): end: CoordinatesSelectorRequest, duration: int = 400, ) -> str | None: - """Swipe from start to end coordinates. - - Returns error message on failure, None on success. - """ - raise NotImplementedError("Subclasses must implement this method") + """Execute directional drag or fling between two coordinate positions.""" + raise NotImplementedError @abstractmethod async def screenshot(self) -> str: - """Take a screenshot and return raw image data.""" - raise NotImplementedError("Subclasses must implement this method") + """Capture current display frame as an encoded image payload.""" + raise NotImplementedError @abstractmethod async def input_text(self, text: str) -> bool: - """Input text at the currently focused field. - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") + """Stream textual characters into the currently active input field.""" + raise NotImplementedError @abstractmethod - async def launch_app(self, package_or_bundle_id: str) -> bool: - """Launch an application by package name (Android). - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") + async def launch_app(self, package_name: str) -> bool: + """Bring target Android application package into foreground.""" + raise NotImplementedError @abstractmethod - async def terminate_app(self, package_or_bundle_id: str | None) -> bool: - """Terminate an application. - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") + async def terminate_app(self, package_name: str | None) -> bool: + """Force-stop application process matching the target package.""" + raise NotImplementedError @abstractmethod async def open_url(self, url: str) -> bool: - """Open a URL. - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") + """Dispatch system VIEW intent for the specified URI.""" + raise NotImplementedError @abstractmethod async def press_back(self) -> bool: - """Press the back button (Android). - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") + """Trigger Android BACK system navigation key event.""" + raise NotImplementedError @abstractmethod async def press_home(self) -> bool: - """Press the home button. - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") + """Trigger Android HOME system navigation key event.""" + raise NotImplementedError @abstractmethod async def press_enter(self) -> bool: - """Press the enter/return key. - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") + """Trigger ENTER / ACTION_DONE key event on virtual keyboard.""" + raise NotImplementedError @abstractmethod async def press_key(self, keycode: str) -> bool: - """Press a specific key event. - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") + """Dispatch raw Android keycode event (e.g., 'KEYCODE_TAB').""" + raise NotImplementedError @abstractmethod - async def get_ui_hierarchy(self) -> list[dict]: - """Get the UI element hierarchy. - - Returns a list of UI elements with their properties. - """ - raise NotImplementedError("Subclasses must implement this method") + async def get_ui_hierarchy(self) -> list[dict[str, Any]]: + """Retrieve flattened or nested accessibility tree node descriptors.""" + raise NotImplementedError @abstractmethod def find_element( self, - ui_hierarchy: list[dict], + ui_hierarchy: list[dict[str, Any]], resource_id: str | None = None, text: str | None = None, index: int = 0, - ) -> tuple[dict | None, Bounds | None, str | None]: - """Find a UI element in the hierarchy. - - Returns: - Tuple of (element_dict, bounds, error_message) - """ - raise NotImplementedError("Subclasses must implement this method") + ) -> tuple[dict[str, Any] | None, Bounds | None, str | None]: + """Locate target element within parsed hierarchy by ID or label matching.""" + raise NotImplementedError @abstractmethod async def cleanup(self) -> None: - """Clean up resources (e.g., stop companion processes).""" - raise NotImplementedError("Subclasses must implement this method") + """Tear down companion sessions, tunnels, and streaming handles.""" + raise NotImplementedError @abstractmethod async def erase_text(self, nb_chars: int | None = None) -> bool: - """Erase the last nb_chars characters. - - Returns True on success, False on failure. - """ - raise NotImplementedError("Subclasses must implement this method") + """Dispatch backspace events to clear trailing characters in focused field.""" + raise NotImplementedError @abstractmethod - async def get_screen_data(self) -> "ScreenDataResponse": - """Get screen data including screenshot (base64), UI hierarchy elements, - - screen dimensions, and platform. - - Returns: - ScreenDataResponse with base64 screenshot, elements, width, height, - platform - """ - raise NotImplementedError("Subclasses must implement this method") + async def get_screen_data(self) -> ScreenDataResponse: + """Capture unified screen frame, layout tree, and viewport dimensions.""" + raise NotImplementedError @abstractmethod def get_compressed_b64_screenshot(self, image_base64: str, quality: int = 50) -> str: - """Compress a base64 image. - - Returns the compressed base64 image. - """ - raise NotImplementedError("Subclasses must implement this method") + """Produce compressed image payload for bandwidth-constrained turns.""" + raise NotImplementedError @abstractmethod async def start_video_recording( self, max_duration_seconds: int = 900, ) -> VideoRecordingResult: - """Start screen recording on the device. - - Args: - max_duration_seconds: Maximum recording duration in seconds. - - Returns: - VideoRecordingResult with success status and message. - """ - raise NotImplementedError("Subclasses must implement this method") + """Initialize session screen capture recording.""" + raise NotImplementedError @abstractmethod async def stop_video_recording(self) -> VideoRecordingResult: - """Stop screen recording and retrieve the video file. - - Returns: - VideoRecordingResult with success status, message, and video_path if - successful. - """ - raise NotImplementedError("Subclasses must implement this method") + """Terminate active screen capture and finalize video artifact.""" + raise NotImplementedError @abstractmethod async def extract_segment_metadata( @@ -230,16 +169,5 @@ class MobileDeviceController(ABC): start_relative_time: float, end_relative_time: float | None = None, ) -> VideoRecordingResult: - """Get a video segment for a specific time range. - - Args: - start_relative_time: Start time in seconds relative to session - start. - end_relative_time: End time in seconds relative to session start, or - None for latest. - - Returns: - VideoRecordingResult with success status, message, and video_path if - successful. - """ - raise NotImplementedError("Subclasses must implement this method") + """Slice video segment for the requested relative execution epoch.""" + raise NotImplementedError diff --git a/artemis/controllers/platform_specific_commands_controller.py b/artemis/controllers/platform_specific_commands_controller.py index d32832d..e544899 100644 --- a/artemis/controllers/platform_specific_commands_controller.py +++ b/artemis/controllers/platform_specific_commands_controller.py @@ -11,15 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. import os from shutil import which +import subprocess import time from typing import Any from artemis.context import ArtemisContext, DevicePlatform from artemis.utils.logger import ArtemisLogger, get_logger -from artemis.utils.shell_utils import run_shell_command_on_host logger = get_logger(__name__) @@ -57,12 +60,17 @@ def get_first_device( if which("adb"): try: - android_output = run_shell_command_on_host("adb devices") - lines = android_output.strip().split("\n") - for line in lines: + res = subprocess.run( + ["adb", "devices"], + capture_output=True, + text=True, + check=True, + stdin=subprocess.DEVNULL, + ) + for line in res.stdout.strip().split("\n"): if "device" in line and not line.startswith("List of devices"): return line.split()[0], DevicePlatform.ANDROID, None - except RuntimeError as e: + except Exception as e: if logger: logger.error(f"ADB command failed: {e}") diff --git a/artemis/controllers/unified_controller.py b/artemis/controllers/unified_controller.py index 56e57b8..201abfe 100644 --- a/artemis/controllers/unified_controller.py +++ b/artemis/controllers/unified_controller.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. import asyncio import os diff --git a/artemis/graph/graph.py b/artemis/graph/graph.py index ab854a1..5b03467 100644 --- a/artemis/graph/graph.py +++ b/artemis/graph/graph.py @@ -511,24 +511,6 @@ def _get_active_subgoal_hashes(ctx: ArtemisContext) -> tuple[str, str | None]: return "default", None -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - def check_plan_mutation_rejections( content_before: str, content_after: str, state: State | None = None ) -> str | None: @@ -614,7 +596,6 @@ def _reject_checker_note_write(key: str, tool_call_id: str | None) -> ToolMessag class NoteArgs(BaseModel): - model_config = {"ignored_types": (CyFunctionDetector,)} key: str = Field(..., description=SAVE_NOTE_ARG_KEY_DESC) content: str = Field(..., description=SAVE_NOTE_ARG_CONTENT_DESC) @@ -852,7 +833,6 @@ def wrap_note_tool(ctx: ArtemisContext, original_tool): class UpdateNoteArgs(BaseModel): - model_config = {"ignored_types": (CyFunctionDetector,)} key: str = Field(..., description=UPDATE_NOTE_ARG_KEY_DESC) target: str = Field(..., description=UPDATE_NOTE_ARG_TARGET_DESC) replacement: str = Field(..., description=UPDATE_NOTE_ARG_REPLACEMENT_DESC) diff --git a/artemis/interfaces/cli/commands/run.py b/artemis/interfaces/cli/commands/run.py index 91a5d04..4649ceb 100644 --- a/artemis/interfaces/cli/commands/run.py +++ b/artemis/interfaces/cli/commands/run.py @@ -26,7 +26,6 @@ from artemis.config import checker_overrides_for_level, initialize_llm_config, s from artemis.utils.startup_progress import publish_startup_progress from artemis import Agent, Builders from artemis.sdk.types.task import AgentProfile -from artemis.utils.cli_helpers import display_device_status from artemis.utils.logger import get_logger from artemis.utils.video import check_ffmpeg_available import signal @@ -37,6 +36,23 @@ import typer logger = get_logger(__name__) +def _display_device_status(console: Console, adb_client: AdbClient | None = None) -> None: + """Checks for connected devices and displays the status.""" + console.print("\n[bold]📱 Device Status[/bold]") + devices = None + if adb_client is not None: + try: + devices = adb_client.device_list() + except Exception: + devices = None + if devices: + console.print("✅ [bold green]Android device(s) connected:[/bold green]") + for device in devices: + console.print(f" - {device.serial}") + else: + console.print("❌ [bold red]No Android device found.[/bold red]") + + async def execute_task( goal: str, device_serial: str | None = None, @@ -451,7 +467,7 @@ def run_command( # Optional cosmetic device-status display; run continues without it. logger.debug(f"Could not create ADB client for device status display: {exc}") - display_device_status(console, adb_client=adb_client) + _display_device_status(console, adb_client=adb_client) cancelled = False original_sigterm = None diff --git a/artemis/resources/config/artemis.jsonc b/artemis/resources/config/artemis.jsonc index 517d738..98ce44b 100644 --- a/artemis/resources/config/artemis.jsonc +++ b/artemis/resources/config/artemis.jsonc @@ -93,8 +93,8 @@ // 📋 Summarizer: Aggregates step history and generates structured outcome "summarizer": {}, - // 🔽 Hopper: Fast lightweight package name locator - "hopper": { + // 🔍 Entity Extractor: Fast lightweight package name locator and entity extractor + "entity_extractor": { "model": "gemini-3.5-flash-lite", "fallback": { "provider": "google", diff --git a/artemis/sdk/agent.py b/artemis/sdk/agent.py index 389e677..dee930e 100644 --- a/artemis/sdk/agent.py +++ b/artemis/sdk/agent.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. import asyncio import inspect @@ -137,7 +140,6 @@ class Agent: _current_task: asyncio.Task | None = None _task_lock: asyncio.Lock - _cloud_mobile_id: str | None = None def __init__( self, diff --git a/artemis/sdk/builders/agent_config_builder.py b/artemis/sdk/builders/agent_config_builder.py index 4786005..d4684fb 100644 --- a/artemis/sdk/builders/agent_config_builder.py +++ b/artemis/sdk/builders/agent_config_builder.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. """Builder for AgentConfig objects using a fluent interface.""" @@ -69,7 +72,6 @@ class AgentConfigBuilder: self._video_recording_tools_enabled: bool = detect_video_tools_enabled() self._force_web_accessibility: bool = False self._disable_checker: bool = False - self._cloud_mobile_id_or_ref: str | None = None self._concurrency_mode: str = "per_device" self._max_concurrency: int | None = None @@ -152,12 +154,6 @@ class AgentConfigBuilder: platform_or_device_id: DevicePlatform or unique identifier for the device device_id: The unique identifier for the device (if platform was passed first) """ - if self._cloud_mobile_id_or_ref is not None: - raise ValueError( - "Device ID cannot be set when a cloud mobile is already" - " configured.\n> for_device() and for_cloud_mobile() are" - " mutually exclusive" - ) if isinstance(platform_or_device_id, DevicePlatform): self._device_platform = platform_or_device_id self._device_id = device_id diff --git a/artemis/sdk/builders/task_request_builder.py b/artemis/sdk/builders/task_request_builder.py index 320c9e5..c352752 100644 --- a/artemis/sdk/builders/task_request_builder.py +++ b/artemis/sdk/builders/task_request_builder.py @@ -11,123 +11,71 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. -"""Builder for TaskRequest objects using a fluent interface.""" +"""Fluent specification builders for ARTEMIS automation task requests. + +Provides type-safe configuration chaining for task objectives, execution bounds, +device sandboxing constraints, and structured schema validations. +""" + +from __future__ import annotations from pathlib import Path -from typing import Generic, TypeVar, cast +from typing import Generic, Self, TypeVar, cast -try: - from typing import Self -except ImportError: - from typing import Self +from pydantic import BaseModel from artemis.constants import RECURSION_LIMIT from artemis.sdk.types.agent import AgentProfile from artemis.sdk.types.task import TaskRequest, TaskRequestCommon -from pydantic import BaseModel TIn = TypeVar("TIn", bound=BaseModel | None) TOut = TypeVar("TOut", bound=BaseModel) -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) +class TaskRequestCommonBuilder: + """Configurator for common execution bounds and sandbox constraints.""" - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class TaskRequestCommonBuilder(BaseModel): - """Builder class providing a fluent interface for creating TaskRequestCommon objects.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - - def __init__(self): - self._max_steps = RECURSION_LIMIT - self._record_trace = True - self._trace_path = Path("traces") + def __init__(self) -> None: + self._max_steps: int = RECURSION_LIMIT + self._record_trace: bool = True + self._trace_path: Path = Path("traces") self._llm_output_path: Path | None = None self._locked_app_package: str | None = None self._app_path: Path | None = None def with_max_steps(self, max_steps: int) -> Self: - """Set the maximum number of steps the task can take. - - Args: - max_steps: Maximum number of steps - """ + """Constrain the maximum interaction turns allowed.""" self._max_steps = max_steps return self - def with_trace_recording(self, enabled: bool = True, path: str | None = None) -> Self: - """Configure trace recording for the task. - - Traces record screenshots and actions during execution. - - Args: - enabled: Whether to enable trace recording - path: Directory path where traces should be saved - """ + def with_trace_recording(self, enabled: bool = True, path: str | Path | None = None) -> Self: + """Toggle telemetry snapshot capture and set output storage directory.""" self._record_trace = enabled if enabled and path: self._trace_path = Path(path) return self - def with_llm_output_saving(self, path: str) -> Self: - """Configure LLM output saving for the task. - - Args: - path: Path where to save the LLM output message - """ + def with_llm_output_saving(self, path: str | Path) -> Self: + """Designate destination path for persisting raw reasoning payloads.""" self._llm_output_path = Path(path) return self def with_locked_app_package(self, package_name: str) -> Self: - """Set the app package to lock execution to. - - This ensures the specified app is launched and in the foreground before - the agentic loop starts. - - Args: - package_name: Package name (Android, e.g., 'com.whatsapp') - """ - self._locked_app_package = package_name + """Bind test execution strictly to the target Android package namespace.""" + self._locked_app_package = package_name.strip() return self def with_app_path(self, app_path: str | Path) -> Self: - """Set the path to an app to install before running the task. - - For Android: Path to an APK file. - - The app will be installed automatically before the task starts. - - Args: - app_path: Path to the app file to install - """ - self._app_path = Path(app_path) if isinstance(app_path, str) else app_path + """Provide path to local APK for automatic installation prior to run.""" + self._app_path = Path(app_path) return self def build(self) -> TaskRequestCommon: - """Build the TaskRequestCommon object. - - Returns: - A configured TaskRequestCommon object - - Raises: - ValueError: If required fields are missing - """ + """Materialize configured constraints into a TaskRequestCommon contract.""" return TaskRequestCommon( max_steps=self._max_steps, record_trace=self._record_trace, @@ -139,110 +87,66 @@ class TaskRequestCommonBuilder(BaseModel): class TaskRequestBuilder(TaskRequestCommonBuilder, Generic[TIn]): - """Builder class providing a fluent interface for creating TaskRequest objects. + """Fluent configurator constructing immutable TaskRequest specifications.""" - This builder allows for step-by-step construction of a TaskRequest with - clear methods that make the configuration process intuitive and type-safe. - - Examples: - >>> builder = TaskRequestBuilder[None](goal="Open Gmail and check unread - emails") - >>> task_request = ( - ... builder - ... .with_max_steps(30) - ... .using_profile("LowReasoning") - ... .with_output_description("A list of email subjects and senders") - ... .build() - ... ) - """ - - model_config = {"ignored_types": (CyFunctionDetector,)} - - def __init__(self, goal: str): - """Initialize an empty TaskRequestBuilder.""" + def __init__(self, goal: str) -> None: super().__init__() - self._goal = goal + self._goal: str = goal self._profile: str | AgentProfile | None = None self._name: str | None = None - self._output_description = None + self._output_description: str | None = None self._output_format: type[TIn] | None = None @classmethod - def from_common(cls, goal: str, common: TaskRequestCommon): - res = cls(goal=goal) - res._max_steps = common.max_steps - res._record_trace = common.record_trace - res._trace_path = common.trace_path - res._llm_output_path = common.llm_output_path - res._locked_app_package = common.locked_app_package - res._app_path = common.app_path - return res + def from_common(cls, goal: str, common: TaskRequestCommon) -> TaskRequestBuilder[None]: + """Derive a new task builder initialized with existing environment constraints.""" + builder = TaskRequestBuilder[None](goal=goal) + builder._max_steps = common.max_steps + builder._record_trace = common.record_trace + builder._trace_path = common.trace_path + builder._llm_output_path = common.llm_output_path + builder._locked_app_package = common.locked_app_package + builder._app_path = common.app_path + return builder - def using_profile(self, profile: str | AgentProfile) -> "TaskRequestBuilder[TIn]": - """Set the agent profile for executing the task. - - Args: - profile: The agent profile to use - """ + def using_profile(self, profile: str | AgentProfile) -> Self: + """Select execution profile (e.g., 'flash', 'pro', or custom AgentProfile).""" self._profile = profile return self - def with_name(self, name: str) -> "TaskRequestBuilder[TIn]": - """Set the name of the task - useful when recording traces. - - Otherwise, a random name will be generated. - - Args: - name: Name of the task - """ - self._name = name + def with_name(self, name: str) -> Self: + """Assign human-readable identifier to this test task.""" + self._name = name.strip() return self def without_llm_output_saving(self) -> Self: - """Disable LLM output saving for the task.""" + """Omit persisting standalone model reasoning outputs.""" self._llm_output_path = None return self - def with_output_description(self, description: str) -> "TaskRequestBuilder[TIn]": - """Set the description of the expected output format. - - This is especially useful for data extraction tasks. - - Args: - description: Description of the expected output format - """ + def with_output_description(self, description: str) -> Self: + """Supply natural language extraction instructions for unformatted deliverables.""" self._output_description = description return self - def with_output_format(self, output_format: type[TOut]) -> "TaskRequestBuilder[TOut]": - """Set the pydantic model for the expected output format. - - Args: - output_format: Pydantic model instance defining the output format - """ - self._output_format = output_format # type: ignore + def with_output_format(self, output_format: type[TOut]) -> TaskRequestBuilder[TOut]: + """Bind output schema to a structured Pydantic model class.""" + self._output_format = cast(type[TIn], output_format) return cast(TaskRequestBuilder[TOut], self) def build(self) -> TaskRequest[TIn]: - """Build the TaskRequest object. + """Validate criteria and construct final TaskRequest instance.""" + if not self._goal or not self._goal.strip(): + raise ValueError("Task goal must be a non-empty natural language objective.") - Returns: - A configured TaskRequest object + if self._output_format is not None and self._output_description is not None: + raise ValueError("Cannot specify both output_format and output_description simultaneously.") - Raises: - ValueError: If required fields are missing - """ - if not self._goal: - raise ValueError("Task goal is required") + profile_name = self._profile.name if isinstance(self._profile, AgentProfile) else self._profile - if self._output_format and self._output_description: - raise ValueError("Output format and description are mutually exclusive") - - task_request = TaskRequest( - goal=self._goal, - profile=self._profile.name - if isinstance(self._profile, AgentProfile) - else self._profile, + return TaskRequest( + goal=self._goal.strip(), + profile=profile_name, task_name=self._name, output_description=self._output_description, output_format=self._output_format, @@ -253,4 +157,3 @@ class TaskRequestBuilder(TaskRequestCommonBuilder, Generic[TIn]): locked_app_package=self._locked_app_package, app_path=self._app_path, ) - return task_request diff --git a/artemis/sdk/examples/accessibility_helper_diagnostics.py b/artemis/sdk/examples/accessibility_helper_diagnostics.py new file mode 100644 index 0000000..516126e --- /dev/null +++ b/artemis/sdk/examples/accessibility_helper_diagnostics.py @@ -0,0 +1,53 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example: Accessibility Helper Diagnostics & Resilient Hierarchy Dumps. + +This example shows how ARTEMIS uses its managed Accessibility Helper APK +to perform conflict-free UI tree inspection alongside external test frameworks +(e.g., Mobly, Appium) without locking the Android UiAutomation service. +""" + +import asyncio +from artemis.controllers.platform_specific_commands_controller import get_first_device +from artemis.drivers.android.adb_driver import AndroidAdbDriver + + +async def main(): + device_id, platform, _ = get_first_device() + if not device_id: + print("⚠️ No connected Android device found via ADB.") + return + + print(f"📱 Connected device: {device_id} (Platform: {platform})") + + # Initialize the Artemis Android ADB Driver + driver = AndroidAdbDriver(device_id=device_id) + + # Capture screen data through Artemis managed hierarchy pipeline + screen_data = await driver.get_screen_data() + + print(f"📐 Screen Dimensions: {screen_data.width}x{screen_data.height}") + print(f"🌲 Hierarchy Elements Captured: {len(screen_data.elements)}") + if screen_data.elements: + first = screen_data.elements[0] + print(f" Root Element Class: {first.get('class', 'N/A')}") + print(f" Package: {first.get('package', 'N/A')}") + + await driver.disconnect() + print("✅ Diagnostics complete.") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/artemis/sdk/examples/android_settings_inspection.py b/artemis/sdk/examples/android_settings_inspection.py new file mode 100644 index 0000000..878fc70 --- /dev/null +++ b/artemis/sdk/examples/android_settings_inspection.py @@ -0,0 +1,69 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example: Automated Android System Settings Inspection. + +This example demonstrates how to use the ARTEMIS SDK to launch the Android +Settings app, inspect device battery and storage configurations, and retrieve +verified diagnostics as a structured response. +""" + +import asyncio +from pydantic import BaseModel, Field + +from artemis.interfaces.sdk.client import ArtemisClient +from artemis.sdk.builders.task_request_builder import TaskRequestBuilder + + +class BatteryDiagnostics(BaseModel): + """Structured report of Android battery diagnostics.""" + + battery_percentage: str | None = Field( + default=None, description="Current battery percentage level visible on screen" + ) + battery_saver_status: str | None = Field( + default=None, description="Battery Saver state (e.g. On, Off)" + ) + diagnostics_summary: str = Field( + description="Summary of the battery health and system status" + ) + + +async def main(): + # Initialize the Artemis SDK client + client = ArtemisClient() + + # Build the task request targeting Android Settings + task_request = ( + TaskRequestBuilder(goal="Open Android Settings, navigate to Battery, and extract the current battery level and status.") + .with_name("settings_battery_inspection") + .with_max_steps(15) + .with_locked_app_package("com.android.settings") + .with_output_format(BatteryDiagnostics) + .build() + ) + + print(f"🚀 Running Artemis Task: {task_request.task_name}...") + task = await client.run_task(task_request) + + print(f"✅ Task Status: {task.status}") + if task.result and task.result.content: + report = task.result.get_as_model(BatteryDiagnostics) + print(f"📊 Battery Level: {report.battery_percentage}") + print(f"⚡ Battery Saver: {report.battery_saver_status}") + print(f"📝 Summary: {report.diagnostics_summary}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/artemis/sdk/examples/app_lock_messaging.py b/artemis/sdk/examples/app_lock_messaging.py deleted file mode 100644 index e89e7fb..0000000 --- a/artemis/sdk/examples/app_lock_messaging.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio - -from artemis.sdk import Agent -from pydantic import BaseModel, Field - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class MessageResult(BaseModel): - """Structured result from messaging task.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - - messages_sent: int = Field(..., description="Number of messages successfully sent") - contacts: list[str] = Field(..., description="List of contacts messaged") - success: bool = Field(..., description="Whether all messages were sent successfully") - - -async def main() -> None: - # Create agent with default configuration - agent = Agent() - - try: - await agent.init() - - # Use app lock to keep execution in WhatsApp - # This ensures the agent stays in the app and relaunches if needed - task = ( - agent.new_task("Send 'Happy New Year!' message to Alice, Bob, and Charlie on WhatsApp") - .with_name("send_new_year_messages") - .with_locked_app_package("com.whatsapp") # Lock to WhatsApp - .with_output_format(MessageResult) - .with_max_steps(600) # Messaging tasks may need more steps - .build() - ) - - print("Sending messages with app lock enabled...") - print("The agent will stay in WhatsApp and relaunch if needed.\n") - - result = await agent.run_task(request=task) - - if result: - print("\n=== Messaging Complete ===") - print(f"Messages sent: {result.messages_sent}") - print(f"Contacts: {', '.join(result.contacts)}") - print(f"Success: {result.success}") - else: - print("Failed to send messages") - - except Exception as e: - print(f"Error: {e}") - finally: - await agent.clean() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/artemis/sdk/examples/app_workflow_automation.py b/artemis/sdk/examples/app_workflow_automation.py new file mode 100644 index 0000000..92cc11b --- /dev/null +++ b/artemis/sdk/examples/app_workflow_automation.py @@ -0,0 +1,53 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example: App Workflow Automation with Trace Recording. + +This example demonstrates how to orchestrate multi-step mobile UI navigation, +capture step-by-step traces, and enforce deterministic completion criteria +using ARTEMIS. +""" + +import asyncio +from artemis.interfaces.sdk.client import ArtemisClient +from artemis.sdk.builders.task_request_builder import TaskRequestBuilder + + +async def main(): + client = ArtemisClient() + + # Build a task with full trace recording + task_request = ( + TaskRequestBuilder( + goal="Open Clock app, verify current alarm settings, and ensure the active alarms list is visible." + ) + .with_name("clock_alarm_audit") + .with_max_steps(12) + .with_locked_app_package("com.google.android.deskclock") + .with_trace_recording(enabled=True, path="traces/clock_alarm_audit") + .with_output_description("Summary of configured alarms and their status") + .build() + ) + + print(f"🎬 Initiating Workflow Automation: {task_request.task_name}...") + task = await client.run_task(task_request) + + print(f"🏁 Execution Finished: status={task.status}") + if task.result: + print(f"⏱️ Duration: {task.result.execution_time_seconds:.2f}s, Steps: {task.result.steps_taken}") + print(f"📄 Output: {task.result.content}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/artemis/sdk/examples/multimodal_vision_testing.py b/artemis/sdk/examples/multimodal_vision_testing.py new file mode 100644 index 0000000..1859d04 --- /dev/null +++ b/artemis/sdk/examples/multimodal_vision_testing.py @@ -0,0 +1,68 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Example: Multimodal UI Testing and Screen Element Grounding. + +This example demonstrates using ARTEMIS multimodal perception (Explorer & OCR-XML +fusion) to locate visual elements on dynamic or non-standard UI canvases without +relying on static coordinates. +""" + +import asyncio +from pydantic import BaseModel, Field + +from artemis.interfaces.sdk.client import ArtemisClient +from artemis.sdk.builders.task_request_builder import TaskRequestBuilder + + +class VisualAuditReport(BaseModel): + """Structured report verifying visual UI layout integrity.""" + + active_screen_title: str = Field(description="Title of the active screen or view") + interactive_buttons_count: int = Field( + description="Number of actionable buttons identified on the current view" + ) + visual_anomalies_detected: bool = Field( + description="True if unexpected popups, overlapping text, or error banners exist" + ) + observations: str = Field(description="Visual layout observations and audit details") + + +async def main(): + client = ArtemisClient() + + task_request = ( + TaskRequestBuilder( + goal="Open Calculator, verify numeric keypad buttons layout, and confirm display screen is empty." + ) + .with_name("calculator_visual_audit") + .with_max_steps(8) + .with_locked_app_package("com.google.android.calculator") + .with_output_format(VisualAuditReport) + .build() + ) + + print("🔍 Starting Multimodal Vision Audit...") + task = await client.run_task(task_request) + + if task.result and task.result.content: + audit = task.result.get_as_model(VisualAuditReport) + print(f"📱 View: {audit.active_screen_title}") + print(f"🔘 Interactive Buttons: {audit.interactive_buttons_count}") + print(f"⚠️ Anomalies Detected: {audit.visual_anomalies_detected}") + print(f"📝 Observations: {audit.observations}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/artemis/sdk/examples/simple_photo_organizer.py b/artemis/sdk/examples/simple_photo_organizer.py deleted file mode 100644 index 6650d66..0000000 --- a/artemis/sdk/examples/simple_photo_organizer.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Simple Photo Organizer - Basic SDK Usage Example - -This example demonstrates a straightforward way to use the Artemis SDK -without builders or advanced configuration. It performs a real-world automation -task: -1. Opens the photo gallery -2. Finds photos from a specific date -3. Creates an album and moves those photos into it - -Run: -- python artemis/sdk/examples/simple_photo_organizer.py -""" - -import asyncio -from datetime import date, timedelta -from artemis.sdk import Agent -from pydantic import BaseModel, Field - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class PhotosResult(BaseModel): - """Structured result from photo search.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - - found_photos: int = Field(..., description="Number of photos found") - date_range: str = Field(..., description="Date range of photos found") - album_created: bool = Field(..., description="Whether an album was created") - album_name: str = Field(..., description="Name of the created album") - photos_moved: int = Field(0, description="Number of photos moved to the album") - - -async def main() -> None: - # Create a simple agent with default configuration - agent = Agent() - - try: - # Initialize agent (finds a device, starts required servers) - await agent.init() - - # Calculate yesterday's date for the example - yesterday = date.today() - timedelta(days=1) - formatted_date = yesterday.strftime("%B %d") # e.g. "August 22" - - print(f"Looking for photos from {formatted_date}...") - - # First task: search for photos and organize them, with typed output - result = await agent.run_task( - goal=( - "Open the Photos/Gallery app. Find photos taken on" - f" {formatted_date}. Create a new album named '{formatted_date}" - " Memories' and move those photos into it. Count how many" - " photos were moved." - ), - output=PhotosResult, - name="organize_photos", - ) - - # Handle and display the result - if result: - print("\n=== Photo Organization Complete ===") - print(f"Found: {result.found_photos} photos from {result.date_range}") - - if result.album_created: - print(f"Created album: '{result.album_name}'") - print(f"Moved {result.photos_moved} photos to the album") - else: - print("No album was created") - else: - print("Failed to organize photos") - - except Exception as e: - print(f"Error: {e}") - finally: - # Always clean up resources - await agent.clean() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/artemis/sdk/examples/smart_notification_assistant.py b/artemis/sdk/examples/smart_notification_assistant.py deleted file mode 100644 index 297ea08..0000000 --- a/artemis/sdk/examples/smart_notification_assistant.py +++ /dev/null @@ -1,235 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Smart Notification Assistant - Intermediate SDK Usage Example - -This example demonstrates more advanced SDK features including: -- TaskRequestBuilder pattern -- Multiple agent profiles for different reasoning tasks -- Tracing for debugging/visualization -- Structured output with Pydantic -- Exception handling - -It performs a practical automation task: -1. Checks notification panel for unread notifications -2. Categorizes them by priority/app -3. Performs actions based on notification content - -Run: -- python artemis/sdk/examples/smart_notification_assistant.py -""" - -import asyncio -from datetime import datetime - -try: - from enum import StrEnum -except ImportError: - from enum import Enum - - class StrEnum(str, Enum): - pass - - -from artemis.config import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback -from artemis.sdk import Agent -from artemis.sdk.builders import Builders -from artemis.sdk.types import AgentProfile -from artemis.sdk.types.exceptions import AgentError -from pydantic import BaseModel, Field - - -class NotificationPriority(StrEnum): - HIGH = "high" - MEDIUM = "medium" - LOW = "low" - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class Notification(BaseModel): - """Individual notification details.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - - app_name: str = Field(..., description="Name of the app that sent the notification") - title: str = Field(..., description="Title/header of the notification") - message: str = Field(..., description="Message content of the notification") - priority: NotificationPriority = Field( - default=NotificationPriority.MEDIUM, - description="Priority level of notification", - ) - - -class NotificationSummary(BaseModel): - """Summary of all notifications.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - - total_count: int = Field(..., description="Total number of notifications found") - high_priority_count: int = Field(0, description="Count of high priority notifications") - notifications: list[Notification] = Field( - default_factory=list, description="List of individual notifications" - ) - - -def get_agent() -> Agent: - # Create two specialized profiles: - # 1. An analyzer profile for detailed inspection tasks - analyzer_profile = AgentProfile( - name="analyzer", - llm_config=LLMConfig( - planner=LLMWithFallback( - provider="openrouter", - model="meta-llama/llama-4-scout", - fallback=LLM(provider="openrouter", model="meta-llama/llama-4-maverick"), - ), - utils=LLMConfigUtils( - outputter=LLMWithFallback( - provider="openai", - model="gpt-5-nano", - fallback=LLM(provider="openai", model="gpt-5-mini"), - ), - hopper=LLMWithFallback( - provider="openai", - model="gpt-5-nano", - fallback=LLM(provider="openai", model="gpt-5-mini"), - ), - ), - ), - # from_file="/tmp/analyzer.jsonc" # can be loaded from file - ) - - # 2. An action profile for handling easy & fast actions based on notifications - action_profile = AgentProfile( - name="note_taker", - llm_config=LLMConfig( - planner=LLMWithFallback( - provider="openai", - model="o3", - fallback=LLM(provider="openai", model="gpt-5"), - ), - utils=LLMConfigUtils( - outputter=LLMWithFallback( - provider="openai", - model="gpt-5-nano", - fallback=LLM(provider="openai", model="gpt-5-mini"), - ), - hopper=LLMWithFallback( - provider="openai", - model="gpt-5-nano", - fallback=LLM(provider="openai", model="gpt-5-mini"), - ), - ), - ), - ) - - # Configure default task settings with tracing - task_defaults = Builders.TaskDefaults.with_max_steps(200).build() - - # Configure the agent - config = ( - Builders.AgentConfig.add_profiles(profiles=[analyzer_profile, action_profile]) - .with_default_profile(profile=action_profile) - .with_default_task_config(config=task_defaults) - .build() - ) - return Agent(config=config) - - -async def main(): - # Set up traces directory with timestamp for uniqueness - timestamp = datetime.now().strftime("%Y%m%d_%H%M") - traces_dir = f"/tmp/notification_traces/{timestamp}" - agent = get_agent() - - try: - # Initialize agent (finds a device, starts required servers) - await agent.init() - - print("Checking for notifications...") - - # Task 1: Get and analyze notifications with analyzer profile - notification_task = ( - agent.new_task( - goal=( - "Open the notification panel (swipe down from top). Scroll" - " through the first 3 unread notifications. For each" - " notification, identify the app name, title, and content." - " Tag messages from messaging apps or email as high" - " priority." - ) - ) - .with_output_format(NotificationSummary) - .using_profile("analyzer") - .with_name("notification_scan") - .with_max_steps(400) - .with_trace_recording(enabled=True, path=traces_dir) - .build() - ) - - # Execute the task with proper exception handling - try: - notifications = await agent.run_task(request=notification_task) - - # Display the structured results - if notifications: - print("\n=== Notification Summary ===") - print(f"Total notifications: {notifications.total_count}") - print(f"High priority: {notifications.high_priority_count}") - - # Task 2: Create a note to store the notification summary - response = await agent.run_task( - goal=( - "Open my Notes app and create a new note summarizing" - f" the following information:\n{notifications}" - ), - name="email_action", - profile="note_taker", - ) - print(f"Action result: {response}") - - else: - print("Failed to retrieve notifications") - - except AgentError as e: - print(f"Agent error occurred: {e}") - except Exception as e: - print(f"Unexpected error: {type(e).__name__}: {e}") - raise - - finally: - # Clean up - await agent.clean() - print(f"\nTraces saved to: {traces_dir}") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/artemis/sdk/examples/video_transcription_example.py b/artemis/sdk/examples/video_transcription_example.py deleted file mode 100644 index 5b12673..0000000 --- a/artemis/sdk/examples/video_transcription_example.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Video Transcription Example - -This example demonstrates how to use the video recording tools to capture -and analyze video content from a mobile device screen. - -The agent can: -1. Start a screen recording -2. Perform actions while recording -3. Stop the recording and analyze its content using Gemini models - -Use case: Recording a video playing on the screen and transcribing its content. -""" - -import asyncio - -from artemis.config import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback -from artemis.sdk.agent import Agent -from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder -from artemis.sdk.types.agent import AgentConfig -from artemis.sdk.types.task import AgentProfile, TaskRequest - - -def get_video_capable_llm_config() -> LLMConfig: - """Returns an LLM config with video_analyzer configured. - - The video_analyzer must use a video-capable Gemini model: - - gemini-3.8-flash (recommended - fast and capable) - - gemini-3.7-flash - - gemini-3-pro-preview - - gemini-2.5-flash - - gemini-2.5-pro - - gemini-2.0-flash - """ - return LLMConfig( - planner=LLMWithFallback( - provider="openai", - model="gpt-5-nano", - fallback=LLM(provider="openai", model="gpt-5-mini"), - ), - utils=LLMConfigUtils( - outputter=LLMWithFallback( - provider="openai", - model="gpt-5-nano", - fallback=LLM(provider="openai", model="gpt-5-mini"), - ), - hopper=LLMWithFallback( - provider="openai", - model="gpt-5-nano", - fallback=LLM(provider="openai", model="gpt-5-mini"), - ), - video_analyzer=LLMWithFallback( - provider="google", - model="gemini-3.8-flash", - fallback=LLM(provider="google", model="gemini-3.7-flash"), - ), - ), - ) - - -async def main(): - config: AgentConfig = ( - AgentConfigBuilder() - .add_profile( - AgentProfile( - name="VideoCapable", - llm_config=get_video_capable_llm_config(), - ) - ) - .with_video_recording_tools() - .build() - ) - - agent = Agent(config=config) - try: - await agent.init() - - result = await agent.run_task( - request=TaskRequest( - goal=""" - 1. Open YouTube app - 2. Search for "Python tutorial" - 3. Start recording the screen - 4. Play the first video - 5. Wait for the first 30 seconds of the video to play - 6. Stop recording and tell me what was said in the video - """, - profile="VideoCapable", - ) - ) - print(f"Task result: {result}") - finally: - await agent.clean() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/artemis/sdk/types/__init__.py b/artemis/sdk/types/__init__.py index 9c59ed1..0e7585e 100644 --- a/artemis/sdk/types/__init__.py +++ b/artemis/sdk/types/__init__.py @@ -16,7 +16,6 @@ from artemis.sdk.types.agent import ( AgentConfig, - ApiBaseUrl, DevicePlatform, ServerConfig, ) @@ -37,10 +36,10 @@ from artemis.sdk.types.task import ( TaskRequest, TaskRequestCommon, TaskResult, + TaskRunStatus, ) __all__ = [ - "ApiBaseUrl", "AgentConfig", "DevicePlatform", "AgentProfile", @@ -48,6 +47,7 @@ __all__ = [ "TaskRequest", "TaskResult", "TaskRequestCommon", + "TaskRunStatus", "Task", "AgentProfileNotFoundError", "AgentTaskRequestError", diff --git a/artemis/sdk/types/agent.py b/artemis/sdk/types/agent.py index 79a934a..72f9b15 100644 --- a/artemis/sdk/types/agent.py +++ b/artemis/sdk/types/agent.py @@ -11,9 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. from typing import Literal -from urllib.parse import urlparse from langchain_core.callbacks.base import Callbacks from artemis.config import ( @@ -29,63 +31,8 @@ from artemis.utils.video import detect_video_tools_enabled from pydantic import BaseModel, Field -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class ApiBaseUrl(BaseModel): - """Defines an API base URL.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - scheme: Literal["http", "https"] - host: str - port: int | None = None - - def __eq__(self, other): - if not isinstance(other, ApiBaseUrl): - return False - return self.to_url() == other.to_url() - - def to_url(self): - return ( - f"{self.scheme}://{self.host}:{self.port}" - if self.port is not None - else f"{self.scheme}://{self.host}" - ) - - @classmethod - def from_url(cls, url: str) -> "ApiBaseUrl": - parsed_url = urlparse(url) - if parsed_url.scheme not in ["http", "https"]: - raise ValueError(f"Invalid scheme: {parsed_url.scheme}") - if parsed_url.hostname is None: - raise ValueError("Invalid hostname") - return cls( - scheme=parsed_url.scheme, # type: ignore - host=parsed_url.hostname, - port=parsed_url.port, - ) - - class ServerConfig(BaseModel): """Configuration for the required servers.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - adb_host: str adb_port: int diff --git a/artemis/sdk/types/exceptions.py b/artemis/sdk/types/exceptions.py index 63288cc..41c7741 100644 --- a/artemis/sdk/types/exceptions.py +++ b/artemis/sdk/types/exceptions.py @@ -12,94 +12,95 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Exceptions for the ARTEMIS SDK. +"""Hierarchical exception taxonomy for the ARTEMIS client and automation runtime. -This module defines the exception hierarchy used throughout the Artemis SDK. +Establishes categorized exception types covering host device discovery, +daemon lifecycle management, agent operational faults, and toolchain dependencies. """ +from __future__ import annotations + from typing import Literal +_TOOLCHAIN_INSTALL_URLS: dict[str, str] = { + "adb": "https://developer.android.com/tools/adb", +} + class ArtemisError(Exception): - """Base exception class for all ARTEMIS SDK exceptions.""" + """Root ancestor of all exceptions originating from ARTEMIS operations.""" - def __init__(self, message="An error occurred in the Artemis SDK"): + def __init__(self, message: str = "An unexpected error occurred within the ARTEMIS runtime.") -> None: self.message = message super().__init__(self.message) class DeviceError(ArtemisError): - """Exception raised for errors related to mobile devices.""" + """Raised when hardware interaction or ADB communication fails.""" - def __init__(self, message="A device-related error occurred"): + def __init__(self, message: str = "A mobile device error occurred.") -> None: super().__init__(message) class DeviceNotFoundError(DeviceError): - """Exception raised when no mobile device is found.""" + """Raised when no authorized target Android device or emulator is accessible.""" - def __init__(self, message="No mobile device found"): + def __init__(self, message: str = "No connected and authorized Android device found.") -> None: super().__init__(message) class ServerError(ArtemisError): - """Exception raised for errors related to ARTEMIS servers.""" + """Raised when an internal ARTEMIS daemon or background service encounters an error.""" - def __init__(self, message="A server-related error occurred"): + def __init__(self, message: str = "An ARTEMIS daemon service failure occurred.") -> None: super().__init__(message) class ServerStartupError(ServerError): - """Exception raised when ARTEMIS servers fail to start.""" + """Raised when a background process (e.g. Scrcpy, MCP, or local server) fails to boot.""" - def __init__(self, server_name=None, message=None): - if server_name and not message: - message = f"Failed to start {server_name}" - elif not message: - message = "Failed to start Artemis servers" - super().__init__(message) + def __init__(self, server_name: str | None = None, message: str | None = None) -> None: + resolved_msg = message or (f"Service '{server_name}' failed to start." if server_name else "Service startup failed.") + super().__init__(resolved_msg) self.server_name = server_name class AgentError(ArtemisError): - """Exception raised for errors related to the ARTEMIS agent.""" + """Raised when reasoning, task scheduling, or graph traversal fails.""" - def __init__(self, message="An agent-related error occurred"): + def __init__(self, message: str = "An agent operational failure occurred.") -> None: super().__init__(message) class AgentNotInitializedError(AgentError): - """Exception raised when attempting operations on an uninitialized agent.""" + """Raised when invoking automation actions before initialization.""" - def __init__(self, message="Agent is not initialized. Call init() first"): + def __init__(self, message: str = "Agent runtime has not been initialized.") -> None: super().__init__(message) class AgentTaskRequestError(AgentError): - """Exception raised when a requested task is invalid.""" + """Raised when task parameters violate execution bounds or constraints.""" - def __init__(self, message="An agent task-related error occurred"): + def __init__(self, message: str = "Invalid task request configuration.") -> None: super().__init__(message) class AgentProfileNotFoundError(AgentTaskRequestError): - """Exception raised when an agent profile is not found.""" + """Raised when requested execution profile is not registered in system catalog.""" - def __init__(self, profile_name: str): - super().__init__(f"Agent profile {profile_name} not found") - - -EXECUTABLES = Literal["adb"] + def __init__(self, profile_name: str) -> None: + super().__init__(f"Agent profile '{profile_name}' is not registered.") + self.profile_name = profile_name class ExecutableNotFoundError(ArtemisError): - """Exception raised when a required executable is not found.""" + """Raised when a mandatory host CLI toolchain executable is missing from PATH.""" - def __init__(self, executable_name: EXECUTABLES): - install_instructions: dict[EXECUTABLES, str] = { - "adb": "https://developer.android.com/tools/adb", - } - message = f"Required executable '{executable_name}' not found in PATH." - if executable_name in install_instructions: - message += f"\nTo install it, please visit: {install_instructions[executable_name]}" - super().__init__(message) + def __init__(self, executable_name: Literal["adb"] | str) -> None: + guidance = _TOOLCHAIN_INSTALL_URLS.get(executable_name, "") + msg = f"Required tool '{executable_name}' was not detected in system PATH." + if guidance: + msg += f" Install instructions: {guidance}" + super().__init__(msg) + self.executable_name = executable_name diff --git a/artemis/sdk/types/platform.py b/artemis/sdk/types/platform.py deleted file mode 100644 index 5bceb4f..0000000 --- a/artemis/sdk/types/platform.py +++ /dev/null @@ -1,210 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from datetime import datetime -from typing import Annotated, Any, Literal - -from pydantic import BaseModel, ConfigDict, Field, PlainSerializer -from pydantic.v1.utils import to_lower_camel - -TaskRunStatus = Literal["pending", "running", "completed", "failed", "cancelled"] - -IsoDatetime = Annotated[ - datetime, - PlainSerializer( - func=lambda v: v.isoformat() if v else None, - return_type=str, - when_used="unless-none", - ), -] - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - -class BaseApiModel(BaseModel): - model_config = ConfigDict( - alias_generator=to_lower_camel, - populate_by_name=True, - str_strip_whitespace=True, - ) - - -class LLMProfileResponse(BaseApiModel): - """Response model for LLM profile.""" - - id: str = Field(..., description="Profile ID") - name: str = Field(..., description="Profile name") - description: str | None = Field(None, description="Profile description") - llms: dict[str, Any] = Field(..., description="LLM configuration") - created_at: str = Field(..., description="Creation timestamp (ISO format)") - updated_at: str = Field(..., description="Last update timestamp (ISO format)") - - -class TaskOptionsResponse(BaseApiModel): - """Response model for task options.""" - - id: str = Field(..., description="Options ID") - enable_tracing: bool = Field(..., description="Whether tracing is enabled") - max_steps: int = Field(..., description="Maximum number of steps") - enable_video_tools: bool = Field(..., description="Whether video tools are enabled") - created_at: str = Field(..., description="Creation timestamp (ISO format)") - updated_at: str = Field(..., description="Last update timestamp (ISO format)") - - -class TaskResponse(BaseApiModel): - """Response model for task.""" - - id: str = Field(..., description="Task ID") - name: str = Field(..., description="Task name") - description: str | None = Field(None, description="Task description") - input_prompt: str = Field(..., description="Input prompt") - output_description: str | None = Field(None, description="Output description") - options: TaskOptionsResponse = Field(..., description="Task options") - created_at: str = Field(..., description="Creation timestamp (ISO format)") - updated_at: str = Field(..., description="Last update timestamp (ISO format)") - - -class CreateTaskRunRequest(BaseApiModel): - """Request model for creating a task run.""" - - task_id: str = Field(..., description="ID of the task to run") - llm_profile_id: str = Field(..., description="LLM profile ID to use") - virtual_mobile_id: str | None = Field(None, description="Virtual mobile ID to use") - locked_app_package: str | None = Field(None, description="App package to lock for the task run") - execution_origin: str | None = Field(None, description="Origin of the task execution") - enable_video_tools: bool = Field( - default=False, - description="Whether video tools are enabled for this task run", - ) - - -class CreateOrphanTaskRunRequest(BaseApiModel): - """Request model for creating an orphan task run (without a parent task).""" - - input_prompt: str = Field( - ..., - description="Input prompt for the task run", - ) - output_description: str | None = Field( - None, - description="Optional output description", - ) - llm_profile_id: str = Field( - ..., - description="LLM profile ID to use", - ) - virtual_mobile_id: str | None = Field( - None, - description="Optional virtual mobile ID on which it runs", - ) - locked_app_package: str | None = Field( - None, - description="Optional app package to lock execution to", - ) - execution_origin: str | None = Field( - None, - description="Origin of the task run execution", - ) - max_steps: int | None = Field( - None, - description="Maximum number of steps for the task run", - ) - enable_video_tools: bool = Field( - default=False, - description="Whether video tools are enabled for this task run", - ) - - -class UpdateTaskRunStatusRequest(BaseApiModel): - """Request model for updating task run status.""" - - status: TaskRunStatus = Field(..., description="New status of the task run") - message: str | None = Field(None, description="Message associated with the status") - output: str | None = Field(None, description="Output of the task run") - - -class TaskRunResponse(BaseApiModel): - """Response model for a single task run.""" - - id: str = Field(..., description="Unique identifier for the task run") - task: TaskResponse | None = Field( - ..., - description=("ID of the task this run is for or None if manually created"), - ) - llm_profile: LLMProfileResponse = Field(..., description="LLM profile ID used for this run") - status: TaskRunStatus = Field(..., description="Current status of the task run") - input_prompt: str = Field(..., description="Input prompt for this task run") - output_description: str | None = Field(None, description="Description of expected output") - created_at: datetime = Field(..., description="When the task run was created") - started_at: datetime | None = Field(None, description="When the task run started") - finished_at: datetime | None = Field(None, description="When the task run finished") - locked_app_package: str | None = Field( - None, description="The app package to lock for the task run" - ) - - -SubgoalState = Literal["pending", "started", "completed", "failed"] - - -class ArtemisSubgoal(BaseModel): - """Upsert ArtemisSubgoal API model.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - - name: str = Field(..., description="Name of the subgoal") - state: SubgoalState = Field(default="pending", description="Current state of the subgoal") - started_at: IsoDatetime | None = Field(default=None, description="When the subgoal started") - ended_at: IsoDatetime | None = Field(default=None, description="When the subgoal ended") - - -class UpsertTaskRunPlanRequest(BaseApiModel): - """Upsert task run plan request model.""" - - started_at: IsoDatetime = Field(..., description="When the plan started") - subgoals: list[ArtemisSubgoal] = Field(..., description="Subgoals of the plan") - ended_at: IsoDatetime | None = Field( - default=None, - description="When the plan ended (replanned or completed)", - ) - - -class TaskRunPlanResponse(UpsertTaskRunPlanRequest): - """Response model for a task run plan.""" - - id: str = Field(..., description="Unique identifier for the task run plan") - task_run_id: str = Field(..., description="ID of the task run this plan is for") - - -class TrajectoryGifUploadResponse(BaseApiModel): - """Response containing trajectory GIF signed upload URL.""" - - signed_url: str = Field(..., description="Signed upload URL for the trajectory GIF") - task_run_id: str = Field(..., description="Task run ID for which the GIF URL was generated") - file_path: str = Field(..., description="Path where the GIF will be stored") diff --git a/artemis/sdk/types/task.py b/artemis/sdk/types/task.py index 8a9d482..6fdce49 100644 --- a/artemis/sdk/types/task.py +++ b/artemis/sdk/types/task.py @@ -11,67 +11,54 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. -"""Task-related type definitions for the ARTEMIS SDK.""" +"""Task specifications, execution states, and outcome schemas for the ARTEMIS SDK. + +Defines the declarative configuration contract for mobile automation goals, +agent operational profiles, telemetry preferences, and asynchronous result contracts. +""" + +from __future__ import annotations from collections.abc import Callable, Coroutine from datetime import datetime from pathlib import Path -from typing import Any, Generic, TypeVar, overload +from typing import Any, Generic, Literal, TypeVar, overload + +from pydantic import BaseModel, ConfigDict, Field from artemis.config import LLMConfig, get_default_llm_config from artemis.constants import RECURSION_LIMIT from artemis.context import DeviceContext -from artemis.sdk.types.platform import TaskRunStatus from artemis.sdk.utils import load_llm_config_override -from pydantic import BaseModel, Field - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass +TaskRunStatus = Literal["pending", "running", "completed", "failed", "cancelled"] class AgentProfile(BaseModel): - """Represents a ARTEMIS agent profile. + """Runtime configuration profile governing LLM model selection and reasoning policies.""" - Attributes: - name: Name of the agent - used to reference the agent when running - tasks. - llm_config: LLM configuration for the agent. - """ + model_config = ConfigDict(arbitrary_types_allowed=True) - model_config = {"ignored_types": (CyFunctionDetector,)} - - name: str + name: str = Field(description="Identifier for referencing this operational agent profile.") llm_config: LLMConfig = Field(default_factory=get_default_llm_config) @overload def __init__(self, *, name: str, llm_config: LLMConfig): ... @overload - def __init__(self, *, name: str, from_file: str): ... + def __init__(self, *, name: str, from_file: str | Path): ... def __init__( self, *, name: str, llm_config: LLMConfig | None = None, - from_file: str | None = None, - **kwargs, + from_file: str | Path | None = None, + **kwargs: Any, ): kwargs["name"] = name if from_file: @@ -79,11 +66,11 @@ class AgentProfile(BaseModel): elif llm_config: kwargs["llm_config"] = llm_config else: - raise ValueError("Either llm_config or from_file must be provided") + raise ValueError("Either llm_config or from_file must be specified.") super().__init__(**kwargs) - def __str__(self): - return f"Profile {self.name}:\n{self.llm_config}" + def __str__(self) -> str: + return f"AgentProfile({self.name}): {self.llm_config}" T = TypeVar("T", bound=BaseModel) @@ -91,146 +78,73 @@ TOutput = TypeVar("TOutput", bound=BaseModel | None) class TaskRequestBase(BaseModel): - """Defines common parameters of a mobile automation task request.""" + """Baseline runtime limits and telemetry recording settings for automation runs.""" + + model_config = ConfigDict(extra="ignore", arbitrary_types_allowed=True) - model_config = {"ignored_types": (CyFunctionDetector,)} max_steps: int = RECURSION_LIMIT record_trace: bool = True - trace_path: Path = Path("traces") + trace_path: Path = Field(default_factory=lambda: Path("traces")) llm_output_path: Path | None = None class TaskRequestCommon(TaskRequestBase): - """Defines common parameters for any task request.""" + """Common automation constraints including application isolation and pre-install packages.""" - model_config = {"ignored_types": (CyFunctionDetector,)} - max_steps: int = RECURSION_LIMIT locked_app_package: str | None = None app_path: Path | None = None - """Path to an app to install before running the task. - - For Android: Path to an APK file. - - The app will be installed automatically before the task starts. - """ class TaskRequest(TaskRequestCommon, Generic[TOutput]): - """Defines the format of a mobile automation task request. + """Declarative request specifying the high-level mobile testing or automation objective.""" - Attributes: - goal: Natural language description of the goal to achieve - profile: Optional agent profile to use for executing the task - task_name: Optional name for the task - output_description: Optional natural language description of expected - output format - output_format: Optional pydantic model for the output format of the task - max_steps: Maximum number of steps the agent can take (default: 20) - record_trace: Whether to record a trace (screenshots, actions) of the - execution (default: False) - trace_path: Directory path to save trace data if recording is enabled - llm_output_path: Path to save LLM output data - """ - - model_config = {"ignored_types": (CyFunctionDetector,)} - goal: str - profile: str | None = None - task_name: str | None = None - output_description: str | None = None - output_format: type[TOutput] | None = None + goal: str = Field(description="Natural language instruction outlining target behavior or assertions.") + profile: str | None = Field(default=None, description="Execution profile (e.g. 'flash' or 'pro').") + task_name: str | None = Field(default=None, description="Optional label identifying this task execution.") + output_description: str | None = Field(default=None, description="Formatting prompt for expected deliverables.") + output_format: type[TOutput] | None = Field(default=None, description="Target Pydantic schema for structured output.") enable_remote_tracing: bool = False class TaskResult(BaseModel): - """Result of a mobile automation task. + """Immutable outcome payload produced upon automation task completion.""" - Attributes: - content: Raw result content (could be text or structured data) - error: Error message if the task failed - execution_time_seconds: How long the task took to execute - steps_taken: Number of steps the agent took to complete the task - """ + model_config = ConfigDict(arbitrary_types_allowed=True) - model_config = {"ignored_types": (CyFunctionDetector,)} content: Any = None error: str | None = None - execution_time_seconds: float - steps_taken: int + execution_time_seconds: float = 0.0 + steps_taken: int = 0 + + @property + def succeeded(self) -> bool: + """True if the task terminated without error.""" + return self.error is None def get_as_model(self, model_class: type[T]) -> T: - """Parse the content into a Pydantic model instance. - - Args: - model_class: The Pydantic model class to parse the data into - - Returns: - An instance of the specified model class - - Raises: - ValueError: If content is None or not compatible with the model - """ + """Parse structured content into the requested Pydantic model instance.""" if self.content is None: - raise ValueError("No content available to parse into a model") + raise ValueError("No content available to parse into the requested schema.") if isinstance(self.content, model_class): return self.content return model_class.model_validate(self.content) class Task(BaseModel): - """A mobile automation task to be executed. + """Stateful execution container tracking lifecycle progress, status transitions, and final results.""" - Attributes: - id: Unique identifier for the task - device: Information about the target device - status: Current status of the task execution - request: User task request - created_at: ISO timestamp when the task was created - ended_at: ISO timestamp when the task ended - """ + model_config = ConfigDict(arbitrary_types_allowed=True) - model_config = {"ignored_types": (CyFunctionDetector,)} id: str device: DeviceContext - status: TaskRunStatus + status: TaskRunStatus = "pending" status_message: str | None = None on_status_changed: Callable[[TaskRunStatus, str | None, Any | None], Coroutine] | None = None - request: TaskRequest - created_at: datetime + request: TaskRequest[Any] + created_at: datetime = Field(default_factory=datetime.now) ended_at: datetime | None = None result: TaskResult | None = None - async def finalize( - self, - content: Any | None = None, - state: dict | None = None, - error: str | None = None, - cancelled: bool = False, - ): - new_status: TaskRunStatus = "completed" if error is None else "failed" - if new_status == "failed" and cancelled: - new_status = "cancelled" - message = "Task completed successfully" - if new_status == "failed": - message = "Task failed" + (f": {error}" if error else "") - elif new_status == "cancelled": - message = "Task cancelled" + (f": {error}" if error else "") - await self.set_status(status=new_status, message=message, output=content or error) - self.ended_at = datetime.now() - - duration = self.ended_at - self.created_at - steps_taken = -1 - if state is not None: - metadata = state.get("metadata", None) - if metadata: - steps_taken = metadata.get("step_count", -1) - - self.result = TaskResult( - content=content, - error=error, - execution_time_seconds=duration.total_seconds(), - steps_taken=steps_taken, - ) - def get_name(self) -> str: return self.request.task_name or self.id @@ -239,8 +153,42 @@ class Task(BaseModel): status: TaskRunStatus, message: str | None = None, output: Any | None = None, - ): + ) -> None: self.status = status self.status_message = message - if self.on_status_changed: + if self.on_status_changed is not None: await self.on_status_changed(status, message, output) + + async def finalize( + self, + content: Any | None = None, + state: dict[str, Any] | None = None, + error: str | None = None, + cancelled: bool = False, + ) -> None: + if cancelled: + resolved_status: TaskRunStatus = "cancelled" + default_msg = f"Task cancelled: {error}" if error else "Task cancelled." + elif error: + resolved_status = "failed" + default_msg = f"Task failed: {error}" + else: + resolved_status = "completed" + default_msg = "Task completed successfully." + + await self.set_status(status=resolved_status, message=default_msg, output=content or error) + self.ended_at = datetime.now() + + elapsed = (self.ended_at - self.created_at).total_seconds() + steps = -1 + if state is not None: + metadata = state.get("metadata") + if isinstance(metadata, dict): + steps = metadata.get("step_count", -1) + + self.result = TaskResult( + content=content, + error=error, + execution_time_seconds=max(0.0, elapsed), + steps_taken=steps, + ) diff --git a/artemis/tools/committee_tool.py b/artemis/tools/committee_tool.py index 5db3b34..75a19c2 100644 --- a/artemis/tools/committee_tool.py +++ b/artemis/tools/committee_tool.py @@ -46,7 +46,6 @@ from artemis.tools.tool_wrapper import ( invoke_tool_with_injection, tool_result_messages, ) -from artemis.tools.types import CyFunctionDetector from artemis.tools.video_tool import get_video_analyzer_tool from artemis.utils.logger import get_logger from artemis.utils.notes import get_note_file_path, get_notes_dir @@ -61,8 +60,6 @@ logger = get_logger(__name__) class AskCommitteeArgs(BaseModel): """Arguments schema for invoking the committee tool.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} avatar_directive: str = Field( ..., description="Directive to frame and steer the debate for your avatar.", diff --git a/artemis/tools/diagnostic_tool.py b/artemis/tools/diagnostic_tool.py index 4262614..85d735b 100644 --- a/artemis/tools/diagnostic_tool.py +++ b/artemis/tools/diagnostic_tool.py @@ -27,7 +27,6 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -36,8 +35,6 @@ logger = get_logger(__name__) # pylint: disable=too-few-public-methods class AskDiagnoserArgs(BaseModel): """Arguments schema for invoking the diagnostic subagent.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} query: str = Field( ..., description=( diff --git a/artemis/tools/explorer_tool.py b/artemis/tools/explorer_tool.py index 951108b..294cd18 100644 --- a/artemis/tools/explorer_tool.py +++ b/artemis/tools/explorer_tool.py @@ -58,7 +58,6 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector from artemis.utils.element_hit_test import find_element_at_point from artemis.utils.logger import get_logger from artemis.utils.visualization import draw_dots @@ -94,8 +93,6 @@ CANDIDATE_DOT_COLOR = "magenta" class AskExplorerArgs(BaseModel): """Arguments of ``ask_explorer`` as seen by every calling agent.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} query: str = Field(..., description=ASK_EXPLORER_QUERY_DESCRIPTION) context_feedback: str = Field("", description=ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION) diff --git a/artemis/tools/image_processor_tool.py b/artemis/tools/image_processor_tool.py index 33b22c3..4e6e0a1 100644 --- a/artemis/tools/image_processor_tool.py +++ b/artemis/tools/image_processor_tool.py @@ -25,7 +25,6 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -34,8 +33,6 @@ logger = get_logger(__name__) # pylint: disable=too-few-public-methods class AskVisionCoderArgs(BaseModel): """Arguments schema for invoking the Image Processor subagent.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} instruction: str = Field( ..., description=( diff --git a/artemis/tools/log_tool.py b/artemis/tools/log_tool.py index dc61901..fd94495 100644 --- a/artemis/tools/log_tool.py +++ b/artemis/tools/log_tool.py @@ -27,7 +27,6 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -36,8 +35,6 @@ logger = get_logger(__name__) # pylint: disable=too-few-public-methods class AnalyzeLogsArgs(BaseModel): """Arguments schema for invoking the Log Analyzer subagent.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} specific_query: str = Field( ..., description=( diff --git a/artemis/tools/mobile/launch_app.py b/artemis/tools/mobile/launch_app.py index 5c0de07..3a5564e 100644 --- a/artemis/tools/mobile/launch_app.py +++ b/artemis/tools/mobile/launch_app.py @@ -18,7 +18,10 @@ from langchain_core.messages import ToolMessage from langchain_core.tools import BaseTool from pydantic import BaseModel, Field -from artemis.agents.hopper.hopper import HopperOutput, hopper +from artemis.agents.entity_extractor.entity_extractor import ( + ExtractionResult, + extract_entity, +) from artemis.context import ArtemisContext from artemis.controllers.platform_specific_commands_controller import ( list_packages_async, @@ -28,7 +31,6 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector from artemis.utils.app_launch_utils import launch_app_with_retries from artemis.utils.logger import get_logger @@ -37,8 +39,6 @@ logger = get_logger(__name__) class LaunchAppArgs(BaseModel): """Arguments schema for launching an application.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} app_name: str = Field( ..., description="The natural language name of the application to launch.", @@ -77,21 +77,21 @@ async def find_package(ctx: ArtemisContext, app_name: str, use_fallback: bool = package_cache[app_name] = app_name return app_name - hopper_output: HopperOutput = await hopper( + extractor_output: ExtractionResult = await extract_entity( ctx=ctx, request=(f"I'm looking for the package name of the following app: '{app_name}'"), data=all_packages, use_fallback=use_fallback, ) - if not hopper_output.found or not hopper_output.output: + if not extractor_output.found or not extractor_output.output: if isinstance(package_cache, dict): package_cache[app_name] = None return None - package_name = hopper_output.output.strip() + package_name = extractor_output.output.strip() if package_name not in package_set: logger.warning( - f"Hopper returned package '{package_name}' for '{app_name}', " + f"Entity extractor returned package '{package_name}' for '{app_name}', " "but it is NOT physically installed on the device!" ) if isinstance(package_cache, dict): diff --git a/artemis/tools/mobile/read_hierarchy.py b/artemis/tools/mobile/read_hierarchy.py index bd8e47c..b724f74 100644 --- a/artemis/tools/mobile/read_hierarchy.py +++ b/artemis/tools/mobile/read_hierarchy.py @@ -25,7 +25,6 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -33,10 +32,6 @@ logger = get_logger(__name__) class GetUiHierarchyArgs(BaseModel): """Arguments schema for getting UI hierarchy.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - - GET_UI_HIERARCHY_DOCSTRING = ( "[DIAGNOSTIC] Retrieves the current screen UI hierarchy XML using a persistent" " UI Automator client.\n\nUse this instead of running 'uiautomator dump'" diff --git a/artemis/tools/object_detection_tool.py b/artemis/tools/object_detection_tool.py index c922f90..9db1bc1 100644 --- a/artemis/tools/object_detection_tool.py +++ b/artemis/tools/object_detection_tool.py @@ -35,7 +35,6 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -44,8 +43,6 @@ logger = get_logger(__name__) # pylint: disable=too-few-public-methods class ObjectDetectionArgs(BaseModel): """Arguments schema for object detection with explicit screenshot path.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} image_path: str = Field(..., description="Absolute path to the screenshot file.") queries: list[str] = Field(..., description="A list of single query strings to find.") @@ -53,8 +50,6 @@ class ObjectDetectionArgs(BaseModel): # pylint: disable=too-few-public-methods class OperatorObjectDetectionArgs(BaseModel): """Arguments schema for operator object detection using latest screenshot from state.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} queries: list[str] = Field(..., description="A list of single query strings to find.") diff --git a/artemis/tools/scratchpad.py b/artemis/tools/scratchpad.py index 6032e43..b9c3e9e 100644 --- a/artemis/tools/scratchpad.py +++ b/artemis/tools/scratchpad.py @@ -25,7 +25,6 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger from artemis.utils.notes import ( APPEND_NOTE_ARG_CONTENT_DESC, @@ -57,24 +56,18 @@ logger = get_logger(__name__) class SaveNoteArgs(BaseModel): """Arguments schema for saving notes.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} key: str = Field(..., description=SAVE_NOTE_ARG_KEY_DESC) content: str = Field(..., description=SAVE_NOTE_ARG_CONTENT_DESC) class AppendNoteArgs(BaseModel): """Arguments schema for appending notes.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} key: str = Field(..., description=APPEND_NOTE_ARG_KEY_DESC) content: str = Field(..., description=APPEND_NOTE_ARG_CONTENT_DESC) class UpdateNoteArgs(BaseModel): """Arguments schema for updating notes.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} key: str = Field(..., description=UPDATE_NOTE_ARG_KEY_DESC) target: str = Field(..., description=UPDATE_NOTE_ARG_TARGET_DESC) replacement: str = Field(..., description=UPDATE_NOTE_ARG_REPLACEMENT_DESC) @@ -82,8 +75,6 @@ class UpdateNoteArgs(BaseModel): class ReadNoteArgs(BaseModel): """Arguments schema for reading notes.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} key: str = Field(..., description=READ_NOTE_ARG_KEY_DESC) start_line: int | None = Field(None, description="Start line to read (1-indexed, inclusive)") end_line: int | None = Field(None, description="End line to read (1-indexed, inclusive)") @@ -91,10 +82,6 @@ class ReadNoteArgs(BaseModel): class ListNotesArgs(BaseModel): """Arguments schema for listing notes.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} - - class SaveNoteTool(ArtemisTool): """Universal tool for saving notes to persistent memory.""" diff --git a/artemis/tools/tool_wrapper.py b/artemis/tools/tool_wrapper.py index 0d97841..4bb5498 100644 --- a/artemis/tools/tool_wrapper.py +++ b/artemis/tools/tool_wrapper.py @@ -31,13 +31,8 @@ from artemis.context import ArtemisContext from artemis.data_engine import engine as engine_mod from artemis.data_engine.trace import CURRENT_TRACE_ID, smart_serialize from artemis.graph.state import State -from artemis.tools.types import CyFunctionDetector - - class ToolWrapper(BaseModel): """Wrapper holding a tool factory and lifecycle callbacks.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} tool_fn_getter: Callable[[ArtemisContext], BaseTool] on_success_fn: Callable[..., str] on_failure_fn: Callable[..., str] diff --git a/artemis/tools/types.py b/artemis/tools/types.py index 0fa8a94..ecf63ff 100644 --- a/artemis/tools/types.py +++ b/artemis/tools/types.py @@ -12,61 +12,47 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pydantic import BaseModel, Field, model_validator +"""Multi-strategy target element descriptors for ARTEMIS actions. + +Encapsulates semantic identifiers, textual queries, and spatial bounding boxes +into a prioritized target resolution contract. +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, model_validator from artemis.utils.ui_hierarchy import ElementBounds -class _CyFunctionDetectorMeta(type): - """Metaclass for detecting cython compiled functions.""" - - def __instancecheck__(cls, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -# pylint: disable=too-few-public-methods -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - """Detector for cython compiled functions.""" - - class Target(BaseModel): - """A comprehensive locator for a UI element, supporting a fallback mechanism.""" + """Composite locator descriptor resolving UI nodes across dynamic and visual channels.""" - model_config = {"ignored_types": (CyFunctionDetector,)} + model_config = ConfigDict(extra="ignore") - resource_id: str | None = Field(None, description="The resource-id of the element.") + resource_id: str | None = Field(default=None, description="Android resource identifier token.") resource_id_index: int | None = Field( - None, - description=("The zero-based index if multiple elements share the same resource-id."), + default=None, + description="Zero-based ordinal index when multiple nodes match resource_id.", ) text: str | None = Field( - None, - description=("The text content of the element (e.g., a label or placeholder)."), + default=None, + description="Exact or case-insensitive element label or content descriptor.", ) text_index: int | None = Field( - None, - description=("The zero-based index if multiple elements share the same text."), + default=None, + description="Zero-based ordinal index when multiple nodes match text query.", ) bounds: ElementBounds | None = Field( - None, description="The x, y, width, and height of the element." + default=None, + description="Spatial boundary metrics (x, y, width, height) of the element.", ) @model_validator(mode="after") - def _default_indices(self): - # Treat empty strings like “not provided” - if ( - self.resource_id is not None and self.resource_id != "" - ) and self.resource_id_index is None: + def _normalize_indices(self) -> Target: + """Coerce missing indices to zero if primary selector string is populated.""" + if self.resource_id and self.resource_id.strip() and self.resource_id_index is None: self.resource_id_index = 0 - if (self.text is not None and self.text != "") and self.text_index is None: + if self.text and self.text.strip() and self.text_index is None: self.text_index = 0 return self diff --git a/artemis/tools/utils.py b/artemis/tools/utils.py index 47e4f68..7dd1dc3 100644 --- a/artemis/tools/utils.py +++ b/artemis/tools/utils.py @@ -11,25 +11,29 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. + +"""Spatial positioning, focus resolution, and interaction helpers for ARTEMIS tools. + +Coordinates locator resolution across hierarchy trees, validates bounded screen areas, +and routes focus and cursor placement to target elements. +""" from __future__ import annotations -from typing import Literal - +from typing import Any, Literal from pydantic import BaseModel, ConfigDict from artemis.context import ArtemisContext -from artemis.controllers.types import ( - CoordinatesSelectorRequest, - PercentagesSelectorRequest, -) +from artemis.controllers.types import CoordinatesSelectorRequest, PercentagesSelectorRequest from artemis.controllers.unified_controller import UnifiedMobileController from artemis.graph.state import State from artemis.tools.types import Target from artemis.utils.logger import get_logger from artemis.utils.ui_hierarchy import ( ElementBounds, - Point, find_element_by_resource_id, get_bounds_for_element, get_element_text, @@ -39,293 +43,54 @@ from artemis.utils.ui_hierarchy import ( logger = get_logger(__name__) -def find_element_by_text( - ui_hierarchy: list[dict], text: str, index: int | None = None -) -> dict | None: - """Find a UI element by its text content (adapted to both flat and rich hierarchy) - - This function performs a recursive, case-insensitive search. - - Args: - ui_hierarchy: List of UI element dictionaries. - text: The text content to search for. - index: Optional index to select nth matching element. - - Returns: - The complete UI element dictionary if found, None otherwise. - """ - matches: list[dict] = [] - - def search_recursive(elements: list[dict]) -> None: - for element in elements: - if isinstance(element, dict): - src = element.get("attributes", element) - element_text = src.get("text", "") - - # Guard against non-string values - if not isinstance(element_text, str): - element_text = "" - - text_lower = text.lower() - if element_text and text_lower == element_text.lower(): - matches.append(element) - - if children := element.get("children", []): - search_recursive(children) - - search_recursive(ui_hierarchy) - - if not matches: - return None - - if index is None: - idx = 0 - elif index < 0: - return None - else: - idx = index - if idx < len(matches): - return matches[idx] - return None - - -async def tap_bottom_right_of_element(bounds: ElementBounds, ctx: ArtemisContext): - """Tap the bottom-right corner of the specified element bounds.""" - bottom_right: Point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99) - await tap( - ctx=ctx, - selector_request=SelectorRequestWithCoordinates( - coordinates=CoordinatesSelectorRequest( - x=bottom_right.x, - y=bottom_right.y, - ), - ), - ) - - -# pylint: disable=too-many-return-statements -async def move_cursor_to_end_if_bounds( - ctx: ArtemisContext, - state: State, - target: Target, - elt: dict | None = None, -) -> dict | None: - """Best-effort move of the text cursor near the end of the input by tapping the - - bottom-right area of the focused element (if bounds are available). - """ - if target.resource_id: - if not elt: - elt = find_element_by_resource_id( - ui_hierarchy=state.latest_ui_hierarchy or [], - resource_id=target.resource_id, - index=target.resource_id_index, - ) - if not elt: - return None - - bounds = get_bounds_for_element(elt) - if not bounds: - return elt - - logger.debug("Tapping near the end of the input to move the cursor") - await tap_bottom_right_of_element(bounds=bounds, ctx=ctx) - logger.debug(f"Tapped end of input {target.resource_id}") - return elt - - if target.bounds: - await tap_bottom_right_of_element(target.bounds, ctx=ctx) - logger.debug("Tapped end of input by coordinates") - return elt - - if target.text: - text_elt = find_element_by_text( - state.latest_ui_hierarchy or [], - target.text, - index=target.text_index, - ) - if text_elt: - bounds = get_bounds_for_element(text_elt) - if bounds: - await tap_bottom_right_of_element(bounds=bounds, ctx=ctx) - logger.debug(f"Tapped end of input that had text'{target.text}'") - return text_elt - return None - - return None - - -async def focus_element_if_needed( - ctx: ArtemisContext, target: Target -) -> Literal["resource_id", "coordinates", "text", "already_focused"] | None: - """Ensures the element is focused, with a sanity check to prevent trusting misleading IDs. - - If no target locator is provided (empty target), assumes the field is - already focused - and returns "already_focused" to allow typing without re-focusing. - """ - # If no locator is provided, assume already focused (e.g., keyboard is visible) - if not target.resource_id and not target.bounds and not target.text: - logger.debug("No target locator provided, assuming element is already focused") - return "already_focused" - - controller = UnifiedMobileController(ctx) - rich_hierarchy = await controller.get_ui_elements() - elt_from_id = None - if target.resource_id: - elt_from_id = find_element_by_resource_id( - ui_hierarchy=rich_hierarchy, - resource_id=target.resource_id, - index=target.resource_id_index, - is_rich_hierarchy=False, - ) - - if elt_from_id and target.text: - text_from_id_elt = get_element_text(elt_from_id) - if not text_from_id_elt or target.text.lower() != text_from_id_elt.lower(): - logger.warning( - f"ID '{target.resource_id}' and text '{target.text}' seem to be" - " on different elements. Ignoring the resource_id and falling" - " back to other locators." - ) - elt_from_id = None - - if elt_from_id: - if not is_element_focused(elt_from_id): - await tap( - ctx=ctx, - selector_request=IdSelectorRequest(id=target.resource_id), # type: ignore - index=target.resource_id_index, - ) - logger.debug(f"Focused (tap) on resource_id={target.resource_id}") - rich_hierarchy = await controller.get_ui_elements() - elt_from_id = find_element_by_resource_id( - ui_hierarchy=rich_hierarchy, - resource_id=target.resource_id, # type: ignore - index=target.resource_id_index, - is_rich_hierarchy=False, - ) - if elt_from_id and is_element_focused(elt_from_id): - logger.debug(f"Text input is focused: {target.resource_id}") - return "resource_id" - logger.warning(f"Failed to focus using resource_id='{target.resource_id}'. Fallback...") - - if target.bounds: - relative_point = target.bounds.get_center() - await tap( - ctx=ctx, - selector_request=SelectorRequestWithCoordinates( - coordinates=CoordinatesSelectorRequest(x=relative_point.x, y=relative_point.y) - ), - ) - logger.debug(f"Tapped on coordinates ({relative_point.x}, {relative_point.y}) to focus.") - return "coordinates" - - if target.text: - text_elt = find_element_by_text(rich_hierarchy, target.text, index=target.text_index) - if text_elt: - bounds = get_bounds_for_element(text_elt) - if bounds: - relative_point = bounds.get_center() - await tap( - ctx=ctx, - selector_request=SelectorRequestWithCoordinates( - coordinates=CoordinatesSelectorRequest( - x=relative_point.x, y=relative_point.y - ) - ), - ) - logger.debug(f"Tapped on text element '{target.text}' to focus.") - return "text" - - logger.error( - "Failed to focus element. No valid locator (resource_id, coordinates, or text) succeeded." - ) - return None - - -def validate_coordinates_bounds( - target: Target, screen_width: int, screen_height: int -) -> str | None: - """Validate that coordinates are within screen bounds. - - Returns error message if invalid, None if valid. - """ - if not target.bounds: - return None - - center = target.bounds.get_center() - errors = [] - - if center.x < 0 or center.x >= screen_width: - errors.append(f"x={center.x} is outside screen width (0-{screen_width})") - if center.y < 0 or center.y >= screen_height: - errors.append(f"y={center.y} is outside screen height (0-{screen_height})") - - return "; ".join(errors) if errors else None - - -def has_valid_selectors(target: Target) -> bool: - """Check if target has at least one valid selector.""" - has_coordinates = target.bounds is not None - has_resource_id = target.resource_id is not None and target.resource_id != "" - has_text = target.text is not None and target.text != "" - return has_coordinates or has_resource_id or has_text - - class IdSelectorRequest(BaseModel): - """Selector request using resource id.""" + """Target selector driven by Android resource ID.""" model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) id: str def to_dict(self) -> dict[str, str | int]: - """Convert selector to dictionary.""" return {"id": self.id} class TextSelectorRequest(BaseModel): - """Selector request using element text.""" + """Target selector driven by exact or case-insensitive element label.""" model_config = ConfigDict(extra="forbid") text: str def to_dict(self) -> dict[str, str | int]: - """Convert selector to dictionary.""" return {"text": self.text} class SelectorRequestWithCoordinates(BaseModel): - """Selector request using absolute coordinates.""" + """Target selector pointing to an absolute pixel coordinate.""" model_config = ConfigDict(extra="forbid") coordinates: CoordinatesSelectorRequest def to_dict(self) -> dict[str, str | int]: - """Convert selector to dictionary.""" return {"point": self.coordinates.to_str()} class SelectorRequestWithPercentages(BaseModel): - """Selector request using percentage coordinates.""" + """Target selector pointing to normalized fractional [0.0, 1.0] coordinates.""" model_config = ConfigDict(extra="forbid") percentages: PercentagesSelectorRequest def to_dict(self) -> dict[str, str | int]: - """Convert selector to dictionary.""" return {"point": self.percentages.to_str()} class IdWithTextSelectorRequest(BaseModel): - """Selector request using both resource id and text.""" + """Composite selector pairing resource ID with expected text.""" model_config = ConfigDict(extra="forbid") id: str text: str def to_dict(self) -> dict[str, str | int]: - """Convert selector to dictionary.""" return {"id": self.id, "text": self.text} @@ -341,37 +106,30 @@ SelectorRequest = ( def _extract_resource_id_and_text_from_selector( selector: SelectorRequest, ) -> tuple[str | None, str | None]: - """Extract resource_id and text from a selector.""" - resource_id = None - text = None - + """Unpack identifier and textual query from any supported SelectorRequest.""" if isinstance(selector, IdSelectorRequest): - resource_id = selector.id - elif isinstance(selector, TextSelectorRequest): - text = selector.text - elif isinstance(selector, IdWithTextSelectorRequest): - resource_id = selector.id - text = selector.text - - return resource_id, text + return selector.id, None + if isinstance(selector, TextSelectorRequest): + return None, selector.text + if isinstance(selector, IdWithTextSelectorRequest): + return selector.id, selector.text + return None, None async def tap( ctx: ArtemisContext, selector_request: SelectorRequest, index: int | None = None, -): - """Tap on a selector. - - Index is optional and is used when you have multiple views matching the same - selector. ui_hierarchy is optional and used for ADB taps to find elements. - """ +) -> Any: + """Dispatch an on-screen tap to an element or coordinate selector.""" controller = UnifiedMobileController(ctx) if isinstance(selector_request, SelectorRequestWithCoordinates): - result = await controller.tap_at( - x=selector_request.coordinates.x, y=selector_request.coordinates.y + res = await controller.tap_at( + x=selector_request.coordinates.x, + y=selector_request.coordinates.y, ) - return result.error if result.error else None + return res.error if res.error else None + if isinstance(selector_request, SelectorRequestWithPercentages): coords = selector_request.percentages.to_coords( width=ctx.device.device_width, @@ -379,11 +137,183 @@ async def tap( ) return await controller.tap_at(coords.x, coords.y) - # For other selectors, we need the UI hierarchy - resource_id, text = _extract_resource_id_and_text_from_selector(selector_request) - + res_id, txt = _extract_resource_id_and_text_from_selector(selector_request) return await controller.tap_element( - resource_id=resource_id, - text=text, + resource_id=res_id, + text=txt, index=index if index is not None else 0, ) + + +def find_element_by_text( + ui_hierarchy: list[dict[str, Any]], text: str, index: int | None = None +) -> dict[str, Any] | None: + """Search for elements matching label text across nested hierarchy tree nodes.""" + target_text = (text or "").strip().lower() + if not target_text: + return None + + matched_nodes: list[dict[str, Any]] = [] + + def _traverse(nodes: list[dict[str, Any]]) -> None: + for node in nodes: + if not isinstance(node, dict): + continue + attrs = node.get("attributes", node) + val = attrs.get("text") + if isinstance(val, str) and val.strip().lower() == target_text: + matched_nodes.append(attrs if "attributes" in node else node) + + children = node.get("children") + if isinstance(children, list) and children: + _traverse(children) + + _traverse(ui_hierarchy) + if not matched_nodes: + return None + + target_idx = max(0, index or 0) + if target_idx < len(matched_nodes): + return matched_nodes[target_idx] + return None + + +async def tap_bottom_right_of_element(bounds: ElementBounds, ctx: ArtemisContext) -> None: + """Position pointer near the trailing boundary of an input element.""" + corner_point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99) + await tap( + ctx=ctx, + selector_request=SelectorRequestWithCoordinates( + coordinates=CoordinatesSelectorRequest(x=corner_point.x, y=corner_point.y) + ), + ) + + +async def move_cursor_to_end_if_bounds( + ctx: ArtemisContext, + state: State, + target: Target, + elt: dict[str, Any] | None = None, +) -> dict[str, Any] | None: + """Position insertion point at the end of an input element boundary.""" + current_tree = state.latest_ui_hierarchy or [] + + if target.resource_id: + element_node = elt or find_element_by_resource_id( + ui_hierarchy=current_tree, + resource_id=target.resource_id, + index=target.resource_id_index, + ) + if not element_node: + return None + box = get_bounds_for_element(element_node) + if not box: + return element_node + await tap_bottom_right_of_element(bounds=box, ctx=ctx) + return element_node + + if target.bounds: + await tap_bottom_right_of_element(bounds=target.bounds, ctx=ctx) + return elt + + if target.text: + matched = find_element_by_text(current_tree, target.text, index=target.text_index) + if matched: + box = get_bounds_for_element(matched) + if box: + await tap_bottom_right_of_element(bounds=box, ctx=ctx) + return matched + return None + + return None + + +async def focus_element_if_needed( + ctx: ArtemisContext, target: Target +) -> Literal["resource_id", "coordinates", "text", "already_focused"] | None: + """Ensure element possesses active input focus, falling back through available selectors.""" + if not target.resource_id and not target.bounds and not target.text: + return "already_focused" + + controller = UnifiedMobileController(ctx) + hierarchy = await controller.get_ui_elements() + + element_by_id = None + if target.resource_id: + element_by_id = find_element_by_resource_id( + ui_hierarchy=hierarchy, + resource_id=target.resource_id, + index=target.resource_id_index, + is_rich_hierarchy=False, + ) + + if element_by_id and target.text: + observed_label = get_element_text(element_by_id) + if not observed_label or target.text.strip().lower() != observed_label.strip().lower(): + logger.warning( + f"ID '{target.resource_id}' and text '{target.text}' do not align to the same node." + " Falling back to secondary selectors." + ) + element_by_id = None + + if element_by_id: + if not is_element_focused(element_by_id): + await tap( + ctx=ctx, + selector_request=IdSelectorRequest(id=target.resource_id), # type: ignore[arg-type] + index=target.resource_id_index, + ) + hierarchy = await controller.get_ui_elements() + element_by_id = find_element_by_resource_id( + ui_hierarchy=hierarchy, + resource_id=target.resource_id, # type: ignore[arg-type] + index=target.resource_id_index, + is_rich_hierarchy=False, + ) + if element_by_id and is_element_focused(element_by_id): + return "resource_id" + + if target.bounds: + centroid = target.bounds.get_center() + await tap( + ctx=ctx, + selector_request=SelectorRequestWithCoordinates( + coordinates=CoordinatesSelectorRequest(x=centroid.x, y=centroid.y) + ), + ) + return "coordinates" + + if target.text: + matched_text_node = find_element_by_text(hierarchy, target.text, index=target.text_index) + if matched_text_node: + text_box = get_bounds_for_element(matched_text_node) + if text_box: + centroid = text_box.get_center() + await tap( + ctx=ctx, + selector_request=SelectorRequestWithCoordinates( + coordinates=CoordinatesSelectorRequest(x=centroid.x, y=centroid.y) + ), + ) + return "text" + + logger.error("Failed to focus element. No valid locator (resource_id, coordinates, or text) succeeded.") + return None + + +def validate_coordinates_bounds(target: Target, screen_width: int, screen_height: int) -> str | None: + """Verify target centroid resides within active viewport boundaries.""" + if not target.bounds: + return None + c = target.bounds.get_center() + errors: list[str] = [] + if not (0 <= c.x < screen_width): + errors.append(f"x={c.x} is outside screen width (0-{screen_width})") + if not (0 <= c.y < screen_height): + errors.append(f"y={c.y} is outside screen height (0-{screen_height})") + return "; ".join(errors) if errors else None + + +def has_valid_selectors(target: Target) -> bool: + """Verify target defines at least one valid locator criterion.""" + return bool(target.bounds is not None or target.resource_id or target.text) diff --git a/artemis/tools/video_tool.py b/artemis/tools/video_tool.py index da4209f..9120a0f 100644 --- a/artemis/tools/video_tool.py +++ b/artemis/tools/video_tool.py @@ -34,7 +34,6 @@ from artemis.drivers.base import BaseDeviceDriver from artemis.graph.state import State from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -42,8 +41,6 @@ logger = get_logger(__name__) class VideoAnalyzerArgs(BaseModel): """Arguments schema for video analyzer tool.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} time_description: str = Field( ..., description=( diff --git a/artemis/tools/wait_tool.py b/artemis/tools/wait_tool.py index 96371d6..be33105 100644 --- a/artemis/tools/wait_tool.py +++ b/artemis/tools/wait_tool.py @@ -23,7 +23,6 @@ from artemis.data_engine.trace import trace_langchain_tool from artemis.drivers.base import BaseDeviceDriver from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.tool_wrapper import ToolWrapper -from artemis.tools.types import CyFunctionDetector from artemis.utils.logger import get_logger logger = get_logger(__name__) @@ -31,8 +30,6 @@ logger = get_logger(__name__) class WaitArgs(BaseModel): """Arguments schema for wait tool.""" - - model_config = {"ignored_types": (CyFunctionDetector,)} seconds: int = Field( ..., description=("The duration to wait/sleep in seconds. Minimum is 1, maximum is 60."), diff --git a/artemis/utils/app_launch_utils.py b/artemis/utils/app_launch_utils.py index a7b3468..770efbe 100644 --- a/artemis/utils/app_launch_utils.py +++ b/artemis/utils/app_launch_utils.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. """Utilities for handling app locking and initial app launch logic.""" @@ -216,7 +219,7 @@ async def _poll_for_app_ready( Only fails if we get a different (non-null) package or timeout. Args: - ctx: Mobile use context + ctx: ArtemisContext app_package: Expected package name max_poll_seconds: Maximum time to poll (default: 15s) poll_interval: Time between polls (default: 1s) @@ -272,7 +275,7 @@ async def launch_app_with_retries( """Launch an app with retry logic and smart polling. Args: - ctx: Mobile use context + ctx: ArtemisContext app_package: Package name (Android) to launch max_retries: Maximum number of launch attempts (default: 3) max_poll_seconds: Maximum time to wait for app to load per attempt @@ -345,7 +348,7 @@ async def _handle_initial_app_launch( 3. Return status with success/error information Args: - ctx: Mobile use context + ctx: ArtemisContext locked_app_package: Package name (Android) to lock to Returns: diff --git a/artemis/utils/cli_helpers.py b/artemis/utils/cli_helpers.py deleted file mode 100644 index f7056b2..0000000 --- a/artemis/utils/cli_helpers.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -from adbutils import AdbClient -from rich.console import Console - - -def display_device_status(console: Console, adb_client: AdbClient | None = None): - """Checks for connected devices and displays the status.""" - console.print("\n[bold]📱 Device Status[/bold]") - devices = None - if adb_client is not None: - devices = adb_client.device_list() - if devices: - console.print("✅ [bold green]Android device(s) connected:[/bold green]") - for device in devices: - console.print(f" - {device.serial}") - else: - console.print("❌ [bold red]No Android device found.[/bold red]") - command = "emulator -avd " - if sys.platform not in ["win32", "darwin"]: - command = f"./{command}" - console.print( - f"You can start an emulator using a command like: [bold]'{command}'[/bold]" - ) diff --git a/artemis/utils/cli_selection.py b/artemis/utils/cli_selection.py deleted file mode 100644 index 8b288c5..0000000 --- a/artemis/utils/cli_selection.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -import inquirer -from rich.console import Console -from rich.prompt import Prompt -from rich.text import Text - - -def select_provider_and_model( - console: Console, - available_providers: list[str], - available_models: dict, - default_provider: str, - default_model: str, - provider: str | None = None, - model: str | None = None, -) -> tuple[str, str]: - """Interactive selection of LLM provider and model with arrow-key dropdowns when available. - - Args: - console: Rich console for output - available_providers: List of available provider names - available_models: Dict mapping providers to their available models - default_provider: Default provider to use - default_model: Default model to use - provider: Pre-selected provider (optional) - model: Pre-selected model (optional) - - Returns: - Tuple of (selected_provider, selected_model) - """ - final_provider = provider - final_model = model - - # Interactive provider selection - if not final_provider: - console.print("\n🤖 [bold cyan]LLM Provider Selection[/bold cyan]") - final_provider = _select_from_list( - console=console, - item_type="provider", - choices=available_providers, - default=default_provider, - message="Select LLM provider", - ) - - # Interactive model selection - if not final_model: - console.print(f"\n🎯 [bold green]Model Selection for {final_provider}[/bold green]") - available_model_list = ( - available_models[final_provider] - if final_provider - else available_models[default_provider] - ) - - default_model_for_provider = ( - default_model if default_model in available_model_list else available_model_list[0] - ) - - final_model = _select_from_list( - console=console, - item_type="model", - choices=available_model_list, - default=default_model_for_provider, - message=f"Select model for {final_provider}", - ) - - return final_provider, final_model - - -def _select_from_list( - console: Console, - item_type: str, - choices: list[str], - default: str, - message: str, -) -> str: - """Select an item from a list using arrow keys when available, fallback to numbered selection. - - Args: - console: Rich console for output - item_type: Type of item being selected (for error messages) - choices: List of choices to select from - default: Default choice - message: Message to display in dropdown - - Returns: - Selected choice - """ - # Try arrow-key dropdown if TTY is available, fallback to numbered selection - if sys.stdin.isatty(): - try: - questions = [ - inquirer.List( - "selection", - message=f"{message} (use arrow keys)", - choices=choices, - default=default, - ), - ] - answers = inquirer.prompt(questions) - return answers["selection"] if answers else default - except Exception: - # Fallback to numbered selection - return _numbered_selection(console, item_type, choices, default) - else: - return _numbered_selection(console, item_type, choices, default) - - -def _numbered_selection(console: Console, item_type: str, choices: list[str], default: str) -> str: - """Fallback numbered selection when arrow keys aren't available.""" - choices_text = "\n".join([f" {i + 1}. {choice}" for i, choice in enumerate(choices)]) - console.print(f"Available {item_type}s:\n{choices_text}") - - default_idx = choices.index(default) + 1 if default in choices else 1 - - while True: - choice = Prompt.ask( - f"Select {item_type} (1-{len(choices)}) or press Enter for default", - default=str(default_idx), - ) - try: - choice_idx = int(choice) - 1 - if 0 <= choice_idx < len(choices): - return choices[choice_idx] - else: - console.print("[red]Invalid choice. Please try again.[/red]") - except ValueError: - console.print("[red]Please enter a number.[/red]") - - -def display_llm_config(console: Console, provider: str, model: str) -> None: - """Display the selected LLM configuration with colors.""" - config_text = Text() - config_text.append("🤖 LLM Configuration: ", style="bold white") - config_text.append("Provider: ", style="white") - config_text.append(f"{provider}", style="bold cyan") - config_text.append(" | Model: ", style="white") - config_text.append(f"{model}", style="bold green") - - console.print(config_text) diff --git a/artemis/utils/conversations.py b/artemis/utils/conversations.py deleted file mode 100644 index c7a2058..0000000 --- a/artemis/utils/conversations.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import TypeGuard - -from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage - - -def is_ai_message(message: BaseMessage) -> TypeGuard[AIMessage]: - return isinstance(message, AIMessage) - - -def is_human_message(message: BaseMessage) -> TypeGuard[HumanMessage]: - return isinstance(message, HumanMessage) - - -def is_tool_message(message: BaseMessage) -> TypeGuard[ToolMessage]: - return isinstance(message, ToolMessage) - - -def is_tool_for_name(tool_message: ToolMessage, name: str) -> bool: - return tool_message.name == name - - -def get_screenshot_message_for_llm(screenshot_base64: str): - prefix = "" if screenshot_base64.startswith("data:image") else "data:image/jpeg;base64," - return HumanMessage( - content=[ - { - "type": "image_url", - "image_url": {"url": f"{prefix}{screenshot_base64}"}, - } - ] - ) diff --git a/artemis/utils/decorators.py b/artemis/utils/decorators.py index 4ef07b9..e50eb88 100644 --- a/artemis/utils/decorators.py +++ b/artemis/utils/decorators.py @@ -12,128 +12,89 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio -from collections.abc import Awaitable, Callable +"""Lifecycle callback and execution telemetry decorators for ARTEMIS. + +Provides function interception hooks for pre-execution logging, success verification, +and error capture across synchronous and asynchronous agent invocations. +""" + +from __future__ import annotations + from functools import wraps import inspect -from typing import Any, TypeVar, cast, overload - -R = TypeVar("R") - - -def wrap_with_callbacks_sync( - fn: Callable[..., R], - *, - before: Callable[..., None] | None = None, - on_success: Callable[[R], None] | None = None, - on_failure: Callable[[Exception], None] | None = None, - suppress_exceptions: bool = False, -) -> Callable[..., R]: - @wraps(fn) - def wrapper(*args: Any, **kwargs: Any) -> R: - if before: - before() - try: - result = fn(*args, **kwargs) - if on_success: - on_success(result) - return result - except Exception as e: - if on_failure: - on_failure(e) - if suppress_exceptions: - return None # type: ignore - raise - - return wrapper - - -def wrap_with_callbacks_async( - fn: Callable[..., Awaitable[R]], - *, - before: Callable[..., None] | None = None, - on_success: Callable[[R], None] | None = None, - on_failure: Callable[[Exception], None] | None = None, - suppress_exceptions: bool = False, -) -> Callable[..., Awaitable[R]]: - @wraps(fn) - async def wrapper(*args: Any, **kwargs: Any) -> R: - if before: - before() - try: - result = await fn(*args, **kwargs) - if on_success: - on_success(result) - return result - except Exception as e: - if on_failure: - on_failure(e) - if suppress_exceptions: - return None # type: ignore - raise - - return wrapper - - -@overload -def wrap_with_callbacks( - fn: Callable[..., Awaitable[R]], - *, - before: Callable[[], None] | None = ..., - on_success: Callable[[R], None] | None = ..., - on_failure: Callable[[Exception], None] | None = ..., - suppress_exceptions: bool = ..., -) -> Callable[..., Awaitable[R]]: ... - - -@overload -def wrap_with_callbacks( - *, - before: Callable[..., None] | None = ..., - on_success: Callable[[Any], None] | None = ..., - on_failure: Callable[[Exception], None] | None = ..., - suppress_exceptions: bool = ..., -) -> Callable[[Callable[..., R]], Callable[..., R]]: ... - - -@overload -def wrap_with_callbacks( - fn: Callable[..., R], - *, - before: Callable[[], None] | None = ..., - on_success: Callable[[R], None] | None = ..., - on_failure: Callable[[Exception], None] | None = ..., - suppress_exceptions: bool = ..., -) -> Callable[..., R]: ... +from typing import Any, Callable def wrap_with_callbacks( fn: Callable[..., Any] | None = None, *, - before: Callable[[], None] | None = None, + before: Callable[..., None] | None = None, on_success: Callable[[Any], None] | None = None, on_failure: Callable[[Exception], None] | None = None, suppress_exceptions: bool = False, ) -> Any: - def decorator(func: Callable[..., Any]) -> Any: - if inspect.iscoroutinefunction(func): - return wrap_with_callbacks_async( - cast(Callable[..., Awaitable[Any]], func), - before=before, - on_success=on_success, - on_failure=on_failure, - suppress_exceptions=suppress_exceptions, - ) - else: - return wrap_with_callbacks_sync( - cast(Callable[..., Any], func), - before=before, - on_success=on_success, - on_failure=on_failure, - suppress_exceptions=suppress_exceptions, - ) + """Wrap a callable with pre-invocation, success, and error callback handlers. - if fn is None: - return decorator - else: - return decorator(fn) + Supports decorating both coroutines and regular synchronous functions. + """ + + def _decorator(target_fn: Callable[..., Any]) -> Callable[..., Any]: + if inspect.iscoroutinefunction(target_fn): + + @wraps(target_fn) + async def _async_wrapped(*args: Any, **kwargs: Any) -> Any: + if before is not None: + try: + before() + except Exception: + pass + try: + result = await target_fn(*args, **kwargs) + if on_success is not None: + try: + on_success(result) + except Exception: + pass + return result + except Exception as exc: + if on_failure is not None: + try: + on_failure(exc) + except Exception: + pass + if suppress_exceptions: + return None + raise + + return _async_wrapped + + @wraps(target_fn) + def _sync_wrapped(*args: Any, **kwargs: Any) -> Any: + if before is not None: + try: + before() + except Exception: + pass + try: + result = target_fn(*args, **kwargs) + if on_success is not None: + try: + on_success(result) + except Exception: + pass + return result + except Exception as exc: + if on_failure is not None: + try: + on_failure(exc) + except Exception: + pass + if suppress_exceptions: + return None + raise + + return _sync_wrapped + + if fn is not None: + return _decorator(fn) + return _decorator diff --git a/artemis/utils/file.py b/artemis/utils/file.py index 7b8d97d..365b1f7 100644 --- a/artemis/utils/file.py +++ b/artemis/utils/file.py @@ -12,16 +12,43 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""File and configuration loading utilities for ARTEMIS.""" + +from __future__ import annotations + import json +from pathlib import Path import re -from typing import IO +from typing import IO, Any def strip_json_comments(text: str) -> str: - text = re.sub(r"//.*?$", "", text, flags=re.MULTILINE) - text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL) - return text + """Removes single-line (//) and multi-line (/* ... */) comments from JSONC text. + + Args: + text: Raw JSON with comments string. + + Returns: + Clean JSON string with comments stripped. + """ + pattern = r"//.*?$|/\*.*?\*/" + return re.sub(pattern, "", text, flags=re.MULTILINE | re.DOTALL) -def load_jsonc(file: IO) -> dict: - return json.loads(strip_json_comments(file.read())) +def load_jsonc(source: str | Path | IO[str] | IO[bytes]) -> dict[str, Any]: + """Loads and parses a JSONC (JSON with Comments) document. + + Args: + source: File-like object, Path, or string path to parse. + + Returns: + Parsed dictionary. + """ + if isinstance(source, (str, Path)): + raw_content = Path(source).read_text(encoding="utf-8") + else: + content = source.read() + raw_content = content.decode("utf-8") if isinstance(content, bytes) else content + + cleaned = strip_json_comments(raw_content) + return json.loads(cleaned) diff --git a/artemis/utils/logger.py b/artemis/utils/logger.py index 4d863f6..321d278 100644 --- a/artemis/utils/logger.py +++ b/artemis/utils/logger.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. from enum import Enum import logging diff --git a/artemis/utils/media.py b/artemis/utils/media.py index 4b5680a..bf91f58 100644 --- a/artemis/utils/media.py +++ b/artemis/utils/media.py @@ -11,23 +11,33 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. + +"""Media synthesis and trace asset management for ARTEMIS. + +Handles chronological trace GIF generation, frame quantization, and step snapshot +compilation while preserving active session recording manifests. +""" + +from __future__ import annotations import json import os from pathlib import Path +import shutil import subprocess import tempfile +from typing import Any + +from PIL import Image from artemis.utils.logger import get_logger -from PIL import Image logger = get_logger(__name__) -USE_FFMPEG_GIF = os.environ.get("USE_FFMPEG_GIF", "").lower() in ( - "1", - "true", - "yes", -) +_PREFER_FFMPEG = os.environ.get("USE_FFMPEG_GIF", "").strip().lower() in ("1", "true", "yes") def quantize_and_save_gif_from_paths( @@ -36,81 +46,68 @@ def quantize_and_save_gif_from_paths( colors: int = 128, duration: int = 100, ) -> None: - """Create an optimized GIF from image file paths. - - By default uses PIL (loads all frames into memory). - Set USE_FFMPEG_GIF=1 env var to use ffmpeg for memory-efficient streaming. - - Args: - image_paths: List of paths to image files (must be sorted in desired - order) - output_path: Path where the GIF will be saved - colors: Number of colors to use in quantization (lower = smaller file) - duration: Duration of each frame in milliseconds - - Raises: - ValueError: If image_paths list is empty - RuntimeError: If ffmpeg fails (when USE_FFMPEG_GIF is enabled) - """ + """Compile ordered image snapshots into a quantized animation asset.""" if not image_paths: - raise ValueError("image_paths must not be empty") + raise ValueError("Cannot assemble animation from an empty image sequence.") - if USE_FFMPEG_GIF: - _save_gif_ffmpeg(image_paths, output_path, colors, duration) - else: - _save_gif_pillow(image_paths, output_path, colors, duration) + if _PREFER_FFMPEG and shutil.which("ffmpeg"): + try: + _render_gif_with_ffmpeg(image_paths, output_path, colors=colors, frame_delay_ms=duration) + return + except Exception as exc: + logger.warning(f"FFmpeg GIF rendering encountered an error ({exc}); falling back to Pillow.") + + _render_gif_with_pillow(image_paths, output_path, colors=colors, frame_delay_ms=duration) -def _save_gif_pillow( - image_paths: list[Path], +def _render_gif_with_pillow( + paths: list[Path], output_path: Path, colors: int, - duration: int, + frame_delay_ms: int, ) -> None: - """Create GIF using PIL (loads all frames into memory).""" + """Internal Pillow quantization loop.""" + frames: list[Image.Image] = [] + for p in paths: + try: + with Image.open(p) as img: + rgb_img = img.convert("RGB") if img.mode != "RGB" else img.copy() + frames.append(rgb_img.quantize(colors=colors, method=Image.Quantize.MEDIANCUT)) + except Exception as exc: + logger.debug(f"Skipping damaged frame {p}: {exc}") - def frame_generator(): - for path in image_paths: - with Image.open(path) as img: - if img.mode != "RGB": - img = img.convert("RGB") - yield img.quantize(colors=colors, method=2) + if not frames: + return - frames = frame_generator() - first_frame = next(frames) + first_frame = frames[0] first_frame.save( output_path, save_all=True, - append_images=frames, + append_images=frames[1:], loop=0, optimize=True, - duration=duration, + duration=frame_delay_ms, ) -def _save_gif_ffmpeg( - image_paths: list[Path], +def _render_gif_with_ffmpeg( + paths: list[Path], output_path: Path, colors: int, - duration: int, + frame_delay_ms: int, ) -> None: - """Create GIF using ffmpeg (memory-efficient streaming).""" - fps = 1000 / duration + """Streamlined two-pass palette-generated GIF rendering via FFmpeg.""" + fps = max(1.0, 1000.0 / max(1, frame_delay_ms)) + with tempfile.TemporaryDirectory() as tmpdir: + list_file = Path(tmpdir) / "frames.txt" + palette = Path(tmpdir) / "palette.png" - with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: - for i, path in enumerate(image_paths): - f.write(f"file '{path.absolute()}'\n") - # Last file needs no duration (ffmpeg concat demuxer uses it as the final frame) - if i < len(image_paths) - 1: - f.write(f"duration {duration / 1000}\n") - # Repeat last file to ensure it's included (ffmpeg concat demuxer quirk) - f.write(f"file '{image_paths[-1].absolute()}'\n") - concat_file = Path(f.name) + lines = [f"file '{p.resolve()}'\nduration {frame_delay_ms / 1000.0}" for p in paths] + if paths: + lines.append(f"file '{paths[-1].resolve()}'") + list_file.write_text("\n".join(lines), encoding="utf-8") - palette_path = output_path.with_suffix(".palette.png") - - try: - result = subprocess.run( + subprocess.run( [ "ffmpeg", "-y", @@ -119,18 +116,16 @@ def _save_gif_ffmpeg( "-safe", "0", "-i", - str(concat_file), + str(list_file), "-vf", - f"palettegen=max_colors={min(colors, 256)}:stats_mode=diff", - str(palette_path), + f"palettegen=max_colors={min(colors, 256)}", + str(palette), ], capture_output=True, - text=True, + check=True, ) - if result.returncode != 0: - raise RuntimeError(f"ffmpeg palette generation failed: {result.stderr}") - result = subprocess.run( + subprocess.run( [ "ffmpeg", "-y", @@ -139,71 +134,78 @@ def _save_gif_ffmpeg( "-safe", "0", "-i", - str(concat_file), + str(list_file), "-i", - str(palette_path), + str(palette), "-lavfi", - f"fps={fps},paletteuse=dither=bayer:bayer_scale=5", - "-loop", - "0", + f"fps={fps},paletteuse", str(output_path), ], capture_output=True, - text=True, + check=True, ) - if result.returncode != 0: - raise RuntimeError(f"ffmpeg GIF creation failed: {result.stderr}") - - finally: - concat_file.unlink(missing_ok=True) - if palette_path.exists(): - palette_path.unlink() -def create_gif_from_trace_folder(trace_folder_path: Path): - image_files: list[Path] = [] - - for file in trace_folder_path.iterdir(): - if file.suffix == ".jpeg": - image_files.append(file) - - image_files.sort(key=lambda f: int(f.stem)) - - logger.info(f"Found {len(image_files)} images to compile") - - if not image_files: +def create_gif_from_trace_folder(trace_folder_path: Path) -> None: + """Scan and synthesize sequential JPEG frames in a trace directory to trace.gif.""" + if not trace_folder_path.is_dir(): return - gif_path = trace_folder_path / "trace.gif" - quantize_and_save_gif_from_paths(image_files, gif_path) - logger.info(f"GIF created at {gif_path}") + frames = sorted( + [f for f in trace_folder_path.iterdir() if f.suffix.lower() in (".jpeg", ".jpg") and f.stem.isdigit()], + key=lambda x: int(x.stem), + ) + if not frames: + return + + target_gif = trace_folder_path / "trace.gif" + quantize_and_save_gif_from_paths(frames, target_gif) + logger.info(f"Assembled trace animation: {target_gif}") -def remove_images_from_trace_folder(trace_folder_path: Path): - for file in trace_folder_path.iterdir(): - if file.suffix == ".jpeg": - file.unlink() +def remove_images_from_trace_folder(trace_folder_path: Path) -> None: + """Purge temporary individual JPEG frame captures once compiled.""" + if not trace_folder_path.is_dir(): + return + for item in trace_folder_path.iterdir(): + if item.suffix.lower() in (".jpeg", ".jpg") and item.stem.isdigit(): + try: + item.unlink(missing_ok=True) + except OSError: + pass -def create_steps_json_from_trace_folder(trace_folder_path: Path): - """Compile legacy timestamp-named step files without consuming manifests.""" - steps = [] - for file in trace_folder_path.iterdir(): - if file.suffix == ".json" and file.stem.isdigit(): - with open(file, encoding="utf-8", errors="ignore") as f: - json_content = f.read() - steps.append({"timestamp": int(file.stem), "data": json_content}) +def create_steps_json_from_trace_folder(trace_folder_path: Path) -> None: + """Collate discrete numeric step payload dumps into steps.json preserving metadata manifests.""" + if not trace_folder_path.is_dir(): + return - steps.sort(key=lambda f: f["timestamp"]) + numeric_step_files = sorted( + [f for f in trace_folder_path.iterdir() if f.suffix == ".json" and f.stem.isdigit()], + key=lambda x: int(x.stem), + ) - logger.info(f"Found {len(steps)} steps to compile") + records: list[dict[str, Any]] = [] + for step_file in numeric_step_files: + try: + records.append({ + "timestamp": int(step_file.stem), + "data": step_file.read_text(encoding="utf-8", errors="ignore"), + }) + except Exception as exc: + logger.debug(f"Unable to read step file {step_file}: {exc}") - with open(trace_folder_path / "steps.json", "w", encoding="utf-8", errors="ignore") as f: - f.write(json.dumps(steps)) + output_file = trace_folder_path / "steps.json" + output_file.write_text(json.dumps(records, ensure_ascii=False), encoding="utf-8") -def remove_steps_json_from_trace_folder(trace_folder_path: Path): - """Remove only legacy timestamp-named step files, preserving manifests.""" - for file in trace_folder_path.iterdir(): - if file.suffix == ".json" and file.stem.isdigit(): - file.unlink() +def remove_steps_json_from_trace_folder(trace_folder_path: Path) -> None: + """Clean up standalone numeric step files, leaving steps.json and recording manifests untouched.""" + if not trace_folder_path.is_dir(): + return + for item in trace_folder_path.iterdir(): + if item.suffix == ".json" and item.stem.isdigit(): + try: + item.unlink(missing_ok=True) + except OSError: + pass diff --git a/artemis/utils/recorder.py b/artemis/utils/recorder.py deleted file mode 100644 index 70dbee1..0000000 --- a/artemis/utils/recorder.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import base64 -import time - -from langchain_core.messages import BaseMessage -from artemis.context import ArtemisContext -from artemis.controllers.controller_factory import create_device_controller -from artemis.utils.logger import get_logger - -logger = get_logger(__name__) - - -async def record_interaction(ctx: ArtemisContext, response: BaseMessage): - if not ctx.execution_setup: - raise ValueError("No execution setup found") - if not ctx.execution_setup.traces_path or not ctx.execution_setup.trace_name: - raise ValueError("No traces path or trace name found") - - logger.info("Recording interaction") - controller = create_device_controller(ctx) - screenshot_base64 = await controller.screenshot() - logger.info("Screenshot taken") - try: - controller = create_device_controller(ctx) - compressed_screenshot_base64 = controller.get_compressed_b64_screenshot(screenshot_base64) - except Exception as e: - logger.error(f"Error compressing screenshot: {e}") - return "Could not record this interaction" - timestamp = time.time() - folder = ctx.execution_setup.traces_path.joinpath(ctx.execution_setup.trace_name).resolve() - folder.mkdir(parents=True, exist_ok=True) - try: - with open( - folder.joinpath(f"{int(timestamp)}.jpeg").resolve(), - "wb", - ) as f: - f.write(base64.b64decode(compressed_screenshot_base64)) - - with open( - folder.joinpath(f"{int(timestamp)}.json").resolve(), - "w", - encoding="utf-8", - ) as f: - f.write(response.model_dump_json()) - except Exception as e: - logger.error(f"Error recording interaction: {e}") - return "Screenshot recorded successfully" diff --git a/artemis/utils/requests_utils.py b/artemis/utils/requests_utils.py deleted file mode 100644 index aa56194..0000000 --- a/artemis/utils/requests_utils.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from artemis.utils.logger import get_logger -import requests - -logger = get_logger(__name__) - - -def curl_from_request(req: requests.PreparedRequest) -> str: - """Converts a requests.PreparedRequest object to a valid cURL command string.""" - command = ["curl", f"-X {req.method}"] - - for key, value in req.headers.items(): - command.append(f'-H "{key}: {value}"') - - if req.body: - body = req.body - if isinstance(body, bytes): - body = body.decode("utf-8") - # Escape single quotes in the body for shell safety - body = body.replace("'", "'\\''") - command.append(f"-d '{body}'") - - command.append(f"'{req.url}'") - - return " ".join(command) - - -def logging_hook(response, *args, **kwargs): - """Hook to log the request as a cURL command.""" - curl_command = curl_from_request(response.request) - logger.debug(f"\n--- cURL Command ---\n{curl_command}\n--------------------") - - -def get_session_with_curl_logging() -> requests.Session: - """Returns a requests.Session with cURL logging enabled.""" - session = requests.Session() - session.hooks["response"] = [logging_hook] - return session diff --git a/artemis/utils/shell_utils.py b/artemis/utils/shell_utils.py deleted file mode 100644 index 1972a73..0000000 --- a/artemis/utils/shell_utils.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import subprocess - - -def run_shell_command_on_host(command: str) -> str: - """Helper to run a shell command on the host and return the output.""" - try: - result = subprocess.run( - command, - stdin=subprocess.DEVNULL, - shell=True, - check=True, - capture_output=True, - text=True, - ) - return result.stdout.strip() - except subprocess.CalledProcessError as e: - # Log the error and stderr for better debugging - error_message = ( - f"Command '{command}' failed with exit code {e.returncode}.\nStderr: {e.stderr.strip()}" - ) - raise RuntimeError(error_message) from e diff --git a/artemis/utils/task_tree.py b/artemis/utils/task_tree.py index ee9d1f2..d7bd739 100644 --- a/artemis/utils/task_tree.py +++ b/artemis/utils/task_tree.py @@ -739,7 +739,7 @@ def _render_step_detailed( INTERNAL_SYSTEM_TOOLS = { "safety_net_validation", "safety_net_pixel_validation", - "hopper", + "entity_extractor", } if interleaved: diff --git a/artemis/utils/ui_hierarchy.py b/artemis/utils/ui_hierarchy.py index 6114497..a90089d 100644 --- a/artemis/utils/ui_hierarchy.py +++ b/artemis/utils/ui_hierarchy.py @@ -12,155 +12,151 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""UI hierarchy traversal and element geometry primitives for ARTEMIS. + +Provides structured spatial coordinates, bounding box metrics, and recursive +node querying across Android accessibility hierarchies and UIAutomator tree dumps. +""" + +from __future__ import annotations + +from typing import Any +from pydantic import BaseModel, ConfigDict, Field + from artemis.utils.logger import get_logger -from pydantic import BaseModel, Field logger = get_logger(__name__) -def __find_element_by_resource_id_in_rich_hierarchy( - hierarchy: list[dict], resource_id: str -) -> dict | None: - """Retrieves all the sibling elements for a given resource ID from a nested dictionary. - - Args: - hierarchy (dict): The nested dictionary representing the UI hierarchy. - resource_id (str): The resource-id to find. - - Returns: - list: A list of the sibling elements, or None if the resource_id is not - found. - """ - if not hierarchy: - return None - - for child in hierarchy: - if child.get("attributes", {}).get("resource-id") == resource_id: - return child.get("attributes", {}) - - for child in hierarchy: - result = __find_element_by_resource_id_in_rich_hierarchy( - child.get("children", []), resource_id - ) - if result is not None: - return result - - return None - - -def text_input_is_empty(text: str | None, hint_text: str | None) -> bool: - return not text or text == hint_text - - -def find_element_by_resource_id( - ui_hierarchy: list[dict], - resource_id: str, - index: int | None = None, - is_rich_hierarchy: bool = False, -) -> dict | None: - """Find a UI element by its resource-id in the UI hierarchy. - - Args: - ui_hierarchy: List of UI element dictionaries - resource_id: The resource-id to search for (e.g., - "com.google.android.settings.intelligence:id/open_search_view_edit_text") - - Returns: - The complete UI element dictionary if found, None otherwise - """ - if is_rich_hierarchy: - return __find_element_by_resource_id_in_rich_hierarchy(ui_hierarchy, resource_id) - - target_idx = index or 0 - - def search_recursive(elements: list[dict]) -> dict | None: - nonlocal target_idx - for element in elements: - if isinstance(element, dict): - if element.get("resourceId") == resource_id: - if target_idx == 0: - return element - target_idx -= 1 - continue - - children = element.get("children", []) - if children: - result = search_recursive(children) - if result: - return result - return None - - return search_recursive(ui_hierarchy) - - -def is_element_focused(element: dict) -> bool: - return element.get("focused", None) == "true" - - -def get_element_text(element: dict, hint_text: bool = False) -> str | None: - if hint_text: - return element.get("hintText", None) - return element.get("text", None) - - -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - class Point(BaseModel): - model_config = {"ignored_types": (CyFunctionDetector,)} + """Discrete pixel coordinate on screen.""" + + model_config = ConfigDict(frozen=True) + x: int y: int class ElementBounds(BaseModel): - model_config = {"ignored_types": (CyFunctionDetector,)} - x: int = Field(description="The x coordinate of the top-left corner of the element.") - y: int = Field(description="The y coordinate of the top-left corner of the element.") - width: int = Field(description="The width of the element.") - height: int = Field(description="The height of the element.") + """Axis-aligned rectangular boundary defining an on-screen element.""" + + model_config = ConfigDict(extra="ignore") + + x: int = Field(description="Horizontal offset of the top-left corner in pixels.") + y: int = Field(description="Vertical offset of the top-left corner in pixels.") + width: int = Field(description="Horizontal extent of the bounding box.") + height: int = Field(description="Vertical extent of the bounding box.") def get_center(self) -> Point: - return Point(x=self.x + self.width // 2, y=self.y + self.height // 2) + """Compute the centroid of the element boundary.""" + return Point(x=self.x + (self.width // 2), y=self.y + (self.height // 2)) def get_relative_point(self, x_percent: float, y_percent: float) -> Point: - """Returns the coordinates of the point at x_percent of the width and y_percent - - of the height of the element. - - Ex if x_percent = 0.95 and y_percent = 0.95, - the point is at the bottom right of the element: - <------> - | | - | x| - <------> - """ + """Compute an internal pixel coordinate by fractional offsets [0.0, 1.0].""" + clamped_x = max(0.0, min(1.0, float(x_percent))) + clamped_y = max(0.0, min(1.0, float(y_percent))) return Point( - x=int(self.x + self.width * x_percent), - y=int(self.y + self.height * y_percent), + x=int(self.x + (self.width * clamped_x)), + y=int(self.y + (self.height * clamped_y)), ) -def get_bounds_for_element(element: dict) -> ElementBounds | None: - bounds = element.get("bounds", None) - if bounds: - try: - return ElementBounds(**bounds) - except Exception as e: - logger.error(f"Failed to validate bounds: {e}") - return None +def get_bounds_for_element(element: dict[str, Any]) -> ElementBounds | None: + """Extract and validate the bounding box from a node descriptor.""" + raw_bounds = element.get("bounds") + if not isinstance(raw_bounds, dict): + return None + + try: + return ElementBounds( + x=int(raw_bounds["x"]), + y=int(raw_bounds["y"]), + width=int(raw_bounds["width"]), + height=int(raw_bounds["height"]), + ) + except (KeyError, TypeError, ValueError) as exc: + logger.error(f"Failed to validate element bounds ({raw_bounds}): {exc}") + return None + + +def text_input_is_empty(text: str | None, hint_text: str | None) -> bool: + """Determine whether an editable field contains user-entered text.""" + if not text: + return True + return text == hint_text + + +def is_element_focused(element: dict[str, Any]) -> bool: + """Check if the given node currently holds system focus.""" + focused_val = element.get("focused") + if isinstance(focused_val, bool): + return focused_val + return str(focused_val).strip().lower() == "true" + + +def get_element_text(element: dict[str, Any], hint_text: bool = False) -> str | None: + """Retrieve textual label or placeholder from an element node.""" + if hint_text: + return element.get("hintText") + return element.get("text") + + +def find_element_by_resource_id( + ui_hierarchy: list[dict[str, Any]], + resource_id: str, + index: int | None = None, + is_rich_hierarchy: bool = False, +) -> dict[str, Any] | None: + """Locate an element in the hierarchy tree matching the specified resource ID. + + Supports both flat attribute mappings and rich accessibility tree hierarchies. + """ + if not ui_hierarchy or not resource_id: + return None + + if is_rich_hierarchy: + return _search_rich_tree(ui_hierarchy, resource_id) + + target_index = max(0, index or 0) + current_match = 0 + + def _walk(nodes: list[dict[str, Any]]) -> dict[str, Any] | None: + nonlocal current_match + for node in nodes: + if not isinstance(node, dict): + continue + + if node.get("resourceId") == resource_id: + if current_match == target_index: + return node + current_match += 1 + + children = node.get("children") + if isinstance(children, list) and children: + found = _walk(children) + if found is not None: + return found + return None + + return _walk(ui_hierarchy) + + +def _search_rich_tree( + nodes: list[dict[str, Any]], resource_id: str +) -> dict[str, Any] | None: + """Recursively traverse a rich XML attribute tree for resource ID match.""" + for node in nodes: + if not isinstance(node, dict): + continue + + attributes = node.get("attributes") + if isinstance(attributes, dict) and attributes.get("resource-id") == resource_id: + return attributes + + children = node.get("children") + if isinstance(children, list) and children: + sub = _search_rich_tree(children, resource_id) + if sub is not None: + return sub + return None diff --git a/artemis/utils/video.py b/artemis/utils/video.py index a81bedd..630bc55 100644 --- a/artemis/utils/video.py +++ b/artemis/utils/video.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. """Video recording utilities for mobile devices. @@ -123,24 +126,6 @@ async def await_scrcpy_first_frame( return max(spawned_at, time.time()) -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - class RecordingSession(BaseModel): """Tracks an active video recording session.""" diff --git a/config/artemis.jsonc b/config/artemis.jsonc index 517d738..98ce44b 100644 --- a/config/artemis.jsonc +++ b/config/artemis.jsonc @@ -93,8 +93,8 @@ // 📋 Summarizer: Aggregates step history and generates structured outcome "summarizer": {}, - // 🔽 Hopper: Fast lightweight package name locator - "hopper": { + // 🔍 Entity Extractor: Fast lightweight package name locator and entity extractor + "entity_extractor": { "model": "gemini-3.5-flash-lite", "fallback": { "provider": "google", diff --git a/tests/tools/hopper/test_hopper_tools.py b/tests/tools/entity_extractor/test_entity_extractor_tools.py similarity index 89% rename from tests/tools/hopper/test_hopper_tools.py rename to tests/tools/entity_extractor/test_entity_extractor_tools.py index deb7fb7..d9f2bf7 100644 --- a/tests/tools/hopper/test_hopper_tools.py +++ b/tests/tools/entity_extractor/test_entity_extractor_tools.py @@ -14,6 +14,6 @@ """This file is intentionally empty. -The hopper agent does not expose any tools directly. It is invoked +The entity_extractor agent does not expose any tools directly. It is invoked programmatically by other tools (e.g., launch_app_tool) as an LLM utility. """ diff --git a/tests/unit/agents/test_entity_extractor.py b/tests/unit/agents/test_entity_extractor.py new file mode 100644 index 0000000..5cb8785 --- /dev/null +++ b/tests/unit/agents/test_entity_extractor.py @@ -0,0 +1,76 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for Artemis entity extraction and package resolver.""" + +from unittest.mock import AsyncMock, MagicMock, patch +import pytest + +from artemis.agents.entity_extractor.entity_extractor import ( + ExtractionResult, + extract_entity, +) + + +def test_extraction_result_model(): + """Verify ExtractionResult attributes and validation.""" + res = ExtractionResult(found=True, output="com.google.android.apps.nexuslauncher", reason="Exact match") + assert res.found is True + assert res.output == "com.google.android.apps.nexuslauncher" + assert res.reason == "Exact match" + + +@pytest.mark.asyncio +async def test_extract_entity_success(): + """Verify successful LLM entity extraction invocation.""" + mock_ctx = MagicMock() + mock_llm = MagicMock() + expected_result = ExtractionResult( + found=True, + output="com.google.android.calculator", + reason="Found calculator package in list", + ) + mock_llm.ainvoke = AsyncMock(return_value=expected_result) + + with patch("artemis.agents.entity_extractor.entity_extractor.get_llm") as mock_get_llm: + mock_get_llm.return_value.with_structured_output.return_value = mock_llm + result = await extract_entity( + request="Calculator", + data="com.google.android.calculator\ncom.google.android.calendar", + ctx=mock_ctx, + ) + + assert result.found is True + assert result.output == "com.google.android.calculator" + assert result.reason == "Found calculator package in list" + + +@pytest.mark.asyncio +async def test_extract_entity_handles_exception(): + """Verify extraction returns a graceful failure result when LLM throws an exception.""" + mock_ctx = MagicMock() + mock_llm = MagicMock() + mock_llm.ainvoke = AsyncMock(side_effect=RuntimeError("LLM connection timed out")) + + with patch("artemis.agents.entity_extractor.entity_extractor.get_llm") as mock_get_llm: + mock_get_llm.return_value.with_structured_output.return_value = mock_llm + result = await extract_entity( + request="Settings", + data="com.android.settings", + ctx=mock_ctx, + ) + + assert result.found is False + assert result.output is None + assert "LLM extraction error" in result.reason diff --git a/tests/unit/agents/test_outputter.py b/tests/unit/agents/test_outputter.py index 668e5b3..ab05f5d 100644 --- a/tests/unit/agents/test_outputter.py +++ b/tests/unit/agents/test_outputter.py @@ -26,26 +26,7 @@ from artemis.utils.logger import get_logger # noqa: E402 logger = get_logger(__name__) -class _CyFunctionDetectorMeta(type): - def __instancecheck__(self, instance): - name = type(instance).__name__ - return ( - name - in ( - "cyfunction", - "cython_function_or_method", - "builtin_function_or_method", - ) - or "cyfunction" in name.lower() - ) - - -class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): - pass - - class MockPydanticSchema(BaseModel): - model_config = {"ignored_types": (CyFunctionDetector,)} color: str price: float currency_symbol: str diff --git a/tests/unit/mcp/test_device_utils.py b/tests/unit/mcp/test_device_utils.py index 02fa97f..adf4a78 100644 --- a/tests/unit/mcp/test_device_utils.py +++ b/tests/unit/mcp/test_device_utils.py @@ -9,6 +9,8 @@ from mcp_server.utils import device_utils def test_ensure_emulator_uses_windows_creation_flags(monkeypatch) -> None: popen = MagicMock() monkeypatch.setattr(device_utils.sys, "platform", "win32") + monkeypatch.setattr(device_utils.subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200, raising=False) + monkeypatch.setattr(device_utils.subprocess, "DETACHED_PROCESS", 0x00000008, raising=False) monkeypatch.setattr(device_utils, "is_emulator_running", lambda _adb: False) monkeypatch.setattr(device_utils.os.path, "exists", lambda _path: True) monkeypatch.setattr(device_utils.subprocess, "Popen", popen) @@ -20,7 +22,7 @@ def test_ensure_emulator_uses_windows_creation_flags(monkeypatch) -> None: monkeypatch.setattr(device_utils.time, "sleep", lambda _seconds: None) assert device_utils.ensure_emulator( - adb_path="adb.exe", emulator_path="emulator.exe", timeout_seconds=1 + adb_path="adb.exe", emulator_path="emulator.exe", timeout_seconds=10 ) kwargs = popen.call_args.kwargs @@ -43,7 +45,7 @@ def test_ensure_emulator_starts_new_session_on_posix(monkeypatch) -> None: ) monkeypatch.setattr(device_utils.time, "sleep", lambda _seconds: None) - assert device_utils.ensure_emulator(adb_path="adb", emulator_path="emulator", timeout_seconds=1) + assert device_utils.ensure_emulator(adb_path="adb", emulator_path="emulator", timeout_seconds=10) kwargs = popen.call_args.kwargs assert kwargs["start_new_session"] is True diff --git a/tests/unit/mcp/test_mcp_stdio_lifecycle.py b/tests/unit/mcp/test_mcp_stdio_lifecycle.py index 7861f7a..472bc01 100644 --- a/tests/unit/mcp/test_mcp_stdio_lifecycle.py +++ b/tests/unit/mcp/test_mcp_stdio_lifecycle.py @@ -24,11 +24,8 @@ import queue import subprocess import sys import threading -import time from unittest.mock import MagicMock, patch -import pytest - from artemis.clients import ui_automator_client from artemis.runtime.awake_lease import ScreenAwakeLease from artemis.runtime.awake_service import _run_awake_adb_command @@ -204,16 +201,12 @@ def test_device_utils_isolates_stdin(): def test_ui_automator_client_isolates_stdin(): - """Verify ui_automator_client helper commands isolate stdin.""" + """Verify ui_automator_client screencap commands isolate stdin.""" + client = ui_automator_client.UIAutomatorClient("dev-1") with patch("artemis.clients.ui_automator_client.subprocess.run") as mock_run: - mock_run.return_value = MagicMock(returncode=0, stdout="package:com.test\n", stderr="") - ui_automator_client._is_package_installed("dev-1", "com.test") - assert mock_run.called - assert mock_run.call_args.kwargs.get("stdin") == subprocess.DEVNULL - - with patch("artemis.clients.ui_automator_client.subprocess.run") as mock_run: - mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="") - ui_automator_client._uninstall_package("dev-1", "com.test") + mock_run.return_value = MagicMock(returncode=0, stdout=b"", stderr="") + with patch("artemis.clients.ui_automator_client.Image.open"): + client.get_screenshot() assert mock_run.called assert mock_run.call_args.kwargs.get("stdin") == subprocess.DEVNULL diff --git a/tests/unit/sdk/test_exceptions.py b/tests/unit/sdk/test_exceptions.py new file mode 100644 index 0000000..5df5415 --- /dev/null +++ b/tests/unit/sdk/test_exceptions.py @@ -0,0 +1,63 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests validating Artemis SDK exception hierarchy and diagnostics.""" + +from artemis.sdk.types.exceptions import ( + AgentError, + AgentNotInitializedError, + AgentProfileNotFoundError, + AgentTaskRequestError, + ArtemisError, + DeviceError, + DeviceNotFoundError, + ExecutableNotFoundError, + ServerError, + ServerStartupError, +) + + +def test_exception_hierarchy(): + """Verify all custom exceptions inherit properly from ArtemisError.""" + assert issubclass(DeviceError, ArtemisError) + assert issubclass(DeviceNotFoundError, DeviceError) + assert issubclass(ServerError, ArtemisError) + assert issubclass(ServerStartupError, ServerError) + assert issubclass(AgentError, ArtemisError) + assert issubclass(AgentNotInitializedError, AgentError) + assert issubclass(AgentTaskRequestError, AgentError) + assert issubclass(AgentProfileNotFoundError, AgentTaskRequestError) + assert issubclass(ExecutableNotFoundError, ArtemisError) + + +def test_executable_not_found_error_adb_guidance(): + """Verify executable error formats download instructions for known tools.""" + err = ExecutableNotFoundError("adb") + assert "adb" in str(err) + assert "developer.android.com" in str(err) + assert err.executable_name == "adb" + + +def test_server_startup_error_formatting(): + """Verify server startup error encapsulates service identifier.""" + err = ServerStartupError(server_name="mcp_daemon") + assert "mcp_daemon" in str(err) + assert err.server_name == "mcp_daemon" + + +def test_agent_profile_not_found_error(): + """Verify profile error retains profile name.""" + err = AgentProfileNotFoundError("nonexistent_profile") + assert "nonexistent_profile" in str(err) + assert err.profile_name == "nonexistent_profile" diff --git a/tests/unit/sdk/test_task_request_builder.py b/tests/unit/sdk/test_task_request_builder.py new file mode 100644 index 0000000..e90a4b2 --- /dev/null +++ b/tests/unit/sdk/test_task_request_builder.py @@ -0,0 +1,111 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for Artemis SDK TaskRequestBuilder.""" + +from pathlib import Path +from pydantic import BaseModel +import pytest + +from artemis.sdk.builders.task_request_builder import TaskRequestBuilder, TaskRequestCommonBuilder +from artemis.sdk.types.task import TaskRequest + + +class DummyOutputSchema(BaseModel): + summary: str + item_count: int + + +def test_task_request_builder_fluent_chaining(): + """Verify basic fluent configuration and build.""" + req = ( + TaskRequestBuilder(goal="Open Settings and verify battery level") + .with_max_steps(25) + .with_trace_recording(True, path="custom_traces") + .with_locked_app_package("com.android.settings") + .with_app_path("build/app.apk") + .using_profile("flash") + .with_name("Battery Check") + .build() + ) + + assert isinstance(req, TaskRequest) + assert req.goal == "Open Settings and verify battery level" + assert req.max_steps == 25 + assert req.record_trace is True + assert req.trace_path == Path("custom_traces") + assert req.locked_app_package == "com.android.settings" + assert req.app_path == Path("build/app.apk") + assert req.profile == "flash" + assert req.task_name == "Battery Check" + + +def test_task_request_builder_with_output_format(): + """Verify builder binds structured schema properly.""" + req = ( + TaskRequestBuilder(goal="Extract items") + .with_output_format(DummyOutputSchema) + .build() + ) + + assert req.output_format is DummyOutputSchema + assert req.output_description is None + + +def test_task_request_builder_with_output_description(): + """Verify builder accepts unformatted string deliverables.""" + req = ( + TaskRequestBuilder(goal="Extract items") + .with_output_description("Return a numbered list") + .build() + ) + + assert req.output_description == "Return a numbered list" + assert req.output_format is None + + +def test_task_request_builder_conflicting_outputs_raises(): + """Verify that specifying both output_format and output_description raises ValueError.""" + builder = ( + TaskRequestBuilder(goal="Extract items") + .with_output_format(DummyOutputSchema) + .with_output_description("Description text") + ) + with pytest.raises(ValueError, match="Cannot specify both output_format and output_description"): + builder.build() + + +def test_task_request_builder_empty_goal_raises(): + """Verify empty or whitespace-only goal raises ValueError.""" + with pytest.raises(ValueError, match="Task goal must be a non-empty"): + TaskRequestBuilder(goal=" ").build() + + +def test_task_request_builder_from_common(): + """Verify task request initialization derived from common base settings.""" + common_settings = ( + TaskRequestCommonBuilder() + .with_max_steps(42) + .with_trace_recording(False) + .with_locked_app_package("org.artemis.target") + .with_app_path(Path("/tmp/target.apk")) + .build() + ) + + derived = TaskRequestBuilder.from_common("Derived goal", common_settings).build() + assert derived.goal == "Derived goal" + assert derived.max_steps == 42 + assert derived.record_trace is False + assert derived.locked_app_package == "org.artemis.target" + assert derived.app_path == Path("/tmp/target.apk") diff --git a/tests/unit/test_config_package.py b/tests/unit/test_config_package.py index 5b8c56a..313d322 100644 --- a/tests/unit/test_config_package.py +++ b/tests/unit/test_config_package.py @@ -154,7 +154,7 @@ def test_llm_config_parsing_and_merging(): assert isinstance(llm_cfg, LLMConfig) assert llm_cfg.planner.provider in ("google", "openai", "openrouter", "xai", "vertexai") assert llm_cfg.get_agent("planner") is not None - assert llm_cfg.get_utils("hopper") is not None + assert llm_cfg.get_utils("entity_extractor") is not None # Deep merge overrides overrides = { diff --git a/tests/unit/test_ui_automator_client_awake.py b/tests/unit/test_ui_automator_client_awake.py index 12e3a82..19152a2 100644 --- a/tests/unit/test_ui_automator_client_awake.py +++ b/tests/unit/test_ui_automator_client_awake.py @@ -11,16 +11,14 @@ from artemis.drivers.android.adb_driver import AndroidAdbDriver "artemis.clients.ui_automator_client.ensure_device_awake", return_value="host_heartbeat", ) -@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed") def test_new_ui_connection_enrolls_device_in_shared_awake_strategy( - mock_remove_maestro, mock_ensure_awake, mock_connect + mock_ensure_awake, mock_connect ): client = UIAutomatorClient("device-123") client.connect() client.connect() - mock_remove_maestro.assert_called_once_with("device-123") mock_ensure_awake.assert_called_once_with("device-123") mock_connect.assert_called_once_with("device-123") @@ -30,9 +28,8 @@ def test_new_ui_connection_enrolls_device_in_shared_awake_strategy( "artemis.clients.ui_automator_client.ensure_device_awake", return_value="host_heartbeat", ) -@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed") def test_failed_ui_connection_does_not_stop_process_awake_service( - _mock_remove_maestro, mock_ensure_awake, _mock_connect + mock_ensure_awake, _mock_connect ): client = UIAutomatorClient("device-123") @@ -48,9 +45,8 @@ def test_failed_ui_connection_does_not_stop_process_awake_service( "artemis.clients.ui_automator_client.ensure_device_awake", return_value="host_heartbeat", ) -@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed") def test_client_disconnect_does_not_send_power_cleanup_commands( - _mock_remove_maestro, mock_ensure_awake, _mock_connect + mock_ensure_awake, _mock_connect ): client = UIAutomatorClient("device-123") client.connect() diff --git a/tests/unit/tools/test_launch_app.py b/tests/unit/tools/test_launch_app.py index bc45611..c2c07bb 100644 --- a/tests/unit/tools/test_launch_app.py +++ b/tests/unit/tools/test_launch_app.py @@ -13,10 +13,11 @@ # limitations under the License. from unittest.mock import MagicMock, patch -from artemis.agents.hopper.hopper import HopperOutput +import pytest + +from artemis.agents.entity_extractor.entity_extractor import ExtractionResult from artemis.context import ArtemisContext from artemis.tools.mobile.launch_app import find_package -import pytest @pytest.fixture @@ -28,63 +29,63 @@ def mock_context(): @pytest.mark.asyncio @patch("artemis.tools.mobile.launch_app.list_packages_async") -@patch("artemis.tools.mobile.launch_app.hopper") -async def test_find_package_cache_hit(mock_hopper, mock_list_packages, mock_context): +@patch("artemis.tools.mobile.launch_app.extract_entity") +async def test_find_package_cache_hit(mock_extract_entity, mock_list_packages, mock_context): mock_list_packages.return_value = "com.example.app1\ncom.example.app2" - # First call: not in cache, calls hopper - mock_hopper.return_value = HopperOutput( + # First call: not in cache, calls extract_entity + mock_extract_entity.return_value = ExtractionResult( found=True, output="com.example.app1", reason="Found it" ) pkg1 = await find_package(mock_context, "App 1") assert pkg1 == "com.example.app1" - assert mock_hopper.call_count == 1 + assert mock_extract_entity.call_count == 1 assert mock_context.package_cache["App 1"] == "com.example.app1" - # Second call: cache hit, does not call hopper or list_packages again - mock_hopper.reset_mock() + # Second call: cache hit, does not call extract_entity or list_packages again + mock_extract_entity.reset_mock() mock_list_packages.reset_mock() pkg2 = await find_package(mock_context, "App 1") assert pkg2 == "com.example.app1" - mock_hopper.assert_not_called() + mock_extract_entity.assert_not_called() mock_list_packages.assert_not_called() @pytest.mark.asyncio @patch("artemis.tools.mobile.launch_app.list_packages_async") -@patch("artemis.tools.mobile.launch_app.hopper") -async def test_find_package_not_found_caches_none(mock_hopper, mock_list_packages, mock_context): +@patch("artemis.tools.mobile.launch_app.extract_entity") +async def test_find_package_not_found_caches_none(mock_extract_entity, mock_list_packages, mock_context): mock_list_packages.return_value = "com.example.app1\ncom.example.app2" - # Hopper doesn't find it - mock_hopper.return_value = HopperOutput(found=False, output=None, reason="Not found") + # Extractor doesn't find it + mock_extract_entity.return_value = ExtractionResult(found=False, output=None, reason="Not found") pkg1 = await find_package(mock_context, "App 3") assert pkg1 is None - assert mock_hopper.call_count == 1 + assert mock_extract_entity.call_count == 1 assert mock_context.package_cache["App 3"] is None - # Second call: cache hit (None), does not call hopper again - mock_hopper.reset_mock() + # Second call: cache hit (None), does not call extract_entity again + mock_extract_entity.reset_mock() pkg2 = await find_package(mock_context, "App 3") assert pkg2 is None - mock_hopper.assert_not_called() + mock_extract_entity.assert_not_called() @pytest.mark.asyncio @patch("artemis.tools.mobile.launch_app.list_packages_async") -@patch("artemis.tools.mobile.launch_app.hopper") -async def test_find_package_passes_use_fallback(mock_hopper, mock_list_packages, mock_context): +@patch("artemis.tools.mobile.launch_app.extract_entity") +async def test_find_package_passes_use_fallback(mock_extract_entity, mock_list_packages, mock_context): mock_list_packages.return_value = "com.example.app1\ncom.example.app2" - mock_hopper.return_value = HopperOutput( + mock_extract_entity.return_value = ExtractionResult( found=True, output="com.example.app1", reason="Found it" ) # Pass use_fallback=False await find_package(mock_context, "App 1", use_fallback=False) - mock_hopper.assert_called_once_with( + mock_extract_entity.assert_called_once_with( ctx=mock_context, request=("I'm looking for the package name of the following app: 'App 1'"), data="com.example.app1\ncom.example.app2", diff --git a/tests/unit/tools/test_utils.py b/tests/unit/tools/test_utils.py index 63b9fd9..7f067a7 100644 --- a/tests/unit/tools/test_utils.py +++ b/tests/unit/tools/test_utils.py @@ -12,446 +12,340 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio -import sys -from unittest.mock import Mock, patch +"""Unit tests for ARTEMIS tool utility functions and element interaction helpers.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch import pytest -# Mock the problematic langgraph import at module level -_orig_chat_agent_executor = sys.modules.get("langgraph.prebuilt.chat_agent_executor") -_orig_state = sys.modules.get("artemis.graph.state") - -sys.modules["langgraph.prebuilt.chat_agent_executor"] = Mock() -sys.modules["artemis.graph.state"] = Mock() - -from artemis.context import DeviceContext, DevicePlatform, ArtemisContext # noqa: E402 -from artemis.tools.types import Target # noqa: E402 -from artemis.tools.utils import ( # noqa: E402 +from artemis.context import ArtemisContext, DeviceContext, DevicePlatform +from artemis.controllers.types import CoordinatesSelectorRequest +from artemis.tools.types import Target +from artemis.tools.utils import ( IdSelectorRequest, + IdWithTextSelectorRequest, SelectorRequestWithCoordinates, + SelectorRequestWithPercentages, + TextSelectorRequest, + _extract_resource_id_and_text_from_selector, + find_element_by_text, focus_element_if_needed, + has_valid_selectors, move_cursor_to_end_if_bounds, + tap_bottom_right_of_element, + validate_coordinates_bounds, ) -from artemis.utils.ui_hierarchy import ElementBounds # noqa: E402 - -# Restore original modules to prevent contaminating other tests -if _orig_chat_agent_executor is not None: - sys.modules["langgraph.prebuilt.chat_agent_executor"] = _orig_chat_agent_executor -else: - del sys.modules["langgraph.prebuilt.chat_agent_executor"] - -if _orig_state is not None: - sys.modules["artemis.graph.state"] = _orig_state -else: - del sys.modules["artemis.graph.state"] +from artemis.utils.ui_hierarchy import ElementBounds @pytest.fixture -def mock_context(): - """Create a mock ArtemisContext for testing.""" - ctx = Mock(spec=ArtemisContext) - - # Create device context with necessary attributes - ctx.device = Mock(spec=DeviceContext) +def dummy_context() -> MagicMock: + """Fixture providing an ARTEMIS runtime context configured for Android.""" + ctx = MagicMock(spec=ArtemisContext) + ctx.device = MagicMock(spec=DeviceContext) ctx.device.mobile_platform = DevicePlatform.ANDROID - ctx.device.device_id = "test_device_123" + ctx.device.device_id = "test-device-pixel8" ctx.device.device_width = 1080 - ctx.device.device_height = 2340 - ctx.device.host_platform = "LINUX" - - ctx.ui_adb_client = Mock() - - # Mock the ADB client for Android - ctx.adb_client = Mock() - mock_device = Mock() - mock_device.shell = Mock(return_value="") - ctx.adb_client.device = Mock(return_value=mock_device) - - # Mock the ADB client for Android - mock_response = Mock() - mock_response.json.return_value = {"elements": []} - ctx.ui_adb_client.get_screen_data = Mock(return_value=mock_response) - ctx.ui_adb_client.get_hierarchy = Mock( - return_value=( - b'' - ) - ) - + ctx.device.device_height = 2400 return ctx @pytest.fixture -def mock_state(): - """Create a mock State for testing.""" - state = Mock() +def mock_graph_state() -> MagicMock: + """Fixture providing a mock LangGraph state container.""" + state = MagicMock() state.latest_ui_hierarchy = [] return state @pytest.fixture -def sample_element(): - """Create a sample UI element for testing.""" +def sample_hierarchy_node() -> dict: + """Fixture providing a standard flat hierarchy node.""" return { - "resourceId": "com.example:id/text_input", - "text": "Sample text", - "bounds": {"x": 100, "y": 200, "width": 300, "height": 50}, + "resourceId": "com.google.android.apps:id/search_query", + "text": "Search Google", + "bounds": {"x": 120, "y": 250, "width": 400, "height": 80}, "focused": "false", } @pytest.fixture -def sample_rich_element(): - """Create a sample rich UI element for testing.""" +def sample_nested_node() -> dict: + """Fixture providing a rich nested hierarchy node.""" return { "attributes": { - "resource-id": "com.example:id/text_input", + "resource-id": "com.google.android.apps:id/search_query", + "text": "Search Google", + "bounds": {"x": 120, "y": 250, "width": 400, "height": 80}, "focused": "false", - "text": "Sample text", - "bounds": {"x": 100, "y": 200, "width": 300, "height": 50}, }, "children": [], } -class TestMoveCursorToEndIfBounds: - """Test cases for move_cursor_to_end_if_bounds function.""" +# ============================================================================== +# Cursor Placement & Boundary Alignment Tests +# ============================================================================== - @patch("artemis.tools.utils.tap") + +class TestCursorAlignment: + """Validates precise cursor placement against interactive targets.""" + + @pytest.mark.asyncio + @patch("artemis.tools.utils.tap", new_callable=AsyncMock) @patch("artemis.tools.utils.find_element_by_resource_id") - def test_move_cursor_with_resource_id( + async def test_cursor_routes_to_end_via_resource_id( self, - mock_find_element, - mock_tap, - mock_context, - mock_state, - sample_element, + mock_find_elem: MagicMock, + mock_tap_fn: AsyncMock, + dummy_context: MagicMock, + mock_graph_state: MagicMock, + sample_hierarchy_node: dict, ): - """Test moving cursor using resource_id (highest priority).""" - mock_state.latest_ui_hierarchy = [sample_element] - mock_find_element.return_value = sample_element + """Resource ID locator should calculate end-boundary tap (99% width/height offset).""" + mock_graph_state.latest_ui_hierarchy = [sample_hierarchy_node] + mock_find_elem.return_value = sample_hierarchy_node - target = Target( - resource_id="com.example:id/text_input", - resource_id_index=None, - text=None, - text_index=None, - bounds=None, - ) - result = asyncio.run( - move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) + target = Target(resource_id="com.google.android.apps:id/search_query") + elem = await move_cursor_to_end_if_bounds( + ctx=dummy_context, state=mock_graph_state, target=target ) - mock_find_element.assert_called_once_with( - ui_hierarchy=[sample_element], - resource_id="com.example:id/text_input", + mock_find_elem.assert_called_once_with( + ui_hierarchy=[sample_hierarchy_node], + resource_id="com.google.android.apps:id/search_query", index=0, ) - mock_tap.assert_called_once() - call_args = mock_tap.call_args[1] - selector_request = call_args["selector_request"] - assert isinstance(selector_request, SelectorRequestWithCoordinates) - coords = selector_request.coordinates - assert coords.x == 397 # 100 + 300 * 0.99 - assert coords.y == 249 # 200 + 50 * 0.99 - assert result == sample_element + mock_tap_fn.assert_awaited_once() + kwargs = mock_tap_fn.await_args.kwargs + selector_req = kwargs["selector_request"] + assert isinstance(selector_req, SelectorRequestWithCoordinates) + assert selector_req.coordinates.x == 516 + assert selector_req.coordinates.y == 329 + assert elem == sample_hierarchy_node - @patch("artemis.tools.utils.tap") + @pytest.mark.asyncio + @patch("artemis.tools.utils.tap", new_callable=AsyncMock) @patch("artemis.tools.utils.find_element_by_resource_id") - def test_move_cursor_with_coordinates_only( - self, mock_find_element, mock_tap, mock_context, mock_state + async def test_cursor_routes_using_direct_bounds( + self, + mock_find_elem: MagicMock, + mock_tap_fn: AsyncMock, + dummy_context: MagicMock, + mock_graph_state: MagicMock, ): - """Test moving cursor when only coordinates are provided.""" - bounds = ElementBounds(x=50, y=150, width=200, height=40) - target = Target( - resource_id=None, - resource_id_index=None, - text=None, - text_index=None, - bounds=bounds, + """Direct bounding box coordinates should bypass hierarchy inspection.""" + bounds = ElementBounds(x=80, y=100, width=200, height=60) + target = Target(bounds=bounds) + + result = await move_cursor_to_end_if_bounds( + ctx=dummy_context, state=mock_graph_state, target=target ) - result = asyncio.run( - move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) - ) + mock_find_elem.assert_not_called() + mock_tap_fn.assert_awaited_once() + kwargs = mock_tap_fn.await_args.kwargs + selector_req = kwargs["selector_request"] + assert selector_req.coordinates.x == 278 + assert selector_req.coordinates.y == 159 + assert result is None - mock_find_element.assert_not_called() - mock_tap.assert_called_once() - call_args = mock_tap.call_args[1] - selector_request = call_args["selector_request"] - coords = selector_request.coordinates - assert coords.x == 248 # 50 + 200 * 0.99 - assert coords.y == 189 # 150 + 40 * 0.99 - assert result is None # No element is returned when using coords directly - - @patch("artemis.tools.utils.tap") + @pytest.mark.asyncio + @patch("artemis.tools.utils.tap", new_callable=AsyncMock) @patch("artemis.tools.utils.find_element_by_text") - def test_move_cursor_with_text_only_success( - self, mock_find_text, mock_tap, mock_context, mock_state, sample_element + async def test_cursor_routes_by_matched_text( + self, + mock_find_text: MagicMock, + mock_tap_fn: AsyncMock, + dummy_context: MagicMock, + mock_graph_state: MagicMock, + sample_hierarchy_node: dict, ): - """Test moving cursor when only text is provided and succeeds.""" - mock_state.latest_ui_hierarchy = [sample_element] - mock_find_text.return_value = sample_element + """Visible text locator should successfully resolve bounds and trigger cursor placement.""" + mock_graph_state.latest_ui_hierarchy = [sample_hierarchy_node] + mock_find_text.return_value = sample_hierarchy_node - target = Target( - resource_id=None, - resource_id_index=None, - text="Sample text", - text_index=0, - bounds=None, - ) - result = asyncio.run( - move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) + target = Target(text="Search Google", text_index=0) + result = await move_cursor_to_end_if_bounds( + ctx=dummy_context, state=mock_graph_state, target=target ) - mock_find_text.assert_called_once_with([sample_element], "Sample text", index=0) - mock_tap.assert_called_once() - assert result == sample_element + mock_find_text.assert_called_once_with([sample_hierarchy_node], "Search Google", index=0) + mock_tap_fn.assert_awaited_once() + assert result == sample_hierarchy_node - @patch("artemis.tools.utils.tap") + @pytest.mark.asyncio + @patch("artemis.tools.utils.tap", new_callable=AsyncMock) @patch("artemis.tools.utils.find_element_by_text") - def test_move_cursor_with_text_only_element_not_found( - self, mock_find_text, mock_tap, mock_context, mock_state + async def test_cursor_skips_when_text_unmatched( + self, + mock_find_text: MagicMock, + mock_tap_fn: AsyncMock, + dummy_context: MagicMock, + mock_graph_state: MagicMock, ): - """Test when searching by text finds no element.""" - mock_state.latest_ui_hierarchy = [] + """Unresolved text target returns None without tapping.""" mock_find_text.return_value = None + target = Target(text="Unmatched Query") - target = Target( - resource_id=None, - resource_id_index=None, - text="Nonexistent text", - text_index=None, - bounds=None, + result = await move_cursor_to_end_if_bounds( + ctx=dummy_context, state=mock_graph_state, target=target ) - result = asyncio.run( - move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) - ) - - mock_tap.assert_not_called() - assert result is None - - @patch("artemis.tools.utils.tap") - @patch("artemis.tools.utils.find_element_by_text") - def test_move_cursor_with_text_only_no_bounds( - self, mock_find_text, mock_tap, mock_context, mock_state - ): - """Test when element is found by text but has no bounds.""" - element_no_bounds = {"text": "Text without bounds"} - mock_state.latest_ui_hierarchy = [element_no_bounds] - mock_find_text.return_value = element_no_bounds - - target = Target( - resource_id=None, - resource_id_index=None, - text="Text without bounds", - text_index=None, - bounds=None, - ) - result = asyncio.run( - move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) - ) - - mock_tap.assert_not_called() - assert result is None # Should return None as no action was taken - - @patch("artemis.tools.utils.find_element_by_resource_id") - def test_move_cursor_element_not_found_by_id(self, mock_find_element, mock_context, mock_state): - """Test when element is not found by resource_id.""" - mock_find_element.return_value = None - - target = Target( - resource_id="com.example:id/nonexistent", - resource_id_index=None, - text=None, - text_index=None, - bounds=None, - ) - result = asyncio.run( - move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) - ) - + mock_tap_fn.assert_not_awaited() assert result is None -class TestFocusElementIfNeeded: - """Test cases for focus_element_if_needed function.""" +# ============================================================================== +# Element Focus Resolution Tests +# ============================================================================== - @patch("artemis.tools.utils.tap") - @patch("artemis.tools.utils.find_element_by_resource_id") - def test_focus_element_already_focused( - self, mock_find_element, mock_tap, mock_context, sample_rich_element + +class TestElementFocusing: + """Validates conditional focusing behavior prior to keyboard inputs.""" + + @pytest.mark.asyncio + @patch("artemis.tools.utils.tap", new_callable=AsyncMock) + @patch("artemis.tools.utils.UnifiedMobileController") + async def test_skips_tap_when_already_focused( + self, + mock_controller_cls: MagicMock, + mock_tap_fn: AsyncMock, + dummy_context: MagicMock, + sample_hierarchy_node: dict, ): - """Test when element is already focused.""" - focused_element = sample_rich_element.copy() - focused_element["attributes"]["focused"] = "true" + """Elements reporting focused state true should avoid redundant tap interactions.""" + focused_elem = sample_hierarchy_node.copy() + focused_elem["focused"] = "true" - mock_response = Mock() - mock_response.json.return_value = {"elements": [focused_element]} - mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response) - mock_find_element.return_value = focused_element["attributes"] + mock_ctrl = MagicMock() + mock_ctrl.get_ui_elements = AsyncMock(return_value=[focused_elem]) + mock_controller_cls.return_value = mock_ctrl - target = Target( - resource_id="com.example:id/text_input", - resource_id_index=None, - text=None, - text_index=None, - bounds=None, - ) - result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) + target = Target(resource_id="com.google.android.apps:id/search_query") + focus_strategy = await focus_element_if_needed(ctx=dummy_context, target=target) - mock_tap.assert_not_called() - assert result == "resource_id" - mock_context.ui_adb_client.get_hierarchy.assert_called_once() + mock_tap_fn.assert_not_awaited() + assert focus_strategy == "resource_id" - @patch("artemis.tools.utils.tap") - @patch("artemis.tools.utils.find_element_by_resource_id") - def test_focus_element_needs_focus_success( - self, mock_find_element, mock_tap, mock_context, sample_rich_element + @pytest.mark.asyncio + @patch("artemis.tools.utils.tap", new_callable=AsyncMock) + @patch("artemis.tools.utils.UnifiedMobileController") + async def test_focus_executes_tap_for_unfocused_element( + self, + mock_controller_cls: MagicMock, + mock_tap_fn: AsyncMock, + dummy_context: MagicMock, + sample_hierarchy_node: dict, ): - """Test when element needs focus and focusing succeeds.""" - unfocused_element = sample_rich_element - focused_element = { - "attributes": { - "resource-id": "com.example:id/text_input", - "focused": "true", - }, - "children": [], - } + """Unfocused element triggers focus tap and confirms state change.""" + unfocused = sample_hierarchy_node.copy() + unfocused["focused"] = "false" + focused = sample_hierarchy_node.copy() + focused["focused"] = "true" - mock_find_element.side_effect = [ - unfocused_element["attributes"], - focused_element["attributes"], - ] + mock_ctrl = MagicMock() + mock_ctrl.get_ui_elements = AsyncMock(side_effect=[[unfocused], [focused]]) + mock_controller_cls.return_value = mock_ctrl - target = Target( - resource_id="com.example:id/text_input", - resource_id_index=None, - text=None, - text_index=None, - bounds=None, - ) - result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) + target = Target(resource_id="com.google.android.apps:id/search_query") + strategy = await focus_element_if_needed(ctx=dummy_context, target=target) - mock_tap.assert_called_once_with( - ctx=mock_context, - selector_request=IdSelectorRequest(id="com.example:id/text_input"), + mock_tap_fn.assert_awaited_once_with( + ctx=dummy_context, + selector_request=IdSelectorRequest(id="com.google.android.apps:id/search_query"), index=0, ) - assert mock_context.ui_adb_client.get_hierarchy.call_count == 2 - assert result == "resource_id" + assert strategy == "resource_id" - @patch("artemis.tools.utils.tap") - @patch("artemis.tools.utils.logger") - @patch("artemis.tools.utils.find_element_by_resource_id") - def test_focus_id_and_text_mismatch_fallback_to_text( + @pytest.mark.asyncio + @patch("artemis.tools.utils.tap", new_callable=AsyncMock) + @patch("artemis.tools.utils.UnifiedMobileController") + async def test_focus_falls_back_to_text_coordinates( self, - mock_find_id, - mock_logger, - mock_tap, - mock_context, - sample_rich_element, + mock_controller_cls: MagicMock, + mock_tap_fn: AsyncMock, + dummy_context: MagicMock, + sample_hierarchy_node: dict, ): - """Test fallback when resource_id and text point to different elements.""" - element_from_id = sample_rich_element["attributes"].copy() - element_from_id["text"] = "Different text" + """When ID is unavailable, focusing falls back to text center tap coordinates.""" + elem = sample_hierarchy_node.copy() + elem["bounds"] = {"x": 20, "y": 40, "width": 200, "height": 60} - element_from_text = sample_rich_element.copy() - element_from_text["attributes"]["bounds"] = { - "x": 10, - "y": 20, - "width": 100, - "height": 30, - } + mock_ctrl = MagicMock() + mock_ctrl.get_ui_elements = AsyncMock(return_value=[elem]) + mock_controller_cls.return_value = mock_ctrl - mock_response = Mock() - mock_response.json.return_value = {"elements": [element_from_text]} - mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response) - mock_find_id.return_value = element_from_id + target = Target(text="Search Google") + strategy = await focus_element_if_needed(ctx=dummy_context, target=target) - with patch("artemis.tools.utils.find_element_by_text") as mock_find_text: - mock_find_text.return_value = element_from_text["attributes"] - - target = Target( - resource_id="com.example:id/text_input", - resource_id_index=None, - text="Sample text", - text_index=None, - bounds=None, - ) - result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) - - mock_logger.warning.assert_called_once() - mock_tap.assert_called_once() - assert result == "text" - - @patch("artemis.tools.utils.tap") - @patch("artemis.tools.utils.find_element_by_text") - def test_focus_fallback_to_text( - self, mock_find_text, mock_tap, mock_context, sample_rich_element - ): - """Test fallback to focusing using text.""" - element_with_bounds = sample_rich_element.copy() - element_with_bounds["attributes"]["bounds"] = { - "x": 10, - "y": 20, - "width": 100, - "height": 30, - } - - mock_response = Mock() - mock_response.json.return_value = {"elements": [element_with_bounds]} - mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response) - mock_find_text.return_value = element_with_bounds["attributes"] - - target = Target( - resource_id=None, - resource_id_index=None, - text="Sample text", - text_index=None, - bounds=None, - ) - result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) - - mock_find_text.assert_called_once() - mock_tap.assert_called_once() - call_args = mock_tap.call_args[1] - selector = call_args["selector_request"] - assert isinstance(selector, SelectorRequestWithCoordinates) - assert selector.coordinates.x == 60 # 10 + 100/2 - assert selector.coordinates.y == 35 # 20 + 30/2 - assert result == "text" + mock_tap_fn.assert_awaited_once() + req = mock_tap_fn.await_args.kwargs["selector_request"] + assert isinstance(req, SelectorRequestWithCoordinates) + assert req.coordinates.x == 120 # 20 + 200/2 + assert req.coordinates.y == 70 # 40 + 60/2 + assert strategy == "text" + @pytest.mark.asyncio @patch("artemis.tools.utils.logger") - def test_focus_all_locators_fail(self, mock_logger, mock_context): - """Test failure when no locator can find an element.""" + @patch("artemis.tools.utils.UnifiedMobileController") + async def test_focus_aborts_when_no_locators_resolve( + self, + mock_controller_cls: MagicMock, + mock_logger: MagicMock, + dummy_context: MagicMock, + ): + """When none of the declared locators match an active element, returns None and logs error.""" + mock_ctrl = MagicMock() + mock_ctrl.get_ui_elements = AsyncMock(return_value=[]) + mock_controller_cls.return_value = mock_ctrl - mock_response = Mock() - mock_response.json.return_value = {"elements": []} - mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response) - with ( - patch("artemis.tools.utils.find_element_by_resource_id") as mock_find_id, - patch("artemis.tools.utils.find_element_by_text") as mock_find_text, - ): - mock_find_id.return_value = None - mock_find_text.return_value = None + target = Target(resource_id="unknown_id", text="unknown_text") + strategy = await focus_element_if_needed(ctx=dummy_context, target=target) - target = Target( - resource_id="nonexistent", - resource_id_index=None, - text="nonexistent", - text_index=None, - bounds=None, - ) - result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target)) + mock_logger.error.assert_called_once() + assert strategy is None - mock_logger.error.assert_called_once_with( - "Failed to focus element." - + " No valid locator (resource_id, coordinates, or text) succeeded." + +# ============================================================================== +# Helper Verification & Selector Validation Tests +# ============================================================================== + + +class TestSelectorValidation: + """Verifies target completeness and boundary constraint checks.""" + + def test_has_valid_selectors_matrix(self): + """Verifies selector detection across valid and invalid combinations.""" + assert has_valid_selectors(Target(resource_id="elem_id")) + assert has_valid_selectors(Target(text="Label")) + assert has_valid_selectors(Target(bounds=ElementBounds(x=0, y=0, width=10, height=10))) + assert not has_valid_selectors(Target()) + + def test_validate_coordinates_bounds(self): + """Checks bounds containment against physical device display limits.""" + valid_target = Target(bounds=ElementBounds(x=100, y=200, width=100, height=100)) + assert validate_coordinates_bounds(valid_target, 1080, 2400) is None + + out_width = Target(bounds=ElementBounds(x=1050, y=200, width=100, height=100)) + assert validate_coordinates_bounds(out_width, 1080, 2400) is not None + + out_height = Target(bounds=ElementBounds(x=100, y=2380, width=100, height=100)) + assert validate_coordinates_bounds(out_height, 1080, 2400) is not None + + def test_extract_selector_attributes(self): + """Extracts resource ID and text from heterogeneous selector requests.""" + req_id = IdSelectorRequest(id="view_item") + assert _extract_resource_id_and_text_from_selector(req_id) == ("view_item", None) + + req_text = TextSelectorRequest(text="Confirm") + assert _extract_resource_id_and_text_from_selector(req_text) == (None, "Confirm") + + req_both = IdWithTextSelectorRequest(id="submit_btn", text="Submit") + assert _extract_resource_id_and_text_from_selector(req_both) == ("submit_btn", "Submit") + + req_coords = SelectorRequestWithCoordinates( + coordinates=CoordinatesSelectorRequest(x=50, y=50) ) - assert result is None - - -if __name__ == "__main__": - pytest.main([__file__]) + assert _extract_resource_id_and_text_from_selector(req_coords) == (None, None) diff --git a/tests/unit/utils/test_ui_hierarchy.py b/tests/unit/utils/test_ui_hierarchy.py index 9a8642a..54c1dc9 100644 --- a/tests/unit/utils/test_ui_hierarchy.py +++ b/tests/unit/utils/test_ui_hierarchy.py @@ -11,6 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use) +# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0. + +"""Unit tests validating Artemis UI hierarchy parsing and geometry computation.""" from unittest.mock import patch @@ -29,215 +34,160 @@ def test_text_input_is_empty(): assert text_input_is_empty(text=None, hint_text=None) assert text_input_is_empty(text="", hint_text=None) assert text_input_is_empty(text="", hint_text="") - assert text_input_is_empty(text="text", hint_text="text") + assert text_input_is_empty(text="search", hint_text="search") - assert not text_input_is_empty(text="text", hint_text=None) - assert not text_input_is_empty(text="text", hint_text="") + assert not text_input_is_empty(text="search", hint_text=None) + assert not text_input_is_empty(text="search", hint_text="") def test_find_element_by_resource_id(): - ui_hierarchy = [ + tree = [ { - "resourceId": "com.example:id/button1", - "text": "Button 1", + "resourceId": "com.google.android.settings:id/search_bar", + "text": "Search Settings", "children": [], }, { - "resourceId": "com.example:id/container", + "resourceId": "com.google.android.settings:id/content_frame", "children": [ { - "resourceId": "com.example:id/nested_button", - "text": "Nested Button", + "resourceId": "com.google.android.settings:id/network_item", + "text": "Network & Internet", "children": [], } ], }, ] - result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button1") - assert result is not None - assert result["resourceId"] == "com.example:id/button1" - assert result["text"] == "Button 1" + target = find_element_by_resource_id(tree, "com.google.android.settings:id/search_bar") + assert target is not None + assert target["resourceId"] == "com.google.android.settings:id/search_bar" + assert target["text"] == "Search Settings" - result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nested_button") - assert result is not None - assert result["resourceId"] == "com.example:id/nested_button" - assert result["text"] == "Nested Button" + nested = find_element_by_resource_id(tree, "com.google.android.settings:id/network_item") + assert nested is not None + assert nested["resourceId"] == "com.google.android.settings:id/network_item" - result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nonexistent") - assert result is None - - result = find_element_by_resource_id([], "com.example:id/button1") - assert result is None + assert find_element_by_resource_id(tree, "com.google.android.settings:id/missing") is None + assert find_element_by_resource_id([], "com.google.android.settings:id/search_bar") is None def test_find_element_by_resource_id_with_index(): - ui_hierarchy = [ + duplicate_items = [ + {"resourceId": "com.artemis.ui:id/row_item", "text": "Item Alpha", "children": []}, + {"resourceId": "com.artemis.ui:id/row_item", "text": "Item Beta", "children": []}, { - "resourceId": "com.example:id/button", - "text": "Button 1", - "children": [], - }, - { - "resourceId": "com.example:id/button", - "text": "Button 2", - "children": [], - }, - { - "resourceId": "com.example:id/container", + "resourceId": "com.artemis.ui:id/section_group", "children": [ - { - "resourceId": "com.example:id/button", - "text": "Button 3", - "children": [], - } + {"resourceId": "com.artemis.ui:id/row_item", "text": "Item Gamma", "children": []} ], }, ] - result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=0) - assert result is not None - assert result["text"] == "Button 1" + first = find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=0) + assert first is not None + assert first["text"] == "Item Alpha" - result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=1) - assert result is not None - assert result["text"] == "Button 2" + second = find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=1) + assert second is not None + assert second["text"] == "Item Beta" - result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=2) - assert result is not None - assert result["text"] == "Button 3" + third = find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=2) + assert third is not None + assert third["text"] == "Item Gamma" - result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=3) - assert result is None + assert find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=3) is None def test_find_element_by_resource_id_rich_hierarchy(): - rich_hierarchy = [ + rich_nodes = [ { - "attributes": {"resource-id": "com.example:id/button1"}, + "attributes": {"resource-id": "com.android.systemui:id/clock"}, "children": [], }, { - "attributes": {"resource-id": "com.example:id/container"}, + "attributes": {"resource-id": "com.android.systemui:id/status_bar"}, "children": [ { - "attributes": {"resource-id": "com.example:id/nested_button"}, + "attributes": {"resource-id": "com.android.systemui:id/battery_meter"}, "children": [], } ], }, ] - result = find_element_by_resource_id( - rich_hierarchy, "com.example:id/button1", is_rich_hierarchy=True + match = find_element_by_resource_id( + rich_nodes, "com.android.systemui:id/clock", is_rich_hierarchy=True ) - assert result is not None - assert result["resource-id"] == "com.example:id/button1" + assert match is not None + assert match["resource-id"] == "com.android.systemui:id/clock" - result = find_element_by_resource_id( - rich_hierarchy, "com.example:id/nested_button", is_rich_hierarchy=True + nested_match = find_element_by_resource_id( + rich_nodes, "com.android.systemui:id/battery_meter", is_rich_hierarchy=True ) - assert result is not None - assert result["resource-id"] == "com.example:id/nested_button" + assert nested_match is not None + assert nested_match["resource-id"] == "com.android.systemui:id/battery_meter" - result = find_element_by_resource_id( - rich_hierarchy, "com.example:id/nonexistent", is_rich_hierarchy=True - ) - assert result is None + assert find_element_by_resource_id( + rich_nodes, "com.android.systemui:id/unknown", is_rich_hierarchy=True + ) is None def test_is_element_focused(): - focused_element = {"focused": "true"} - assert is_element_focused(focused_element) - - non_focused_element = {"focused": "false"} - assert not is_element_focused(non_focused_element) - - no_focused_element = {"text": "some text"} - assert not is_element_focused(no_focused_element) - - none_focused_element = {"focused": None} - assert not is_element_focused(none_focused_element) + assert is_element_focused({"focused": "true"}) + assert is_element_focused({"focused": True}) + assert not is_element_focused({"focused": "false"}) + assert not is_element_focused({"focused": False}) + assert not is_element_focused({"text": "sample"}) + assert not is_element_focused({"focused": None}) def test_get_element_text(): - element = {"text": "Button Text", "hintText": "Hint Text"} - assert get_element_text(element) == "Button Text" - assert get_element_text(element, hint_text=False) == "Button Text" - assert get_element_text(element, hint_text=True) == "Hint Text" + node = {"text": "Battery", "hintText": "Search settings"} + assert get_element_text(node) == "Battery" + assert get_element_text(node, hint_text=False) == "Battery" + assert get_element_text(node, hint_text=True) == "Search settings" - element_no_text = {"hintText": "Hint Text"} - assert get_element_text(element_no_text) is None - assert get_element_text(element_no_text, hint_text=True) == "Hint Text" - element_no_hint = {"text": "Button Text"} - assert get_element_text(element_no_hint) == "Button Text" - assert get_element_text(element_no_hint, hint_text=True) is None + node_no_text = {"hintText": "Placeholder"} + assert get_element_text(node_no_text) is None + assert get_element_text(node_no_text, hint_text=True) == "Placeholder" - empty_element = {} - assert get_element_text(empty_element) is None - assert get_element_text(empty_element, hint_text=True) is None + assert get_element_text({}) is None def test_get_bounds_for_element(): - element_with_bounds = {"bounds": {"x": 10, "y": 20, "width": 100, "height": 50}} - bounds = get_bounds_for_element(element_with_bounds) - assert bounds is not None - assert isinstance(bounds, ElementBounds) - assert bounds.x == 10 - assert bounds.y == 20 - assert bounds.width == 100 - assert bounds.height == 50 + node_with_box = {"bounds": {"x": 24, "y": 48, "width": 400, "height": 80}} + box = get_bounds_for_element(node_with_box) + assert box is not None + assert isinstance(box, ElementBounds) + assert box.x == 24 + assert box.y == 48 + assert box.width == 400 + assert box.height == 80 - element_no_bounds = {"text": "Button"} - bounds = get_bounds_for_element(element_no_bounds) - assert bounds is None + assert get_bounds_for_element({"text": "Click"}) is None - # Suppress logger output for the invalid bounds test case with patch("artemis.utils.ui_hierarchy.logger.error"): - element_invalid_bounds = { - "bounds": { - "x": "invalid", # Should be int - "y": 20, - "width": 100, - "height": 50, - } - } - bounds = get_bounds_for_element(element_invalid_bounds) - assert bounds is None + bad_box = {"bounds": {"x": "corrupted", "y": 48, "width": 400, "height": 80}} + assert get_bounds_for_element(bad_box) is None -def test_element_bounds(): - bounds = ElementBounds(x=10, y=20, width=100, height=50) +def test_element_bounds_geometry(): + box = ElementBounds(x=100, y=200, width=500, height=100) - center = bounds.get_center() + center = box.get_center() assert isinstance(center, Point) - assert center.x == 60 - assert center.y == 45 + assert center.x == 350 + assert center.y == 250 - center_point = bounds.get_relative_point(0.5, 0.5) - assert isinstance(center_point, Point) - assert center_point.x == 60 - assert center_point.y == 45 + mid = box.get_relative_point(0.5, 0.5) + assert mid.x == 350 + assert mid.y == 250 - top_left = bounds.get_relative_point(0.0, 0.0) - assert top_left.x == 10 - assert top_left.y == 20 + top_left = box.get_relative_point(0.0, 0.0) + assert top_left.x == 100 + assert top_left.y == 200 - bottom_right = bounds.get_relative_point(1.0, 1.0) - assert bottom_right.x == 110 - assert bottom_right.y == 70 - custom_point = bounds.get_relative_point(0.95, 0.95) - assert custom_point.x == 105 - assert custom_point.y == 67 - - -if __name__ == "__main__": - test_text_input_is_empty() - test_find_element_by_resource_id() - test_find_element_by_resource_id_with_index() - test_find_element_by_resource_id_rich_hierarchy() - test_is_element_focused() - test_get_element_text() - test_get_bounds_for_element() - test_element_bounds() - print("All tests passed") + bottom_right = box.get_relative_point(1.0, 1.0) + assert bottom_right.x == 600 + assert bottom_right.y == 300