refactor: clean up legacy code, rename hopper to entity_extractor, and add third-party notices

- Add NOTICE file and README license references for mobile-use and finalrun-agent
- Add derivative attribution headers to 17 files derived from mobile-use
- Rename hopper agent to entity_extractor across prompts, tools, and configs
- Remove obsolete adb_tunnel, ApiBaseUrl, and legacy CLI utility scripts
- Replace legacy examples with native Artemis diagnostic and workflow examples
- Update unit and tool tests to validate entity_extractor and SDK builders
This commit is contained in:
somew1nd committed 2026-09-11 16:19:42 -07:00
1 parent 0860788192
commit 54fcef9887
83 files changed
+1996 -3660

No files matched your search

+1 -1
View File
@@ -9,7 +9,7 @@ GEMINI_API_KEY=
GOOGLE_API_KEY= GOOGLE_API_KEY=
OPENAI_API_KEY= OPENAI_API_KEY=
ANTHROPIC_API_KEY= ANTHROPIC_API_KEY=
OPEN_ROUTER_API_KEY= OPENROUTER_API_KEY=
XAI_API_KEY= XAI_API_KEY=
# Google Cloud Vision OCR (Optional - for advanced OCR processing) # Google Cloud Vision OCR (Optional - for advanced OCR processing)
+27
View File
@@ -0,0 +1,27 @@
ARTEMIS Android Agent & Automation Platform
Copyright 2026 Google LLC
This product includes software developed by Google LLC.
================================================================================
Third-Party Software Attribution
================================================================================
Portions of this software are derived from and incorporate code originally
developed by the following open-source projects under the Apache License, Version 2.0:
1. mobile-use
Repository: https://github.com/minitap-ai/mobile-use
Copyright 2025-2026 Minitap, Inc.
Original authors & contributors:
Pierre-Louis Favreau, Jean-Pierre Lo, Nicolas Dehandschoewercker,
Clément Guiguet, Karun Agarwal, and the mobile-use open-source community.
License: Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
2. finalrun-agent
Repository: https://github.com/final-run/finalrun-agent
Copyright 2025 FinalRun Inc.
Original authors & contributors:
Ashish Yadav, Arnold Laishram, Srinidhi G S,
and the finalrun-agent open-source community.
License: Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
+4
View File
@@ -314,3 +314,7 @@ Contributions are warmly welcomed!
## License ## License
This project is licensed under the [Apache License 2.0](LICENSE). This project is licensed under the [Apache License 2.0](LICENSE).
This project includes source code developed by [Minitap, Inc.](https://github.com/minitap-ai/mobile-use).
+11
View File
@@ -312,3 +312,14 @@ ARTEMIS 提供两种运行模式以适应不同的自动化需求:
## 开源许可证 ## 开源许可证
本项目基于 [Apache License 2.0](LICENSE) 协议开源。 本项目基于 [Apache License 2.0](LICENSE) 协议开源。
## 致谢
本项目包含由 [Minitap, Inc.](https://github.com/minitap-ai/mobile-use)(`mobile-use`)团队开发的部分源代码与工具组件,遵循 Apache License 2.0 开源许可。我们感谢其团队及开源社区在移动端自动化领域的贡献。
同时感谢 [Finalrun](https://github.com/final-run/finalrun-agent) 在智能滑动与自适应手势机制理念上的启发。
关于完整的第三方开源软件版权与归属声明,请参阅根目录的 [NOTICE](NOTICE) 文件。
@@ -12,9 +12,14 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from datetime import datetime """Entity extractor and package resolver agent for ARTEMIS."""
from artemis.agents.entity_extractor.entity_extractor import (
ExtractionResult,
extract_entity,
)
def convert_timestamp_to_str(ts: float) -> str: __all__ = [
dt = datetime.fromtimestamp(ts) "ExtractionResult",
return dt.strftime("%Y-%m-%dT%H-%M-%S") "extract_entity",
]
@@ -0,0 +1,30 @@
# Artemis Entity Extractor & Package Resolver
You are an autonomous, high-precision structured data extractor in the ARTEMIS mobile testing runtime.
Your responsibility is to extract the exact requested identifier, package name, or configuration token from raw device dump outputs (such as `pm list packages`, `dumpsys`, logcat records, or accessibility tree dumps).
## Operating Directives
1. **Verbatim Extraction**:
- Return the extracted entity exactly as it appears in the source data.
- Do NOT modify capitalization, strip domain segments, translate, or normalize the matched identifier.
2. **Package Resolution Rules**:
- Match target application titles, codenames, or brand identifiers against standard Android package formats (`com.<vendor>.<app>`, `org.<project>.<module>`, etc.).
- Prioritize exact matches over partial or prefix matches.
3. **Strict Disambiguation & Null-Safety**:
- If the requested entity is absent from the input dataset, return `found: false` with `output: null`.
- If multiple candidates conflict and cannot be deterministically verified from the query, do NOT guess. Set `found: false` and `output: null`.
## Structured Deliverable Schema
You must produce your final answer conforming to the following JSON structure:
```json
{
"found": true,
"output": "<exact_matched_string_or_null>",
"reason": "<one_sentence_rationale_confirming_criteria>"
}
```
@@ -0,0 +1,90 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Lightweight entity extraction and package resolver for ARTEMIS.
Dispatches bounded LLM extraction turns to identify application package identifiers
and configuration attributes from raw device listings without manual parsing heuristics.
"""
from __future__ import annotations
from pathlib import Path
from typing import cast
from jinja2 import Template
from langchain_core.messages import HumanMessage, SystemMessage
from pydantic import BaseModel, Field
from artemis.context import ArtemisContext
from artemis.data_engine.trace import trace
from artemis.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
class ExtractionResult(BaseModel):
"""Structured extraction outcome for entity resolution."""
found: bool = Field(description="True if the requested entity was definitively identified.")
output: str | None = Field(default=None, description="Exact extracted identifier or string value.")
reason: str = Field(description="Brief justification of the match criteria applied.")
@trace(type="agent", name="entity_extractor")
async def extract_entity(
ctx: ArtemisContext,
request: str,
data: str,
use_fallback: bool = True,
) -> ExtractionResult:
"""Extract a specific identifier or entity from raw batch device data."""
logger.info(f"Invoking Artemis Entity Extractor (fallback={use_fallback})")
template_path = Path(__file__).parent / "entity_extractor.md"
system_prompt = Template(template_path.read_text(encoding="utf-8")).render()
messages = [
SystemMessage(content=system_prompt),
HumanMessage(content=f"Target Query:\n{request}\n\nCandidate Raw Dataset:\n{data}"),
]
primary_llm = get_llm(ctx=ctx, name="entity_extractor", is_utils=True).with_structured_output(
ExtractionResult
)
try:
if use_fallback:
fallback_llm = get_llm(
ctx=ctx, name="entity_extractor", is_utils=True, use_fallback=True
).with_structured_output(ExtractionResult)
raw = await with_fallback(
main_call=lambda: invoke_llm_with_timeout_message(primary_llm.ainvoke(messages)),
fallback_call=lambda: invoke_llm_with_timeout_message(fallback_llm.ainvoke(messages)),
)
else:
raw = await invoke_llm_with_timeout_message(primary_llm.ainvoke(messages))
if isinstance(raw, ExtractionResult):
return raw
if isinstance(raw, dict):
return ExtractionResult.model_validate(raw)
return cast(ExtractionResult, raw)
except Exception as exc:
logger.error(f"Entity extraction turn failed: {exc}")
return ExtractionResult(
found=False,
output=None,
reason=f"LLM extraction error: {str(exc)[:400]}",
)
-15
View File
@@ -1,15 +0,0 @@
## Hopper
Extract relevant information from batch data. **Keep extracted data exactly as-is** - no reformatting.
## Output
- **found**: `true` if data was found, `false` otherwise
- **output**: Extracted information if found, `null` otherwise
- **reason**: Brief explanation of search logic
## Rules
1. **Search entire input** - may contain hundreds of entries
2. **For app package lookup**: Match app name (or variations) in package identifier
- Common patterns: lowercase app name, company+app, brand name, codenames
3. **Prefer direct matches** over partial matches
4. **Return `null`** if not found or if multiple ambiguous matches exist - don't guess
-93
View File
@@ -1,93 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from jinja2 import Template
from langchain_core.messages import HumanMessage, SystemMessage
from artemis.context import ArtemisContext
from artemis.data_engine.trace import trace
from artemis.services.llm import get_llm, invoke_llm_with_timeout_message, with_fallback
from artemis.utils.logger import get_logger
from pydantic import BaseModel, Field
logger = get_logger(__name__)
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class HopperOutput(BaseModel):
model_config = {"ignored_types": (CyFunctionDetector,)}
found: bool = Field(description="True if the requested data was found, False otherwise.")
output: str | None = Field(description="The extracted data if found, null otherwise.")
reason: str = Field(
description="A short explanation of what you looked for"
+ " and how you decided what to extract."
)
@trace(type="agent", name="hopper")
async def hopper(
ctx: ArtemisContext,
request: str,
data: str,
use_fallback: bool = True,
) -> HopperOutput:
logger.info(f"Starting Hopper Agent (use_fallback={use_fallback})")
system_message = Template(
Path(__file__).parent.joinpath("hopper.md").read_text(encoding="utf-8")
).render()
messages = [
SystemMessage(content=system_message),
HumanMessage(content=f"{request}\nHere is the data you must dig:\n{data}"),
]
llm = get_llm(ctx=ctx, name="hopper", is_utils=True).with_structured_output(HopperOutput)
try:
if use_fallback:
llm_fallback = get_llm(
ctx=ctx, name="hopper", is_utils=True, use_fallback=True
).with_structured_output(HopperOutput)
response: HopperOutput = await with_fallback(
main_call=lambda: invoke_llm_with_timeout_message(llm.ainvoke(messages)),
fallback_call=lambda: invoke_llm_with_timeout_message(
llm_fallback.ainvoke(messages)
),
) # type: ignore
else:
response: HopperOutput = await invoke_llm_with_timeout_message(llm.ainvoke(messages))
return response
except Exception as e:
logger.error(f"Hopper LLM invocation failed: {e}")
return HopperOutput(
found=False,
output=None,
reason=f"Failed due to LLM error: {str(e)[:500]}",
)
@@ -21,26 +21,7 @@ from langchain_core.tools import BaseTool, StructuredTool
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class SpawnLogReaderArgs(BaseModel): class SpawnLogReaderArgs(BaseModel):
model_config = {"ignored_types": (CyFunctionDetector,)}
specific_query: str = Field( specific_query: str = Field(
..., ...,
description=( description=(
@@ -54,7 +35,6 @@ from langgraph.prebuilt import InjectedState
from artemis.core.tool_failure import ToolFailure, is_tool_failure from artemis.core.tool_failure import ToolFailure, is_tool_failure
from artemis.context import ArtemisContext from artemis.context import ArtemisContext
from artemis.data_engine.trace import ( from artemis.data_engine.trace import (
CURRENT_TRACE_ID,
TraceSpan, TraceSpan,
trace, trace,
) )
+3
View File
@@ -11,6 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
import json import json
from pathlib import Path from pathlib import Path
+1 -20
View File
@@ -23,7 +23,7 @@ from pydantic import BaseModel, Field
from artemis.context import ArtemisContext from artemis.context import ArtemisContext
from artemis.controllers.unified_controller import UnifiedMobileController from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.data_engine.trace import CURRENT_TRACE_ID, TraceSpan, trace from artemis.data_engine.trace import TraceSpan, trace
from artemis.graph.state import State from artemis.graph.state import State
from artemis.graph.visibility import strict_state from artemis.graph.visibility import strict_state
from artemis.services.llm import ( from artemis.services.llm import (
@@ -108,26 +108,7 @@ def build_planner_system_blocks(prompts_data: dict, mode: str, include_checks: b
return blocks return blocks
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class ValidationResult(BaseModel): class ValidationResult(BaseModel):
model_config = {"ignored_types": (CyFunctionDetector,)}
is_approved: bool = Field( is_approved: bool = Field(
description=( description=(
"True if the new plan still serves the initial goal and is coherent." "True if the new plan still serves the initial goal and is coherent."
-327
View File
@@ -1,327 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""ADB WebSocket tunnel implementation.
Bridges local TCP connections to a remote ADB server via WebSocket.
This replaces the need for the `lim` CLI for ADB tunneling.
The tunnel runs in a dedicated thread with its own event loop to ensure
it cannot be blocked by synchronous operations in the main thread.
Based on the Go SDK implementation which:
1. Creates a TCP listener
2. Accepts TCP connections from ADB
3. Dials the WebSocket
4. Forwards traffic bidirectionally
"""
import asyncio
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
import socket
import threading
from artemis.utils.logger import get_logger
import websockets
from websockets.asyncio.client import ClientConnection
logger = get_logger(__name__)
BUFFER_SIZE = 32 * 1024 # 32KB, same as Go SDK
PING_INTERVAL = 30 # seconds
class AdbTunnel:
"""ADB tunnel that bridges local TCP connections to a remote WebSocket endpoint.
Runs in a dedicated thread with its own event loop to ensure the tunnel
cannot be blocked by synchronous operations in the caller's thread.
Usage:
tunnel = AdbTunnel(remote_url, token)
addr = await tunnel.start() # Returns "127.0.0.1:PORT"
# Use addr with ADB: adb connect 127.0.0.1:PORT
await tunnel.stop()
"""
def __init__(self, remote_url: str, token: str):
"""Initialize the ADB tunnel.
Args:
remote_url: WebSocket URL for the remote ADB endpoint
token: Bearer token for authentication
"""
self.remote_url = remote_url
self.token = token
self._local_port: int | None = None
self._thread: threading.Thread | None = None
self._loop: asyncio.AbstractEventLoop | None = None
self._started = threading.Event()
self._stop_event: asyncio.Event | None = None
@property
def local_addr(self) -> str:
"""Get the local address to connect ADB to."""
if self._local_port is None:
raise RuntimeError("Tunnel not started")
return f"127.0.0.1:{self._local_port}"
@property
def local_port(self) -> int:
"""Get the local port number."""
if self._local_port is None:
raise RuntimeError("Tunnel not started")
return self._local_port
async def start(self) -> str:
"""Start the ADB tunnel in a dedicated thread.
Returns:
The local address (host:port) to connect ADB to.
"""
if self._thread is not None and self._thread.is_alive():
return self.local_addr
# Create listener in main thread to get the port immediately
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.bind(("127.0.0.1", 0))
listener.listen(5)
listener.setblocking(False)
self._local_port = listener.getsockname()[1]
logger.info(f"ADB tunnel listening on {self.local_addr}")
# Start tunnel thread
self._thread = threading.Thread(
target=self._run_in_thread,
args=(listener,),
name=f"adb-tunnel-{self._local_port}",
daemon=True,
)
self._thread.start()
# Wait for thread to initialize its event loop
self._started.wait(timeout=5.0)
if not self._started.is_set():
await self.stop()
raise RuntimeError("Tunnel thread failed to start")
return self.local_addr
async def stop(self) -> None:
"""Stop the ADB tunnel and cleanup resources."""
if self._loop and self._stop_event:
# Signal the tunnel loop to stop
self._loop.call_soon_threadsafe(self._stop_event.set)
if self._thread and self._thread.is_alive():
await asyncio.to_thread(self._thread.join, 5.0)
if self._thread.is_alive():
logger.warning("ADB tunnel thread did not stop within 5s")
self._thread = None
self._loop = None
self._stop_event = None
logger.info("ADB tunnel stopped")
def _run_in_thread(self, listener: socket.socket) -> None:
"""Run the tunnel event loop in a dedicated thread."""
try:
self._loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._loop)
self._stop_event = asyncio.Event()
self._started.set()
self._loop.run_until_complete(self._run_tunnel(listener))
except Exception as e:
logger.error(f"Tunnel thread error: {e}")
finally:
listener.close()
if self._loop:
self._loop.close()
async def _run_tunnel(self, listener: socket.socket) -> None:
"""Run the tunnel - accept TCP connections and bridge to WebSocket."""
loop = asyncio.get_event_loop()
connection_tasks: list[asyncio.Task] = []
stop_event = self._stop_event
assert stop_event is not None
logger.debug("Tunnel: ready to accept connections")
while not stop_event.is_set():
try:
# Use wait_for to periodically check stop_event
try:
tcp_conn, addr = await asyncio.wait_for(
loop.sock_accept(listener),
timeout=1.0,
)
except TimeoutError:
# Check if we should stop
continue
logger.debug(f"Tunnel: new TCP connection from {addr}")
task = asyncio.create_task(
self._handle_connection(tcp_conn, loop),
name=f"conn-{addr[1]}",
)
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
connection_tasks.append(task)
connection_tasks = [t for t in connection_tasks if not t.done()]
except Exception as e:
if not stop_event.is_set():
logger.error(f"Tunnel accept error: {e}")
break
# Cleanup pending connections
for task in connection_tasks:
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
async def _handle_connection(
self,
tcp_conn: socket.socket,
loop: asyncio.AbstractEventLoop,
) -> None:
"""Handle a single TCP connection."""
tcp_conn.setblocking(False)
ws: ClientConnection | None = None
try:
logger.debug(f"Connecting to WebSocket: {self.remote_url[:80]}...")
ws = await websockets.connect(
self.remote_url,
additional_headers={"Authorization": f"Bearer {self.token}"},
ping_interval=PING_INTERVAL,
ping_timeout=10,
)
logger.debug("WebSocket connected to remote ADB")
await self._bridge(tcp_conn, ws, loop)
except Exception as e:
logger.error(f"Connection error: {e}")
finally:
if ws is not None:
await ws.close()
tcp_conn.close()
logger.debug("Connection closed")
async def _bridge(
self,
tcp_conn: socket.socket,
ws: ClientConnection,
loop: asyncio.AbstractEventLoop,
) -> None:
"""Bridge TCP and WebSocket bidirectionally."""
stop_event = self._stop_event
assert stop_event is not None
async def tcp_to_ws() -> None:
"""Forward data from TCP to WebSocket."""
msg_count = 0
try:
while not stop_event.is_set():
try:
data = await asyncio.wait_for(
loop.sock_recv(tcp_conn, BUFFER_SIZE),
timeout=1.0,
)
except TimeoutError:
continue
except (OSError, ConnectionError) as e:
logger.debug(f"tcp->ws: recv error after {msg_count} msgs: {e}")
break
if not data:
logger.debug(f"tcp->ws: closed by client after {msg_count} msgs")
break
msg_count += 1
logger.debug(f"tcp->ws: [{msg_count}] sending {len(data)} bytes")
await ws.send(data)
logger.debug(f"tcp->ws: loop ended after {msg_count} messages")
except asyncio.CancelledError:
logger.debug(f"tcp->ws: cancelled after {msg_count} messages")
raise
except Exception as e:
logger.error(f"tcp->ws: error after {msg_count} messages: {e}")
async def ws_to_tcp() -> None:
"""Forward data from WebSocket to TCP."""
msg_count = 0
try:
logger.debug("ws->tcp: starting message loop")
async for message in ws:
if stop_event.is_set():
break
if isinstance(message, bytes) and message:
msg_count += 1
logger.debug(f"ws->tcp: [{msg_count}] received {len(message)} bytes")
await loop.sock_sendall(tcp_conn, message)
elif isinstance(message, str):
logger.debug(f"ws->tcp: received string: {message[:100]}")
logger.debug(f"ws->tcp: loop ended after {msg_count} messages")
except asyncio.CancelledError:
logger.debug(f"ws->tcp: cancelled after {msg_count} messages")
raise
except Exception as e:
logger.error(f"ws->tcp: error after {msg_count} messages: {e}")
tcp_task = asyncio.create_task(tcp_to_ws())
ws_task = asyncio.create_task(ws_to_tcp())
done, pending = await asyncio.wait(
[tcp_task, ws_task],
return_when=asyncio.FIRST_COMPLETED,
)
for task in pending:
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
@asynccontextmanager
async def adb_tunnel(remote_url: str, token: str) -> AsyncGenerator[AdbTunnel, None]:
"""Context manager for creating and managing an ADB tunnel.
Args:
remote_url: WebSocket URL for the remote ADB endpoint
token: Bearer token for authentication
Yields:
AdbTunnel instance with the tunnel running
Example:
async with adb_tunnel(ws_url, token) as tunnel:
# Connect ADB to tunnel.local_addr
adb_client = AdbClient()
adb_client.connect(tunnel.local_addr)
"""
tunnel = AdbTunnel(remote_url, token)
try:
await tunnel.start()
yield tunnel
finally:
await tunnel.stop()
+7 -109
View File
@@ -11,12 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
"""UIAutomator2 client for Android device screen data retrieval. """UIAutomator2 client for Android device screen data retrieval.
Provides an alternative to the Maestro-based screen API with direct device Provides direct device access for hierarchy and screenshot retrieval.
access.
Handles Maestro blocker detection and removal before connecting.
""" """
import base64 import base64
@@ -41,32 +42,9 @@ if TYPE_CHECKING:
logger = get_logger(__name__) logger = get_logger(__name__)
MAESTRO_PACKAGE = "dev.mobile.maestro"
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class UIAutomatorScreenData(BaseModel): class UIAutomatorScreenData(BaseModel):
"""Screen data response from UIAutomator2.""" """Screen data response from UIAutomator2."""
model_config = {"ignored_types": (CyFunctionDetector,)}
base64: str base64: str
hierarchy_xml: str hierarchy_xml: str
elements: list[dict] elements: list[dict]
@@ -170,88 +148,11 @@ def _pil_to_base64(img: Image.Image, format: str = "JPEG", quality: int = 80) ->
return base64.b64encode(buffer.getvalue()).decode("utf-8") return base64.b64encode(buffer.getvalue()).decode("utf-8")
def _is_package_installed(device_id: str, pkg: str) -> bool:
"""Check if a package is installed on the device."""
try:
result = subprocess.run(
adb_command(["-s", device_id, "shell", "pm", "list", "packages"]),
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=10,
)
if result.returncode != 0:
logger.warning(f"Failed to list packages: {result.stderr}")
return False
lines = result.stdout.splitlines()
target = f"package:{pkg}"
return target in lines
except subprocess.TimeoutExpired:
logger.warning("Timeout checking installed packages")
return False
except (OSError, subprocess.SubprocessError) as e:
logger.warning(f"Error checking installed packages: {e}")
return False
def _uninstall_package(device_id: str, pkg: str) -> bool:
"""Uninstall a package from the device for the current user."""
try:
result = subprocess.run(
adb_command(
[
"-s",
device_id,
"shell",
"pm",
"uninstall",
"--user",
"0",
pkg,
]
),
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=30,
)
if result.returncode == 0:
logger.info(f"Successfully uninstalled {pkg}")
return True
else:
logger.warning(f"Failed to uninstall {pkg}: {result.stderr}")
return False
except subprocess.TimeoutExpired:
logger.warning(f"Timeout uninstalling {pkg}")
return False
except Exception as e:
logger.warning(f"Error uninstalling {pkg}: {e}")
return False
def _ensure_maestro_not_installed(device_id: str) -> None:
"""Check if Maestro is installed and uninstall it.
Maestro conflicts with uiautomator2 - if Maestro's package is installed,
u2.connect() will fail. This function ensures Maestro is removed before
attempting to connect.
"""
if _is_package_installed(device_id, MAESTRO_PACKAGE):
logger.warning(
f"Maestro ({MAESTRO_PACKAGE}) detected - uninstalling to enable UIAutomator2..."
)
_uninstall_package(device_id, MAESTRO_PACKAGE)
class UIAutomatorClient: class UIAutomatorClient:
"""UIAutomator2 client for Android screen data retrieval. """UIAutomator2 client for Android screen data retrieval.
This client uses uiautomator2 library for direct device communication, This client uses the uiautomator2 library for direct device communication,
providing faster hierarchy and screenshot retrieval compared to Maestro. providing high-performance hierarchy and screenshot retrieval.
Important: Maestro must not be installed on the device as it conflicts
with uiautomator2. This client automatically handles Maestro removal.
""" """
def __init__(self, device_id: str): def __init__(self, device_id: str):
@@ -265,7 +166,7 @@ class UIAutomatorClient:
self._awake_strategy: str | None = None self._awake_strategy: str | None = None
def _ensure_connected(self) -> "Device": def _ensure_connected(self) -> "Device":
"""Ensure connection to the device, handling Maestro blocker. """Ensure connection to the device.
Returns: Returns:
Connected uiautomator2 Device instance Connected uiautomator2 Device instance
@@ -279,9 +180,6 @@ class UIAutomatorClient:
logger.warning("UIAutomator2 connection lost, reconnecting...") logger.warning("UIAutomator2 connection lost, reconnecting...")
self._device = None self._device = None
# Ensure Maestro is not blocking us
_ensure_maestro_not_installed(self._device_id)
# Enroll the device in the containing Artemis service lifetime before # Enroll the device in the containing Artemis service lifetime before
# UIAutomator2 reads the screen. Client disconnects do not release it. # UIAutomator2 reads the screen. Client disconnects do not release it.
if self._awake_strategy is None: if self._awake_strategy is None:
+2 -2
View File
@@ -83,6 +83,7 @@ from artemis.config.constants import (
ENV_ARTEMIS_TRACES_DIR, ENV_ARTEMIS_TRACES_DIR,
ENV_ARTEMIS_USE_FILE_API, ENV_ARTEMIS_USE_FILE_API,
ENV_ARTEMIS_USE_USER_DIR, ENV_ARTEMIS_USE_USER_DIR,
ENV_OPENROUTER_API_KEY,
ENV_OPEN_ROUTER_API_KEY, ENV_OPEN_ROUTER_API_KEY,
ENV_OPENAI_API_KEY, ENV_OPENAI_API_KEY,
ENV_OPENAI_BASE_URL, ENV_OPENAI_BASE_URL,
@@ -100,7 +101,6 @@ from artemis.config.constants import (
LLMUtilsNodeWithFallback, LLMUtilsNodeWithFallback,
) )
from artemis.config.llm import ( from artemis.config.llm import (
CyFunctionDetector,
LLM, LLM,
LLMConfig, LLMConfig,
LLMConfigUtils, LLMConfigUtils,
@@ -196,7 +196,6 @@ __all__ = [
"get_ls_address_file", "get_ls_address_file",
"is_frozen_bundle", "is_frozen_bundle",
# LLM Config & Models # LLM Config & Models
"CyFunctionDetector",
"LLM", "LLM",
"LLMWithFallback", "LLMWithFallback",
"LLMConfigUtils", "LLMConfigUtils",
@@ -296,6 +295,7 @@ __all__ = [
"ENV_OPENAI_API_KEY", "ENV_OPENAI_API_KEY",
"ENV_OPENAI_BASE_URL", "ENV_OPENAI_BASE_URL",
"ENV_ANTHROPIC_API_KEY", "ENV_ANTHROPIC_API_KEY",
"ENV_OPENROUTER_API_KEY",
"ENV_OPEN_ROUTER_API_KEY", "ENV_OPEN_ROUTER_API_KEY",
"ENV_XAI_API_KEY", "ENV_XAI_API_KEY",
"ENV_ARTEMIS_EXPLORER_VERSION", "ENV_ARTEMIS_EXPLORER_VERSION",
+2 -1
View File
@@ -62,6 +62,7 @@ ENV_API_KEY = "API_KEY"
ENV_OPENAI_API_KEY = "OPENAI_API_KEY" ENV_OPENAI_API_KEY = "OPENAI_API_KEY"
ENV_OPENAI_BASE_URL = "OPENAI_BASE_URL" ENV_OPENAI_BASE_URL = "OPENAI_BASE_URL"
ENV_ANTHROPIC_API_KEY = "ANTHROPIC_API_KEY" ENV_ANTHROPIC_API_KEY = "ANTHROPIC_API_KEY"
ENV_OPENROUTER_API_KEY = "OPENROUTER_API_KEY"
ENV_OPEN_ROUTER_API_KEY = "OPEN_ROUTER_API_KEY" ENV_OPEN_ROUTER_API_KEY = "OPEN_ROUTER_API_KEY"
ENV_XAI_API_KEY = "XAI_API_KEY" ENV_XAI_API_KEY = "XAI_API_KEY"
@@ -125,7 +126,7 @@ ExplorerVersion = Literal["flash", "pro", "ultra"]
LLMUtilsNode = Literal[ LLMUtilsNode = Literal[
"outputter", "outputter",
"hopper", "entity_extractor",
"video_analyzer", "video_analyzer",
"object_detector", "object_detector",
] ]
+7 -30
View File
@@ -48,30 +48,11 @@ def validate_vertex_ai_credentials() -> None:
) )
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
"""Utility type detector for Cython and C-extension functions in Pydantic."""
pass pass
class LLM(BaseModel): class LLM(BaseModel):
"""Base model representing an LLM model provider and runtime parameters.""" """Base model representing an LLM model provider and runtime parameters."""
model_config = {"ignored_types": (CyFunctionDetector,)}
provider: LLMProvider provider: LLMProvider
model: str model: str
temperature: float | None = None temperature: float | None = None
@@ -95,8 +76,8 @@ class LLM(BaseModel):
if not (settings.ANTHROPIC_API_KEY or os.environ.get("ANTHROPIC_API_KEY")): if not (settings.ANTHROPIC_API_KEY or os.environ.get("ANTHROPIC_API_KEY")):
raise Exception(f"{name} requires ANTHROPIC_API_KEY in .env") raise Exception(f"{name} requires ANTHROPIC_API_KEY in .env")
elif self.provider == "openrouter": elif self.provider == "openrouter":
if not settings.OPEN_ROUTER_API_KEY: if not (settings.OPENROUTER_API_KEY or settings.OPEN_ROUTER_API_KEY):
raise Exception(f"{name} requires OPEN_ROUTER_API_KEY in .env") raise Exception(f"{name} requires OPENROUTER_API_KEY in .env")
elif self.provider == "xai": elif self.provider == "xai":
if not settings.XAI_API_KEY: if not settings.XAI_API_KEY:
raise Exception(f"{name} requires XAI_API_KEY in .env") raise Exception(f"{name} requires XAI_API_KEY in .env")
@@ -133,18 +114,14 @@ def lightweight_judge_default() -> "LLMWithFallback":
class LLMConfigUtils(BaseModel): class LLMConfigUtils(BaseModel):
"""Configuration container for auxiliary utility agents/nodes.""" """Configuration container for auxiliary utility agents/nodes."""
model_config = {"ignored_types": (CyFunctionDetector,)}
outputter: LLMWithFallback outputter: LLMWithFallback
hopper: LLMWithFallback entity_extractor: LLMWithFallback
video_analyzer: LLMWithFallback | None = None video_analyzer: LLMWithFallback | None = None
object_detector: LLMWithFallback | None = None object_detector: LLMWithFallback | None = None
class LLMConfig(BaseModel): class LLMConfig(BaseModel):
"""Comprehensive LLM configuration mapping every node to primary/fallback models.""" """Comprehensive LLM configuration mapping every node to primary/fallback models."""
model_config = {"ignored_types": (CyFunctionDetector,)}
planner: LLMWithFallback planner: LLMWithFallback
utils: LLMConfigUtils utils: LLMConfigUtils
summarizer: LLMWithFallback summarizer: LLMWithFallback
@@ -167,7 +144,7 @@ class LLMConfig(BaseModel):
"""Validate credentials across all configured agent nodes.""" """Validate credentials across all configured agent nodes."""
self.planner.validate_provider("Planner") self.planner.validate_provider("Planner")
self.utils.outputter.validate_provider("Outputter") self.utils.outputter.validate_provider("Outputter")
self.utils.hopper.validate_provider("Hopper") self.utils.entity_extractor.validate_provider("EntityExtractor")
if self.utils.video_analyzer: if self.utils.video_analyzer:
self.utils.video_analyzer.validate_provider("VideoAnalyzer") self.utils.video_analyzer.validate_provider("VideoAnalyzer")
if self.utils.object_detector: if self.utils.object_detector:
@@ -196,7 +173,7 @@ class LLMConfig(BaseModel):
return f""" return f"""
📃 Planner: {self.planner} 📃 Planner: {self.planner}
🧩 Utils: 🧩 Utils:
🔽 Hopper: {self.utils.hopper} 🔍 Entity Extractor: {self.utils.entity_extractor}
📝 Outputter: {self.utils.outputter} 📝 Outputter: {self.utils.outputter}
🎬 Video Analyzer: {self.utils.video_analyzer or "Not configured"} 🎬 Video Analyzer: {self.utils.video_analyzer or "Not configured"}
👁️ Object Detector: {self.utils.object_detector or "Not configured"} 👁️ Object Detector: {self.utils.object_detector or "Not configured"}
@@ -219,7 +196,7 @@ class LLMConfig(BaseModel):
def get_utils(self, item: LLMUtilsNode) -> LLMWithFallback: def get_utils(self, item: LLMUtilsNode) -> LLMWithFallback:
"""Retrieve model configuration for a specific utility node.""" """Retrieve model configuration for a specific utility node."""
value = getattr(self.utils, item) value = getattr(self.utils, item, None)
if value is None: if value is None:
raise ValueError( raise ValueError(
f"Utils '{item}' is not configured. Please add it to your LLM " f"Utils '{item}' is not configured. Please add it to your LLM "
@@ -264,7 +241,7 @@ def _expand_default_into_nodes(config_dict: dict) -> dict:
all_utils_nodes = [ all_utils_nodes = [
"outputter", "outputter",
"hopper", "entity_extractor",
"video_analyzer", "video_analyzer",
"object_detector", "object_detector",
] ]
-3
View File
@@ -26,7 +26,6 @@ from artemis.config.constants import (
ENV_EVENTS_OUTPUT_PATH, ENV_EVENTS_OUTPUT_PATH,
ENV_RESULTS_OUTPUT_PATH, ENV_RESULTS_OUTPUT_PATH,
) )
from artemis.config.llm import CyFunctionDetector
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
logger = get_logger(__name__) logger = get_logger(__name__)
@@ -34,8 +33,6 @@ logger = get_logger(__name__)
class OutputConfig(BaseModel): class OutputConfig(BaseModel):
"""Configuration for structured output formatting and schemas.""" """Configuration for structured output formatting and schemas."""
model_config = {"ignored_types": (CyFunctionDetector,)}
structured_output: Annotated[ structured_output: Annotated[
type[BaseModel] | dict | None, type[BaseModel] | dict | None,
Field( Field(
+19 -5
View File
@@ -18,7 +18,7 @@ import os
from pathlib import Path from pathlib import Path
from dotenv import load_dotenv from dotenv import load_dotenv
from pydantic import Field, SecretStr, model_validator from pydantic import AliasChoices, Field, SecretStr, model_validator
from pydantic_settings import BaseSettings from pydantic_settings import BaseSettings
from artemis.config.constants import ( from artemis.config.constants import (
@@ -32,6 +32,7 @@ from artemis.config.constants import (
ENV_GEMINI_API_KEY, ENV_GEMINI_API_KEY,
ENV_GOOGLE_API_KEY, ENV_GOOGLE_API_KEY,
ENV_OCR_API_KEY, ENV_OCR_API_KEY,
ENV_OPENROUTER_API_KEY,
ENV_OPEN_ROUTER_API_KEY, ENV_OPEN_ROUTER_API_KEY,
ENV_OPENAI_API_KEY, ENV_OPENAI_API_KEY,
ENV_VISION_API_KEY, ENV_VISION_API_KEY,
@@ -99,6 +100,10 @@ class Settings(BaseSettings):
GCP_API_KEY: SecretStr | None = None GCP_API_KEY: SecretStr | None = None
ANTHROPIC_API_KEY: SecretStr | None = None ANTHROPIC_API_KEY: SecretStr | None = None
XAI_API_KEY: SecretStr | None = None XAI_API_KEY: SecretStr | None = None
OPENROUTER_API_KEY: SecretStr | None = Field(
default=None,
validation_alias=AliasChoices("OPENROUTER_API_KEY", "OPEN_ROUTER_API_KEY"),
)
OPEN_ROUTER_API_KEY: SecretStr | None = None OPEN_ROUTER_API_KEY: SecretStr | None = None
# Google Cloud Vision OCR Authentication # Google Cloud Vision OCR Authentication
@@ -163,6 +168,7 @@ class Settings(BaseSettings):
"GCP_API_KEY", "GCP_API_KEY",
"ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY",
"XAI_API_KEY", "XAI_API_KEY",
"OPENROUTER_API_KEY",
"OPEN_ROUTER_API_KEY", "OPEN_ROUTER_API_KEY",
"OCR_API_KEY", "OCR_API_KEY",
"VISION_API_KEY", "VISION_API_KEY",
@@ -172,6 +178,12 @@ class Settings(BaseSettings):
if val and is_placeholder_key(val): if val and is_placeholder_key(val):
setattr(self, attr, None) setattr(self, attr, None)
# Synchronize OpenRouter keys
if not self.OPENROUTER_API_KEY and self.OPEN_ROUTER_API_KEY:
self.OPENROUTER_API_KEY = self.OPEN_ROUTER_API_KEY
elif not self.OPEN_ROUTER_API_KEY and self.OPENROUTER_API_KEY:
self.OPEN_ROUTER_API_KEY = self.OPENROUTER_API_KEY
if not self.GOOGLE_API_KEY: if not self.GOOGLE_API_KEY:
if self.GEMINI_API_KEY: if self.GEMINI_API_KEY:
self.GOOGLE_API_KEY = self.GEMINI_API_KEY self.GOOGLE_API_KEY = self.GEMINI_API_KEY
@@ -204,8 +216,8 @@ class Settings(BaseSettings):
key = self.OPENAI_API_KEY key = self.OPENAI_API_KEY
elif provider_lower in ("anthropic", "claude"): elif provider_lower in ("anthropic", "claude"):
key = self.ANTHROPIC_API_KEY key = self.ANTHROPIC_API_KEY
elif provider_lower == "openrouter": elif provider_lower in ("openrouter", "open_router"):
key = self.OPEN_ROUTER_API_KEY key = self.OPENROUTER_API_KEY or self.OPEN_ROUTER_API_KEY
elif provider_lower in ("xai", "grok"): elif provider_lower in ("xai", "grok"):
key = self.XAI_API_KEY key = self.XAI_API_KEY
@@ -243,9 +255,11 @@ class Settings(BaseSettings):
self.ANTHROPIC_API_KEY = secret self.ANTHROPIC_API_KEY = secret
env_key_name = ENV_ANTHROPIC_API_KEY env_key_name = ENV_ANTHROPIC_API_KEY
os.environ[ENV_ANTHROPIC_API_KEY] = key os.environ[ENV_ANTHROPIC_API_KEY] = key
elif provider_lower == "openrouter": elif provider_lower in ("openrouter", "open_router"):
self.OPENROUTER_API_KEY = secret
self.OPEN_ROUTER_API_KEY = secret self.OPEN_ROUTER_API_KEY = secret
env_key_name = ENV_OPEN_ROUTER_API_KEY env_key_name = ENV_OPENROUTER_API_KEY
os.environ[ENV_OPENROUTER_API_KEY] = key
os.environ[ENV_OPEN_ROUTER_API_KEY] = key os.environ[ENV_OPEN_ROUTER_API_KEY] = key
elif provider_lower == "xai": elif provider_lower == "xai":
self.XAI_API_KEY = secret self.XAI_API_KEY = secret
+3
View File
@@ -11,6 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
"""Context variables for global state management. """Context variables for global state management.
+70 -142
View File
@@ -11,45 +11,42 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
"""Abstract device controller contract and telemetry interfaces for ARTEMIS.
Defines the hardware interaction lifecycle including touch gestures, text entry,
process lifecycle operations, display capture, and multi-segment recording.
"""
from __future__ import annotations
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import Any from typing import Any
from pydantic import BaseModel, ConfigDict, Field
from artemis.controllers.types import Bounds, CoordinatesSelectorRequest, TapOutput from artemis.controllers.types import Bounds, CoordinatesSelectorRequest, TapOutput
from artemis.utils.video import VideoRecordingResult from artemis.utils.video import VideoRecordingResult
from pydantic import BaseModel, Field
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class ScreenDataResponse(BaseModel): class ScreenDataResponse(BaseModel):
model_config = {"ignored_types": (CyFunctionDetector,)} """Aggregate snapshot containing live display frames and parsed node hierarchies."""
base64: str = Field(default="", description="Base64 encoded screenshot string")
elements: list[dict[str, Any]] = Field( model_config = ConfigDict(extra="ignore")
default_factory=list, description="Parsed UI hierarchy elements"
) base64: str = Field(default="", description="Base64 encoded JPEG/PNG frame capture.")
width: int = Field(default=1080, description="Device screen width in pixels") elements: list[dict[str, Any]] = Field(default_factory=list, description="Parsed hierarchy node elements.")
height: int = Field(default=2400, description="Device screen height in pixels") width: int = Field(default=1080, description="Active viewport width in physical pixels.")
platform: str = Field(default="android", description="Platform identifier") height: int = Field(default=2400, description="Active viewport height in physical pixels.")
platform: str = Field(default="android", description="Target platform identifier.")
class MobileDeviceController(ABC): class MobileDeviceController(ABC):
"""Abstract driver interface establishing device actuation and telemetry operations."""
@abstractmethod @abstractmethod
async def tap( async def tap(
self, self,
@@ -59,7 +56,8 @@ class MobileDeviceController(ABC):
times: int = 1, times: int = 1,
delay_ms: int = 100, delay_ms: int = 100,
) -> TapOutput: ) -> TapOutput:
raise NotImplementedError("Subclasses must implement this method") """Dispatch single, repeated, or long-press tap events to the target coordinates."""
raise NotImplementedError
@abstractmethod @abstractmethod
async def swipe( async def swipe(
@@ -68,161 +66,102 @@ class MobileDeviceController(ABC):
end: CoordinatesSelectorRequest, end: CoordinatesSelectorRequest,
duration: int = 400, duration: int = 400,
) -> str | None: ) -> str | None:
"""Swipe from start to end coordinates. """Execute directional drag or fling between two coordinate positions."""
raise NotImplementedError
Returns error message on failure, None on success.
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
async def screenshot(self) -> str: async def screenshot(self) -> str:
"""Take a screenshot and return raw image data.""" """Capture current display frame as an encoded image payload."""
raise NotImplementedError("Subclasses must implement this method") raise NotImplementedError
@abstractmethod @abstractmethod
async def input_text(self, text: str) -> bool: async def input_text(self, text: str) -> bool:
"""Input text at the currently focused field. """Stream textual characters into the currently active input field."""
raise NotImplementedError
Returns True on success, False on failure.
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
async def launch_app(self, package_or_bundle_id: str) -> bool: async def launch_app(self, package_name: str) -> bool:
"""Launch an application by package name (Android). """Bring target Android application package into foreground."""
raise NotImplementedError
Returns True on success, False on failure.
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
async def terminate_app(self, package_or_bundle_id: str | None) -> bool: async def terminate_app(self, package_name: str | None) -> bool:
"""Terminate an application. """Force-stop application process matching the target package."""
raise NotImplementedError
Returns True on success, False on failure.
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
async def open_url(self, url: str) -> bool: async def open_url(self, url: str) -> bool:
"""Open a URL. """Dispatch system VIEW intent for the specified URI."""
raise NotImplementedError
Returns True on success, False on failure.
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
async def press_back(self) -> bool: async def press_back(self) -> bool:
"""Press the back button (Android). """Trigger Android BACK system navigation key event."""
raise NotImplementedError
Returns True on success, False on failure.
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
async def press_home(self) -> bool: async def press_home(self) -> bool:
"""Press the home button. """Trigger Android HOME system navigation key event."""
raise NotImplementedError
Returns True on success, False on failure.
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
async def press_enter(self) -> bool: async def press_enter(self) -> bool:
"""Press the enter/return key. """Trigger ENTER / ACTION_DONE key event on virtual keyboard."""
raise NotImplementedError
Returns True on success, False on failure.
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
async def press_key(self, keycode: str) -> bool: async def press_key(self, keycode: str) -> bool:
"""Press a specific key event. """Dispatch raw Android keycode event (e.g., 'KEYCODE_TAB')."""
raise NotImplementedError
Returns True on success, False on failure.
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
async def get_ui_hierarchy(self) -> list[dict]: async def get_ui_hierarchy(self) -> list[dict[str, Any]]:
"""Get the UI element hierarchy. """Retrieve flattened or nested accessibility tree node descriptors."""
raise NotImplementedError
Returns a list of UI elements with their properties.
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
def find_element( def find_element(
self, self,
ui_hierarchy: list[dict], ui_hierarchy: list[dict[str, Any]],
resource_id: str | None = None, resource_id: str | None = None,
text: str | None = None, text: str | None = None,
index: int = 0, index: int = 0,
) -> tuple[dict | None, Bounds | None, str | None]: ) -> tuple[dict[str, Any] | None, Bounds | None, str | None]:
"""Find a UI element in the hierarchy. """Locate target element within parsed hierarchy by ID or label matching."""
raise NotImplementedError
Returns:
Tuple of (element_dict, bounds, error_message)
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
async def cleanup(self) -> None: async def cleanup(self) -> None:
"""Clean up resources (e.g., stop companion processes).""" """Tear down companion sessions, tunnels, and streaming handles."""
raise NotImplementedError("Subclasses must implement this method") raise NotImplementedError
@abstractmethod @abstractmethod
async def erase_text(self, nb_chars: int | None = None) -> bool: async def erase_text(self, nb_chars: int | None = None) -> bool:
"""Erase the last nb_chars characters. """Dispatch backspace events to clear trailing characters in focused field."""
raise NotImplementedError
Returns True on success, False on failure.
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
async def get_screen_data(self) -> "ScreenDataResponse": async def get_screen_data(self) -> ScreenDataResponse:
"""Get screen data including screenshot (base64), UI hierarchy elements, """Capture unified screen frame, layout tree, and viewport dimensions."""
raise NotImplementedError
screen dimensions, and platform.
Returns:
ScreenDataResponse with base64 screenshot, elements, width, height,
platform
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
def get_compressed_b64_screenshot(self, image_base64: str, quality: int = 50) -> str: def get_compressed_b64_screenshot(self, image_base64: str, quality: int = 50) -> str:
"""Compress a base64 image. """Produce compressed image payload for bandwidth-constrained turns."""
raise NotImplementedError
Returns the compressed base64 image.
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
async def start_video_recording( async def start_video_recording(
self, self,
max_duration_seconds: int = 900, max_duration_seconds: int = 900,
) -> VideoRecordingResult: ) -> VideoRecordingResult:
"""Start screen recording on the device. """Initialize session screen capture recording."""
raise NotImplementedError
Args:
max_duration_seconds: Maximum recording duration in seconds.
Returns:
VideoRecordingResult with success status and message.
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
async def stop_video_recording(self) -> VideoRecordingResult: async def stop_video_recording(self) -> VideoRecordingResult:
"""Stop screen recording and retrieve the video file. """Terminate active screen capture and finalize video artifact."""
raise NotImplementedError
Returns:
VideoRecordingResult with success status, message, and video_path if
successful.
"""
raise NotImplementedError("Subclasses must implement this method")
@abstractmethod @abstractmethod
async def extract_segment_metadata( async def extract_segment_metadata(
@@ -230,16 +169,5 @@ class MobileDeviceController(ABC):
start_relative_time: float, start_relative_time: float,
end_relative_time: float | None = None, end_relative_time: float | None = None,
) -> VideoRecordingResult: ) -> VideoRecordingResult:
"""Get a video segment for a specific time range. """Slice video segment for the requested relative execution epoch."""
raise NotImplementedError
Args:
start_relative_time: Start time in seconds relative to session
start.
end_relative_time: End time in seconds relative to session start, or
None for latest.
Returns:
VideoRecordingResult with success status, message, and video_path if
successful.
"""
raise NotImplementedError("Subclasses must implement this method")
@@ -11,15 +11,18 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
import os import os
from shutil import which from shutil import which
import subprocess
import time import time
from typing import Any from typing import Any
from artemis.context import ArtemisContext, DevicePlatform from artemis.context import ArtemisContext, DevicePlatform
from artemis.utils.logger import ArtemisLogger, get_logger from artemis.utils.logger import ArtemisLogger, get_logger
from artemis.utils.shell_utils import run_shell_command_on_host
logger = get_logger(__name__) logger = get_logger(__name__)
@@ -57,12 +60,17 @@ def get_first_device(
if which("adb"): if which("adb"):
try: try:
android_output = run_shell_command_on_host("adb devices") res = subprocess.run(
lines = android_output.strip().split("\n") ["adb", "devices"],
for line in lines: capture_output=True,
text=True,
check=True,
stdin=subprocess.DEVNULL,
)
for line in res.stdout.strip().split("\n"):
if "device" in line and not line.startswith("List of devices"): if "device" in line and not line.startswith("List of devices"):
return line.split()[0], DevicePlatform.ANDROID, None return line.split()[0], DevicePlatform.ANDROID, None
except RuntimeError as e: except Exception as e:
if logger: if logger:
logger.error(f"ADB command failed: {e}") logger.error(f"ADB command failed: {e}")
@@ -11,6 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
import asyncio import asyncio
import os import os
-20
View File
@@ -511,24 +511,6 @@ def _get_active_subgoal_hashes(ctx: ArtemisContext) -> tuple[str, str | None]:
return "default", None return "default", None
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
def check_plan_mutation_rejections( def check_plan_mutation_rejections(
content_before: str, content_after: str, state: State | None = None content_before: str, content_after: str, state: State | None = None
) -> str | None: ) -> str | None:
@@ -614,7 +596,6 @@ def _reject_checker_note_write(key: str, tool_call_id: str | None) -> ToolMessag
class NoteArgs(BaseModel): class NoteArgs(BaseModel):
model_config = {"ignored_types": (CyFunctionDetector,)}
key: str = Field(..., description=SAVE_NOTE_ARG_KEY_DESC) key: str = Field(..., description=SAVE_NOTE_ARG_KEY_DESC)
content: str = Field(..., description=SAVE_NOTE_ARG_CONTENT_DESC) content: str = Field(..., description=SAVE_NOTE_ARG_CONTENT_DESC)
@@ -852,7 +833,6 @@ def wrap_note_tool(ctx: ArtemisContext, original_tool):
class UpdateNoteArgs(BaseModel): class UpdateNoteArgs(BaseModel):
model_config = {"ignored_types": (CyFunctionDetector,)}
key: str = Field(..., description=UPDATE_NOTE_ARG_KEY_DESC) key: str = Field(..., description=UPDATE_NOTE_ARG_KEY_DESC)
target: str = Field(..., description=UPDATE_NOTE_ARG_TARGET_DESC) target: str = Field(..., description=UPDATE_NOTE_ARG_TARGET_DESC)
replacement: str = Field(..., description=UPDATE_NOTE_ARG_REPLACEMENT_DESC) replacement: str = Field(..., description=UPDATE_NOTE_ARG_REPLACEMENT_DESC)
+18 -2
View File
@@ -26,7 +26,6 @@ from artemis.config import checker_overrides_for_level, initialize_llm_config, s
from artemis.utils.startup_progress import publish_startup_progress from artemis.utils.startup_progress import publish_startup_progress
from artemis import Agent, Builders from artemis import Agent, Builders
from artemis.sdk.types.task import AgentProfile from artemis.sdk.types.task import AgentProfile
from artemis.utils.cli_helpers import display_device_status
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
from artemis.utils.video import check_ffmpeg_available from artemis.utils.video import check_ffmpeg_available
import signal import signal
@@ -37,6 +36,23 @@ import typer
logger = get_logger(__name__) logger = get_logger(__name__)
def _display_device_status(console: Console, adb_client: AdbClient | None = None) -> None:
"""Checks for connected devices and displays the status."""
console.print("\n[bold]📱 Device Status[/bold]")
devices = None
if adb_client is not None:
try:
devices = adb_client.device_list()
except Exception:
devices = None
if devices:
console.print("✅ [bold green]Android device(s) connected:[/bold green]")
for device in devices:
console.print(f" - {device.serial}")
else:
console.print("❌ [bold red]No Android device found.[/bold red]")
async def execute_task( async def execute_task(
goal: str, goal: str,
device_serial: str | None = None, device_serial: str | None = None,
@@ -451,7 +467,7 @@ def run_command(
# Optional cosmetic device-status display; run continues without it. # Optional cosmetic device-status display; run continues without it.
logger.debug(f"Could not create ADB client for device status display: {exc}") logger.debug(f"Could not create ADB client for device status display: {exc}")
display_device_status(console, adb_client=adb_client) _display_device_status(console, adb_client=adb_client)
cancelled = False cancelled = False
original_sigterm = None original_sigterm = None
+2 -2
View File
@@ -93,8 +93,8 @@
// 📋 Summarizer: Aggregates step history and generates structured outcome // 📋 Summarizer: Aggregates step history and generates structured outcome
"summarizer": {}, "summarizer": {},
// 🔽 Hopper: Fast lightweight package name locator // 🔍 Entity Extractor: Fast lightweight package name locator and entity extractor
"hopper": { "entity_extractor": {
"model": "gemini-3.5-flash-lite", "model": "gemini-3.5-flash-lite",
"fallback": { "fallback": {
"provider": "google", "provider": "google",
+3 -1
View File
@@ -11,6 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
import asyncio import asyncio
import inspect import inspect
@@ -137,7 +140,6 @@ class Agent:
_current_task: asyncio.Task | None = None _current_task: asyncio.Task | None = None
_task_lock: asyncio.Lock _task_lock: asyncio.Lock
_cloud_mobile_id: str | None = None
def __init__( def __init__(
self, self,
+3 -7
View File
@@ -11,6 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
"""Builder for AgentConfig objects using a fluent interface.""" """Builder for AgentConfig objects using a fluent interface."""
@@ -69,7 +72,6 @@ class AgentConfigBuilder:
self._video_recording_tools_enabled: bool = detect_video_tools_enabled() self._video_recording_tools_enabled: bool = detect_video_tools_enabled()
self._force_web_accessibility: bool = False self._force_web_accessibility: bool = False
self._disable_checker: bool = False self._disable_checker: bool = False
self._cloud_mobile_id_or_ref: str | None = None
self._concurrency_mode: str = "per_device" self._concurrency_mode: str = "per_device"
self._max_concurrency: int | None = None self._max_concurrency: int | None = None
@@ -152,12 +154,6 @@ class AgentConfigBuilder:
platform_or_device_id: DevicePlatform or unique identifier for the device platform_or_device_id: DevicePlatform or unique identifier for the device
device_id: The unique identifier for the device (if platform was passed first) device_id: The unique identifier for the device (if platform was passed first)
""" """
if self._cloud_mobile_id_or_ref is not None:
raise ValueError(
"Device ID cannot be set when a cloud mobile is already"
" configured.\n> for_device() and for_cloud_mobile() are"
" mutually exclusive"
)
if isinstance(platform_or_device_id, DevicePlatform): if isinstance(platform_or_device_id, DevicePlatform):
self._device_platform = platform_or_device_id self._device_platform = platform_or_device_id
self._device_id = device_id self._device_id = device_id
+62 -159
View File
@@ -11,123 +11,71 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
"""Builder for TaskRequest objects using a fluent interface.""" """Fluent specification builders for ARTEMIS automation task requests.
Provides type-safe configuration chaining for task objectives, execution bounds,
device sandboxing constraints, and structured schema validations.
"""
from __future__ import annotations
from pathlib import Path from pathlib import Path
from typing import Generic, TypeVar, cast from typing import Generic, Self, TypeVar, cast
try: from pydantic import BaseModel
from typing import Self
except ImportError:
from typing import Self
from artemis.constants import RECURSION_LIMIT from artemis.constants import RECURSION_LIMIT
from artemis.sdk.types.agent import AgentProfile from artemis.sdk.types.agent import AgentProfile
from artemis.sdk.types.task import TaskRequest, TaskRequestCommon from artemis.sdk.types.task import TaskRequest, TaskRequestCommon
from pydantic import BaseModel
TIn = TypeVar("TIn", bound=BaseModel | None) TIn = TypeVar("TIn", bound=BaseModel | None)
TOut = TypeVar("TOut", bound=BaseModel) TOut = TypeVar("TOut", bound=BaseModel)
class _CyFunctionDetectorMeta(type): class TaskRequestCommonBuilder:
def __instancecheck__(self, instance): """Configurator for common execution bounds and sandbox constraints."""
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
def __init__(self) -> None:
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta): self._max_steps: int = RECURSION_LIMIT
pass self._record_trace: bool = True
self._trace_path: Path = Path("traces")
class TaskRequestCommonBuilder(BaseModel):
"""Builder class providing a fluent interface for creating TaskRequestCommon objects."""
model_config = {"ignored_types": (CyFunctionDetector,)}
def __init__(self):
self._max_steps = RECURSION_LIMIT
self._record_trace = True
self._trace_path = Path("traces")
self._llm_output_path: Path | None = None self._llm_output_path: Path | None = None
self._locked_app_package: str | None = None self._locked_app_package: str | None = None
self._app_path: Path | None = None self._app_path: Path | None = None
def with_max_steps(self, max_steps: int) -> Self: def with_max_steps(self, max_steps: int) -> Self:
"""Set the maximum number of steps the task can take. """Constrain the maximum interaction turns allowed."""
Args:
max_steps: Maximum number of steps
"""
self._max_steps = max_steps self._max_steps = max_steps
return self return self
def with_trace_recording(self, enabled: bool = True, path: str | None = None) -> Self: def with_trace_recording(self, enabled: bool = True, path: str | Path | None = None) -> Self:
"""Configure trace recording for the task. """Toggle telemetry snapshot capture and set output storage directory."""
Traces record screenshots and actions during execution.
Args:
enabled: Whether to enable trace recording
path: Directory path where traces should be saved
"""
self._record_trace = enabled self._record_trace = enabled
if enabled and path: if enabled and path:
self._trace_path = Path(path) self._trace_path = Path(path)
return self return self
def with_llm_output_saving(self, path: str) -> Self: def with_llm_output_saving(self, path: str | Path) -> Self:
"""Configure LLM output saving for the task. """Designate destination path for persisting raw reasoning payloads."""
Args:
path: Path where to save the LLM output message
"""
self._llm_output_path = Path(path) self._llm_output_path = Path(path)
return self return self
def with_locked_app_package(self, package_name: str) -> Self: def with_locked_app_package(self, package_name: str) -> Self:
"""Set the app package to lock execution to. """Bind test execution strictly to the target Android package namespace."""
self._locked_app_package = package_name.strip()
This ensures the specified app is launched and in the foreground before
the agentic loop starts.
Args:
package_name: Package name (Android, e.g., 'com.whatsapp')
"""
self._locked_app_package = package_name
return self return self
def with_app_path(self, app_path: str | Path) -> Self: def with_app_path(self, app_path: str | Path) -> Self:
"""Set the path to an app to install before running the task. """Provide path to local APK for automatic installation prior to run."""
self._app_path = Path(app_path)
For Android: Path to an APK file.
The app will be installed automatically before the task starts.
Args:
app_path: Path to the app file to install
"""
self._app_path = Path(app_path) if isinstance(app_path, str) else app_path
return self return self
def build(self) -> TaskRequestCommon: def build(self) -> TaskRequestCommon:
"""Build the TaskRequestCommon object. """Materialize configured constraints into a TaskRequestCommon contract."""
Returns:
A configured TaskRequestCommon object
Raises:
ValueError: If required fields are missing
"""
return TaskRequestCommon( return TaskRequestCommon(
max_steps=self._max_steps, max_steps=self._max_steps,
record_trace=self._record_trace, record_trace=self._record_trace,
@@ -139,110 +87,66 @@ class TaskRequestCommonBuilder(BaseModel):
class TaskRequestBuilder(TaskRequestCommonBuilder, Generic[TIn]): class TaskRequestBuilder(TaskRequestCommonBuilder, Generic[TIn]):
"""Builder class providing a fluent interface for creating TaskRequest objects. """Fluent configurator constructing immutable TaskRequest specifications."""
This builder allows for step-by-step construction of a TaskRequest with def __init__(self, goal: str) -> None:
clear methods that make the configuration process intuitive and type-safe.
Examples:
>>> builder = TaskRequestBuilder[None](goal="Open Gmail and check unread
emails")
>>> task_request = (
... builder
... .with_max_steps(30)
... .using_profile("LowReasoning")
... .with_output_description("A list of email subjects and senders")
... .build()
... )
"""
model_config = {"ignored_types": (CyFunctionDetector,)}
def __init__(self, goal: str):
"""Initialize an empty TaskRequestBuilder."""
super().__init__() super().__init__()
self._goal = goal self._goal: str = goal
self._profile: str | AgentProfile | None = None self._profile: str | AgentProfile | None = None
self._name: str | None = None self._name: str | None = None
self._output_description = None self._output_description: str | None = None
self._output_format: type[TIn] | None = None self._output_format: type[TIn] | None = None
@classmethod @classmethod
def from_common(cls, goal: str, common: TaskRequestCommon): def from_common(cls, goal: str, common: TaskRequestCommon) -> TaskRequestBuilder[None]:
res = cls(goal=goal) """Derive a new task builder initialized with existing environment constraints."""
res._max_steps = common.max_steps builder = TaskRequestBuilder[None](goal=goal)
res._record_trace = common.record_trace builder._max_steps = common.max_steps
res._trace_path = common.trace_path builder._record_trace = common.record_trace
res._llm_output_path = common.llm_output_path builder._trace_path = common.trace_path
res._locked_app_package = common.locked_app_package builder._llm_output_path = common.llm_output_path
res._app_path = common.app_path builder._locked_app_package = common.locked_app_package
return res builder._app_path = common.app_path
return builder
def using_profile(self, profile: str | AgentProfile) -> "TaskRequestBuilder[TIn]": def using_profile(self, profile: str | AgentProfile) -> Self:
"""Set the agent profile for executing the task. """Select execution profile (e.g., 'flash', 'pro', or custom AgentProfile)."""
Args:
profile: The agent profile to use
"""
self._profile = profile self._profile = profile
return self return self
def with_name(self, name: str) -> "TaskRequestBuilder[TIn]": def with_name(self, name: str) -> Self:
"""Set the name of the task - useful when recording traces. """Assign human-readable identifier to this test task."""
self._name = name.strip()
Otherwise, a random name will be generated.
Args:
name: Name of the task
"""
self._name = name
return self return self
def without_llm_output_saving(self) -> Self: def without_llm_output_saving(self) -> Self:
"""Disable LLM output saving for the task.""" """Omit persisting standalone model reasoning outputs."""
self._llm_output_path = None self._llm_output_path = None
return self return self
def with_output_description(self, description: str) -> "TaskRequestBuilder[TIn]": def with_output_description(self, description: str) -> Self:
"""Set the description of the expected output format. """Supply natural language extraction instructions for unformatted deliverables."""
This is especially useful for data extraction tasks.
Args:
description: Description of the expected output format
"""
self._output_description = description self._output_description = description
return self return self
def with_output_format(self, output_format: type[TOut]) -> "TaskRequestBuilder[TOut]": def with_output_format(self, output_format: type[TOut]) -> TaskRequestBuilder[TOut]:
"""Set the pydantic model for the expected output format. """Bind output schema to a structured Pydantic model class."""
self._output_format = cast(type[TIn], output_format)
Args:
output_format: Pydantic model instance defining the output format
"""
self._output_format = output_format # type: ignore
return cast(TaskRequestBuilder[TOut], self) return cast(TaskRequestBuilder[TOut], self)
def build(self) -> TaskRequest[TIn]: def build(self) -> TaskRequest[TIn]:
"""Build the TaskRequest object. """Validate criteria and construct final TaskRequest instance."""
if not self._goal or not self._goal.strip():
raise ValueError("Task goal must be a non-empty natural language objective.")
Returns: if self._output_format is not None and self._output_description is not None:
A configured TaskRequest object raise ValueError("Cannot specify both output_format and output_description simultaneously.")
Raises: profile_name = self._profile.name if isinstance(self._profile, AgentProfile) else self._profile
ValueError: If required fields are missing
"""
if not self._goal:
raise ValueError("Task goal is required")
if self._output_format and self._output_description: return TaskRequest(
raise ValueError("Output format and description are mutually exclusive") goal=self._goal.strip(),
profile=profile_name,
task_request = TaskRequest(
goal=self._goal,
profile=self._profile.name
if isinstance(self._profile, AgentProfile)
else self._profile,
task_name=self._name, task_name=self._name,
output_description=self._output_description, output_description=self._output_description,
output_format=self._output_format, output_format=self._output_format,
@@ -253,4 +157,3 @@ class TaskRequestBuilder(TaskRequestCommonBuilder, Generic[TIn]):
locked_app_package=self._locked_app_package, locked_app_package=self._locked_app_package,
app_path=self._app_path, app_path=self._app_path,
) )
return task_request
@@ -0,0 +1,53 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Example: Accessibility Helper Diagnostics & Resilient Hierarchy Dumps.
This example shows how ARTEMIS uses its managed Accessibility Helper APK
to perform conflict-free UI tree inspection alongside external test frameworks
(e.g., Mobly, Appium) without locking the Android UiAutomation service.
"""
import asyncio
from artemis.controllers.platform_specific_commands_controller import get_first_device
from artemis.drivers.android.adb_driver import AndroidAdbDriver
async def main():
device_id, platform, _ = get_first_device()
if not device_id:
print("⚠️ No connected Android device found via ADB.")
return
print(f"📱 Connected device: {device_id} (Platform: {platform})")
# Initialize the Artemis Android ADB Driver
driver = AndroidAdbDriver(device_id=device_id)
# Capture screen data through Artemis managed hierarchy pipeline
screen_data = await driver.get_screen_data()
print(f"📐 Screen Dimensions: {screen_data.width}x{screen_data.height}")
print(f"🌲 Hierarchy Elements Captured: {len(screen_data.elements)}")
if screen_data.elements:
first = screen_data.elements[0]
print(f" Root Element Class: {first.get('class', 'N/A')}")
print(f" Package: {first.get('package', 'N/A')}")
await driver.disconnect()
print("✅ Diagnostics complete.")
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,69 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Example: Automated Android System Settings Inspection.
This example demonstrates how to use the ARTEMIS SDK to launch the Android
Settings app, inspect device battery and storage configurations, and retrieve
verified diagnostics as a structured response.
"""
import asyncio
from pydantic import BaseModel, Field
from artemis.interfaces.sdk.client import ArtemisClient
from artemis.sdk.builders.task_request_builder import TaskRequestBuilder
class BatteryDiagnostics(BaseModel):
"""Structured report of Android battery diagnostics."""
battery_percentage: str | None = Field(
default=None, description="Current battery percentage level visible on screen"
)
battery_saver_status: str | None = Field(
default=None, description="Battery Saver state (e.g. On, Off)"
)
diagnostics_summary: str = Field(
description="Summary of the battery health and system status"
)
async def main():
# Initialize the Artemis SDK client
client = ArtemisClient()
# Build the task request targeting Android Settings
task_request = (
TaskRequestBuilder(goal="Open Android Settings, navigate to Battery, and extract the current battery level and status.")
.with_name("settings_battery_inspection")
.with_max_steps(15)
.with_locked_app_package("com.android.settings")
.with_output_format(BatteryDiagnostics)
.build()
)
print(f"🚀 Running Artemis Task: {task_request.task_name}...")
task = await client.run_task(task_request)
print(f"✅ Task Status: {task.status}")
if task.result and task.result.content:
report = task.result.get_as_model(BatteryDiagnostics)
print(f"📊 Battery Level: {report.battery_percentage}")
print(f"⚡ Battery Saver: {report.battery_saver_status}")
print(f"📝 Summary: {report.diagnostics_summary}")
if __name__ == "__main__":
asyncio.run(main())
@@ -1,87 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
from artemis.sdk import Agent
from pydantic import BaseModel, Field
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class MessageResult(BaseModel):
"""Structured result from messaging task."""
model_config = {"ignored_types": (CyFunctionDetector,)}
messages_sent: int = Field(..., description="Number of messages successfully sent")
contacts: list[str] = Field(..., description="List of contacts messaged")
success: bool = Field(..., description="Whether all messages were sent successfully")
async def main() -> None:
# Create agent with default configuration
agent = Agent()
try:
await agent.init()
# Use app lock to keep execution in WhatsApp
# This ensures the agent stays in the app and relaunches if needed
task = (
agent.new_task("Send 'Happy New Year!' message to Alice, Bob, and Charlie on WhatsApp")
.with_name("send_new_year_messages")
.with_locked_app_package("com.whatsapp") # Lock to WhatsApp
.with_output_format(MessageResult)
.with_max_steps(600) # Messaging tasks may need more steps
.build()
)
print("Sending messages with app lock enabled...")
print("The agent will stay in WhatsApp and relaunch if needed.\n")
result = await agent.run_task(request=task)
if result:
print("\n=== Messaging Complete ===")
print(f"Messages sent: {result.messages_sent}")
print(f"Contacts: {', '.join(result.contacts)}")
print(f"Success: {result.success}")
else:
print("Failed to send messages")
except Exception as e:
print(f"Error: {e}")
finally:
await agent.clean()
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,53 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Example: App Workflow Automation with Trace Recording.
This example demonstrates how to orchestrate multi-step mobile UI navigation,
capture step-by-step traces, and enforce deterministic completion criteria
using ARTEMIS.
"""
import asyncio
from artemis.interfaces.sdk.client import ArtemisClient
from artemis.sdk.builders.task_request_builder import TaskRequestBuilder
async def main():
client = ArtemisClient()
# Build a task with full trace recording
task_request = (
TaskRequestBuilder(
goal="Open Clock app, verify current alarm settings, and ensure the active alarms list is visible."
)
.with_name("clock_alarm_audit")
.with_max_steps(12)
.with_locked_app_package("com.google.android.deskclock")
.with_trace_recording(enabled=True, path="traces/clock_alarm_audit")
.with_output_description("Summary of configured alarms and their status")
.build()
)
print(f"🎬 Initiating Workflow Automation: {task_request.task_name}...")
task = await client.run_task(task_request)
print(f"🏁 Execution Finished: status={task.status}")
if task.result:
print(f"⏱️ Duration: {task.result.execution_time_seconds:.2f}s, Steps: {task.result.steps_taken}")
print(f"📄 Output: {task.result.content}")
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,68 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Example: Multimodal UI Testing and Screen Element Grounding.
This example demonstrates using ARTEMIS multimodal perception (Explorer & OCR-XML
fusion) to locate visual elements on dynamic or non-standard UI canvases without
relying on static coordinates.
"""
import asyncio
from pydantic import BaseModel, Field
from artemis.interfaces.sdk.client import ArtemisClient
from artemis.sdk.builders.task_request_builder import TaskRequestBuilder
class VisualAuditReport(BaseModel):
"""Structured report verifying visual UI layout integrity."""
active_screen_title: str = Field(description="Title of the active screen or view")
interactive_buttons_count: int = Field(
description="Number of actionable buttons identified on the current view"
)
visual_anomalies_detected: bool = Field(
description="True if unexpected popups, overlapping text, or error banners exist"
)
observations: str = Field(description="Visual layout observations and audit details")
async def main():
client = ArtemisClient()
task_request = (
TaskRequestBuilder(
goal="Open Calculator, verify numeric keypad buttons layout, and confirm display screen is empty."
)
.with_name("calculator_visual_audit")
.with_max_steps(8)
.with_locked_app_package("com.google.android.calculator")
.with_output_format(VisualAuditReport)
.build()
)
print("🔍 Starting Multimodal Vision Audit...")
task = await client.run_task(task_request)
if task.result and task.result.content:
audit = task.result.get_as_model(VisualAuditReport)
print(f"📱 View: {audit.active_screen_title}")
print(f"🔘 Interactive Buttons: {audit.interactive_buttons_count}")
print(f"⚠️ Anomalies Detected: {audit.visual_anomalies_detected}")
print(f"📝 Observations: {audit.observations}")
if __name__ == "__main__":
asyncio.run(main())
@@ -1,111 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Simple Photo Organizer - Basic SDK Usage Example
This example demonstrates a straightforward way to use the Artemis SDK
without builders or advanced configuration. It performs a real-world automation
task:
1. Opens the photo gallery
2. Finds photos from a specific date
3. Creates an album and moves those photos into it
Run:
- python artemis/sdk/examples/simple_photo_organizer.py
"""
import asyncio
from datetime import date, timedelta
from artemis.sdk import Agent
from pydantic import BaseModel, Field
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class PhotosResult(BaseModel):
"""Structured result from photo search."""
model_config = {"ignored_types": (CyFunctionDetector,)}
found_photos: int = Field(..., description="Number of photos found")
date_range: str = Field(..., description="Date range of photos found")
album_created: bool = Field(..., description="Whether an album was created")
album_name: str = Field(..., description="Name of the created album")
photos_moved: int = Field(0, description="Number of photos moved to the album")
async def main() -> None:
# Create a simple agent with default configuration
agent = Agent()
try:
# Initialize agent (finds a device, starts required servers)
await agent.init()
# Calculate yesterday's date for the example
yesterday = date.today() - timedelta(days=1)
formatted_date = yesterday.strftime("%B %d") # e.g. "August 22"
print(f"Looking for photos from {formatted_date}...")
# First task: search for photos and organize them, with typed output
result = await agent.run_task(
goal=(
"Open the Photos/Gallery app. Find photos taken on"
f" {formatted_date}. Create a new album named '{formatted_date}"
" Memories' and move those photos into it. Count how many"
" photos were moved."
),
output=PhotosResult,
name="organize_photos",
)
# Handle and display the result
if result:
print("\n=== Photo Organization Complete ===")
print(f"Found: {result.found_photos} photos from {result.date_range}")
if result.album_created:
print(f"Created album: '{result.album_name}'")
print(f"Moved {result.photos_moved} photos to the album")
else:
print("No album was created")
else:
print("Failed to organize photos")
except Exception as e:
print(f"Error: {e}")
finally:
# Always clean up resources
await agent.clean()
if __name__ == "__main__":
asyncio.run(main())
@@ -1,235 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Smart Notification Assistant - Intermediate SDK Usage Example
This example demonstrates more advanced SDK features including:
- TaskRequestBuilder pattern
- Multiple agent profiles for different reasoning tasks
- Tracing for debugging/visualization
- Structured output with Pydantic
- Exception handling
It performs a practical automation task:
1. Checks notification panel for unread notifications
2. Categorizes them by priority/app
3. Performs actions based on notification content
Run:
- python artemis/sdk/examples/smart_notification_assistant.py
"""
import asyncio
from datetime import datetime
try:
from enum import StrEnum
except ImportError:
from enum import Enum
class StrEnum(str, Enum):
pass
from artemis.config import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback
from artemis.sdk import Agent
from artemis.sdk.builders import Builders
from artemis.sdk.types import AgentProfile
from artemis.sdk.types.exceptions import AgentError
from pydantic import BaseModel, Field
class NotificationPriority(StrEnum):
HIGH = "high"
MEDIUM = "medium"
LOW = "low"
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class Notification(BaseModel):
"""Individual notification details."""
model_config = {"ignored_types": (CyFunctionDetector,)}
app_name: str = Field(..., description="Name of the app that sent the notification")
title: str = Field(..., description="Title/header of the notification")
message: str = Field(..., description="Message content of the notification")
priority: NotificationPriority = Field(
default=NotificationPriority.MEDIUM,
description="Priority level of notification",
)
class NotificationSummary(BaseModel):
"""Summary of all notifications."""
model_config = {"ignored_types": (CyFunctionDetector,)}
total_count: int = Field(..., description="Total number of notifications found")
high_priority_count: int = Field(0, description="Count of high priority notifications")
notifications: list[Notification] = Field(
default_factory=list, description="List of individual notifications"
)
def get_agent() -> Agent:
# Create two specialized profiles:
# 1. An analyzer profile for detailed inspection tasks
analyzer_profile = AgentProfile(
name="analyzer",
llm_config=LLMConfig(
planner=LLMWithFallback(
provider="openrouter",
model="meta-llama/llama-4-scout",
fallback=LLM(provider="openrouter", model="meta-llama/llama-4-maverick"),
),
utils=LLMConfigUtils(
outputter=LLMWithFallback(
provider="openai",
model="gpt-5-nano",
fallback=LLM(provider="openai", model="gpt-5-mini"),
),
hopper=LLMWithFallback(
provider="openai",
model="gpt-5-nano",
fallback=LLM(provider="openai", model="gpt-5-mini"),
),
),
),
# from_file="/tmp/analyzer.jsonc" # can be loaded from file
)
# 2. An action profile for handling easy & fast actions based on notifications
action_profile = AgentProfile(
name="note_taker",
llm_config=LLMConfig(
planner=LLMWithFallback(
provider="openai",
model="o3",
fallback=LLM(provider="openai", model="gpt-5"),
),
utils=LLMConfigUtils(
outputter=LLMWithFallback(
provider="openai",
model="gpt-5-nano",
fallback=LLM(provider="openai", model="gpt-5-mini"),
),
hopper=LLMWithFallback(
provider="openai",
model="gpt-5-nano",
fallback=LLM(provider="openai", model="gpt-5-mini"),
),
),
),
)
# Configure default task settings with tracing
task_defaults = Builders.TaskDefaults.with_max_steps(200).build()
# Configure the agent
config = (
Builders.AgentConfig.add_profiles(profiles=[analyzer_profile, action_profile])
.with_default_profile(profile=action_profile)
.with_default_task_config(config=task_defaults)
.build()
)
return Agent(config=config)
async def main():
# Set up traces directory with timestamp for uniqueness
timestamp = datetime.now().strftime("%Y%m%d_%H%M")
traces_dir = f"/tmp/notification_traces/{timestamp}"
agent = get_agent()
try:
# Initialize agent (finds a device, starts required servers)
await agent.init()
print("Checking for notifications...")
# Task 1: Get and analyze notifications with analyzer profile
notification_task = (
agent.new_task(
goal=(
"Open the notification panel (swipe down from top). Scroll"
" through the first 3 unread notifications. For each"
" notification, identify the app name, title, and content."
" Tag messages from messaging apps or email as high"
" priority."
)
)
.with_output_format(NotificationSummary)
.using_profile("analyzer")
.with_name("notification_scan")
.with_max_steps(400)
.with_trace_recording(enabled=True, path=traces_dir)
.build()
)
# Execute the task with proper exception handling
try:
notifications = await agent.run_task(request=notification_task)
# Display the structured results
if notifications:
print("\n=== Notification Summary ===")
print(f"Total notifications: {notifications.total_count}")
print(f"High priority: {notifications.high_priority_count}")
# Task 2: Create a note to store the notification summary
response = await agent.run_task(
goal=(
"Open my Notes app and create a new note summarizing"
f" the following information:\n{notifications}"
),
name="email_action",
profile="note_taker",
)
print(f"Action result: {response}")
else:
print("Failed to retrieve notifications")
except AgentError as e:
print(f"Agent error occurred: {e}")
except Exception as e:
print(f"Unexpected error: {type(e).__name__}: {e}")
raise
finally:
# Clean up
await agent.clean()
print(f"\nTraces saved to: {traces_dir}")
if __name__ == "__main__":
asyncio.run(main())
@@ -1,110 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Video Transcription Example
This example demonstrates how to use the video recording tools to capture
and analyze video content from a mobile device screen.
The agent can:
1. Start a screen recording
2. Perform actions while recording
3. Stop the recording and analyze its content using Gemini models
Use case: Recording a video playing on the screen and transcribing its content.
"""
import asyncio
from artemis.config import LLM, LLMConfig, LLMConfigUtils, LLMWithFallback
from artemis.sdk.agent import Agent
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
from artemis.sdk.types.agent import AgentConfig
from artemis.sdk.types.task import AgentProfile, TaskRequest
def get_video_capable_llm_config() -> LLMConfig:
"""Returns an LLM config with video_analyzer configured.
The video_analyzer must use a video-capable Gemini model:
- gemini-3.8-flash (recommended - fast and capable)
- gemini-3.7-flash
- gemini-3-pro-preview
- gemini-2.5-flash
- gemini-2.5-pro
- gemini-2.0-flash
"""
return LLMConfig(
planner=LLMWithFallback(
provider="openai",
model="gpt-5-nano",
fallback=LLM(provider="openai", model="gpt-5-mini"),
),
utils=LLMConfigUtils(
outputter=LLMWithFallback(
provider="openai",
model="gpt-5-nano",
fallback=LLM(provider="openai", model="gpt-5-mini"),
),
hopper=LLMWithFallback(
provider="openai",
model="gpt-5-nano",
fallback=LLM(provider="openai", model="gpt-5-mini"),
),
video_analyzer=LLMWithFallback(
provider="google",
model="gemini-3.8-flash",
fallback=LLM(provider="google", model="gemini-3.7-flash"),
),
),
)
async def main():
config: AgentConfig = (
AgentConfigBuilder()
.add_profile(
AgentProfile(
name="VideoCapable",
llm_config=get_video_capable_llm_config(),
)
)
.with_video_recording_tools()
.build()
)
agent = Agent(config=config)
try:
await agent.init()
result = await agent.run_task(
request=TaskRequest(
goal="""
1. Open YouTube app
2. Search for "Python tutorial"
3. Start recording the screen
4. Play the first video
5. Wait for the first 30 seconds of the video to play
6. Stop recording and tell me what was said in the video
""",
profile="VideoCapable",
)
)
print(f"Task result: {result}")
finally:
await agent.clean()
if __name__ == "__main__":
asyncio.run(main())
+2 -2
View File
@@ -16,7 +16,6 @@
from artemis.sdk.types.agent import ( from artemis.sdk.types.agent import (
AgentConfig, AgentConfig,
ApiBaseUrl,
DevicePlatform, DevicePlatform,
ServerConfig, ServerConfig,
) )
@@ -37,10 +36,10 @@ from artemis.sdk.types.task import (
TaskRequest, TaskRequest,
TaskRequestCommon, TaskRequestCommon,
TaskResult, TaskResult,
TaskRunStatus,
) )
__all__ = [ __all__ = [
"ApiBaseUrl",
"AgentConfig", "AgentConfig",
"DevicePlatform", "DevicePlatform",
"AgentProfile", "AgentProfile",
@@ -48,6 +47,7 @@ __all__ = [
"TaskRequest", "TaskRequest",
"TaskResult", "TaskResult",
"TaskRequestCommon", "TaskRequestCommon",
"TaskRunStatus",
"Task", "Task",
"AgentProfileNotFoundError", "AgentProfileNotFoundError",
"AgentTaskRequestError", "AgentTaskRequestError",
+3 -56
View File
@@ -11,9 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
from typing import Literal from typing import Literal
from urllib.parse import urlparse
from langchain_core.callbacks.base import Callbacks from langchain_core.callbacks.base import Callbacks
from artemis.config import ( from artemis.config import (
@@ -29,63 +31,8 @@ from artemis.utils.video import detect_video_tools_enabled
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class ApiBaseUrl(BaseModel):
"""Defines an API base URL."""
model_config = {"ignored_types": (CyFunctionDetector,)}
scheme: Literal["http", "https"]
host: str
port: int | None = None
def __eq__(self, other):
if not isinstance(other, ApiBaseUrl):
return False
return self.to_url() == other.to_url()
def to_url(self):
return (
f"{self.scheme}://{self.host}:{self.port}"
if self.port is not None
else f"{self.scheme}://{self.host}"
)
@classmethod
def from_url(cls, url: str) -> "ApiBaseUrl":
parsed_url = urlparse(url)
if parsed_url.scheme not in ["http", "https"]:
raise ValueError(f"Invalid scheme: {parsed_url.scheme}")
if parsed_url.hostname is None:
raise ValueError("Invalid hostname")
return cls(
scheme=parsed_url.scheme, # type: ignore
host=parsed_url.hostname,
port=parsed_url.port,
)
class ServerConfig(BaseModel): class ServerConfig(BaseModel):
"""Configuration for the required servers.""" """Configuration for the required servers."""
model_config = {"ignored_types": (CyFunctionDetector,)}
adb_host: str adb_host: str
adb_port: int adb_port: int
+39 -38
View File
@@ -12,94 +12,95 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""Exceptions for the ARTEMIS SDK. """Hierarchical exception taxonomy for the ARTEMIS client and automation runtime.
This module defines the exception hierarchy used throughout the Artemis SDK. Establishes categorized exception types covering host device discovery,
daemon lifecycle management, agent operational faults, and toolchain dependencies.
""" """
from __future__ import annotations
from typing import Literal from typing import Literal
_TOOLCHAIN_INSTALL_URLS: dict[str, str] = {
"adb": "https://developer.android.com/tools/adb",
}
class ArtemisError(Exception): class ArtemisError(Exception):
"""Base exception class for all ARTEMIS SDK exceptions.""" """Root ancestor of all exceptions originating from ARTEMIS operations."""
def __init__(self, message="An error occurred in the Artemis SDK"): def __init__(self, message: str = "An unexpected error occurred within the ARTEMIS runtime.") -> None:
self.message = message self.message = message
super().__init__(self.message) super().__init__(self.message)
class DeviceError(ArtemisError): class DeviceError(ArtemisError):
"""Exception raised for errors related to mobile devices.""" """Raised when hardware interaction or ADB communication fails."""
def __init__(self, message="A device-related error occurred"): def __init__(self, message: str = "A mobile device error occurred.") -> None:
super().__init__(message) super().__init__(message)
class DeviceNotFoundError(DeviceError): class DeviceNotFoundError(DeviceError):
"""Exception raised when no mobile device is found.""" """Raised when no authorized target Android device or emulator is accessible."""
def __init__(self, message="No mobile device found"): def __init__(self, message: str = "No connected and authorized Android device found.") -> None:
super().__init__(message) super().__init__(message)
class ServerError(ArtemisError): class ServerError(ArtemisError):
"""Exception raised for errors related to ARTEMIS servers.""" """Raised when an internal ARTEMIS daemon or background service encounters an error."""
def __init__(self, message="A server-related error occurred"): def __init__(self, message: str = "An ARTEMIS daemon service failure occurred.") -> None:
super().__init__(message) super().__init__(message)
class ServerStartupError(ServerError): class ServerStartupError(ServerError):
"""Exception raised when ARTEMIS servers fail to start.""" """Raised when a background process (e.g. Scrcpy, MCP, or local server) fails to boot."""
def __init__(self, server_name=None, message=None): def __init__(self, server_name: str | None = None, message: str | None = None) -> None:
if server_name and not message: resolved_msg = message or (f"Service '{server_name}' failed to start." if server_name else "Service startup failed.")
message = f"Failed to start {server_name}" super().__init__(resolved_msg)
elif not message:
message = "Failed to start Artemis servers"
super().__init__(message)
self.server_name = server_name self.server_name = server_name
class AgentError(ArtemisError): class AgentError(ArtemisError):
"""Exception raised for errors related to the ARTEMIS agent.""" """Raised when reasoning, task scheduling, or graph traversal fails."""
def __init__(self, message="An agent-related error occurred"): def __init__(self, message: str = "An agent operational failure occurred.") -> None:
super().__init__(message) super().__init__(message)
class AgentNotInitializedError(AgentError): class AgentNotInitializedError(AgentError):
"""Exception raised when attempting operations on an uninitialized agent.""" """Raised when invoking automation actions before initialization."""
def __init__(self, message="Agent is not initialized. Call init() first"): def __init__(self, message: str = "Agent runtime has not been initialized.") -> None:
super().__init__(message) super().__init__(message)
class AgentTaskRequestError(AgentError): class AgentTaskRequestError(AgentError):
"""Exception raised when a requested task is invalid.""" """Raised when task parameters violate execution bounds or constraints."""
def __init__(self, message="An agent task-related error occurred"): def __init__(self, message: str = "Invalid task request configuration.") -> None:
super().__init__(message) super().__init__(message)
class AgentProfileNotFoundError(AgentTaskRequestError): class AgentProfileNotFoundError(AgentTaskRequestError):
"""Exception raised when an agent profile is not found.""" """Raised when requested execution profile is not registered in system catalog."""
def __init__(self, profile_name: str): def __init__(self, profile_name: str) -> None:
super().__init__(f"Agent profile {profile_name} not found") super().__init__(f"Agent profile '{profile_name}' is not registered.")
self.profile_name = profile_name
EXECUTABLES = Literal["adb"]
class ExecutableNotFoundError(ArtemisError): class ExecutableNotFoundError(ArtemisError):
"""Exception raised when a required executable is not found.""" """Raised when a mandatory host CLI toolchain executable is missing from PATH."""
def __init__(self, executable_name: EXECUTABLES): def __init__(self, executable_name: Literal["adb"] | str) -> None:
install_instructions: dict[EXECUTABLES, str] = { guidance = _TOOLCHAIN_INSTALL_URLS.get(executable_name, "")
"adb": "https://developer.android.com/tools/adb", msg = f"Required tool '{executable_name}' was not detected in system PATH."
} if guidance:
message = f"Required executable '{executable_name}' not found in PATH." msg += f" Install instructions: {guidance}"
if executable_name in install_instructions: super().__init__(msg)
message += f"\nTo install it, please visit: {install_instructions[executable_name]}" self.executable_name = executable_name
super().__init__(message)
-210
View File
@@ -1,210 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from datetime import datetime
from typing import Annotated, Any, Literal
from pydantic import BaseModel, ConfigDict, Field, PlainSerializer
from pydantic.v1.utils import to_lower_camel
TaskRunStatus = Literal["pending", "running", "completed", "failed", "cancelled"]
IsoDatetime = Annotated[
datetime,
PlainSerializer(
func=lambda v: v.isoformat() if v else None,
return_type=str,
when_used="unless-none",
),
]
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class BaseApiModel(BaseModel):
model_config = ConfigDict(
alias_generator=to_lower_camel,
populate_by_name=True,
str_strip_whitespace=True,
)
class LLMProfileResponse(BaseApiModel):
"""Response model for LLM profile."""
id: str = Field(..., description="Profile ID")
name: str = Field(..., description="Profile name")
description: str | None = Field(None, description="Profile description")
llms: dict[str, Any] = Field(..., description="LLM configuration")
created_at: str = Field(..., description="Creation timestamp (ISO format)")
updated_at: str = Field(..., description="Last update timestamp (ISO format)")
class TaskOptionsResponse(BaseApiModel):
"""Response model for task options."""
id: str = Field(..., description="Options ID")
enable_tracing: bool = Field(..., description="Whether tracing is enabled")
max_steps: int = Field(..., description="Maximum number of steps")
enable_video_tools: bool = Field(..., description="Whether video tools are enabled")
created_at: str = Field(..., description="Creation timestamp (ISO format)")
updated_at: str = Field(..., description="Last update timestamp (ISO format)")
class TaskResponse(BaseApiModel):
"""Response model for task."""
id: str = Field(..., description="Task ID")
name: str = Field(..., description="Task name")
description: str | None = Field(None, description="Task description")
input_prompt: str = Field(..., description="Input prompt")
output_description: str | None = Field(None, description="Output description")
options: TaskOptionsResponse = Field(..., description="Task options")
created_at: str = Field(..., description="Creation timestamp (ISO format)")
updated_at: str = Field(..., description="Last update timestamp (ISO format)")
class CreateTaskRunRequest(BaseApiModel):
"""Request model for creating a task run."""
task_id: str = Field(..., description="ID of the task to run")
llm_profile_id: str = Field(..., description="LLM profile ID to use")
virtual_mobile_id: str | None = Field(None, description="Virtual mobile ID to use")
locked_app_package: str | None = Field(None, description="App package to lock for the task run")
execution_origin: str | None = Field(None, description="Origin of the task execution")
enable_video_tools: bool = Field(
default=False,
description="Whether video tools are enabled for this task run",
)
class CreateOrphanTaskRunRequest(BaseApiModel):
"""Request model for creating an orphan task run (without a parent task)."""
input_prompt: str = Field(
...,
description="Input prompt for the task run",
)
output_description: str | None = Field(
None,
description="Optional output description",
)
llm_profile_id: str = Field(
...,
description="LLM profile ID to use",
)
virtual_mobile_id: str | None = Field(
None,
description="Optional virtual mobile ID on which it runs",
)
locked_app_package: str | None = Field(
None,
description="Optional app package to lock execution to",
)
execution_origin: str | None = Field(
None,
description="Origin of the task run execution",
)
max_steps: int | None = Field(
None,
description="Maximum number of steps for the task run",
)
enable_video_tools: bool = Field(
default=False,
description="Whether video tools are enabled for this task run",
)
class UpdateTaskRunStatusRequest(BaseApiModel):
"""Request model for updating task run status."""
status: TaskRunStatus = Field(..., description="New status of the task run")
message: str | None = Field(None, description="Message associated with the status")
output: str | None = Field(None, description="Output of the task run")
class TaskRunResponse(BaseApiModel):
"""Response model for a single task run."""
id: str = Field(..., description="Unique identifier for the task run")
task: TaskResponse | None = Field(
...,
description=("ID of the task this run is for or None if manually created"),
)
llm_profile: LLMProfileResponse = Field(..., description="LLM profile ID used for this run")
status: TaskRunStatus = Field(..., description="Current status of the task run")
input_prompt: str = Field(..., description="Input prompt for this task run")
output_description: str | None = Field(None, description="Description of expected output")
created_at: datetime = Field(..., description="When the task run was created")
started_at: datetime | None = Field(None, description="When the task run started")
finished_at: datetime | None = Field(None, description="When the task run finished")
locked_app_package: str | None = Field(
None, description="The app package to lock for the task run"
)
SubgoalState = Literal["pending", "started", "completed", "failed"]
class ArtemisSubgoal(BaseModel):
"""Upsert ArtemisSubgoal API model."""
model_config = {"ignored_types": (CyFunctionDetector,)}
name: str = Field(..., description="Name of the subgoal")
state: SubgoalState = Field(default="pending", description="Current state of the subgoal")
started_at: IsoDatetime | None = Field(default=None, description="When the subgoal started")
ended_at: IsoDatetime | None = Field(default=None, description="When the subgoal ended")
class UpsertTaskRunPlanRequest(BaseApiModel):
"""Upsert task run plan request model."""
started_at: IsoDatetime = Field(..., description="When the plan started")
subgoals: list[ArtemisSubgoal] = Field(..., description="Subgoals of the plan")
ended_at: IsoDatetime | None = Field(
default=None,
description="When the plan ended (replanned or completed)",
)
class TaskRunPlanResponse(UpsertTaskRunPlanRequest):
"""Response model for a task run plan."""
id: str = Field(..., description="Unique identifier for the task run plan")
task_run_id: str = Field(..., description="ID of the task run this plan is for")
class TrajectoryGifUploadResponse(BaseApiModel):
"""Response containing trajectory GIF signed upload URL."""
signed_url: str = Field(..., description="Signed upload URL for the trajectory GIF")
task_run_id: str = Field(..., description="Task run ID for which the GIF URL was generated")
file_path: str = Field(..., description="Path where the GIF will be stored")
+86 -138
View File
@@ -11,67 +11,54 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
"""Task-related type definitions for the ARTEMIS SDK.""" """Task specifications, execution states, and outcome schemas for the ARTEMIS SDK.
Defines the declarative configuration contract for mobile automation goals,
agent operational profiles, telemetry preferences, and asynchronous result contracts.
"""
from __future__ import annotations
from collections.abc import Callable, Coroutine from collections.abc import Callable, Coroutine
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Any, Generic, TypeVar, overload from typing import Any, Generic, Literal, TypeVar, overload
from pydantic import BaseModel, ConfigDict, Field
from artemis.config import LLMConfig, get_default_llm_config from artemis.config import LLMConfig, get_default_llm_config
from artemis.constants import RECURSION_LIMIT from artemis.constants import RECURSION_LIMIT
from artemis.context import DeviceContext from artemis.context import DeviceContext
from artemis.sdk.types.platform import TaskRunStatus
from artemis.sdk.utils import load_llm_config_override from artemis.sdk.utils import load_llm_config_override
from pydantic import BaseModel, Field
TaskRunStatus = Literal["pending", "running", "completed", "failed", "cancelled"]
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class AgentProfile(BaseModel): class AgentProfile(BaseModel):
"""Represents a ARTEMIS agent profile. """Runtime configuration profile governing LLM model selection and reasoning policies."""
Attributes: model_config = ConfigDict(arbitrary_types_allowed=True)
name: Name of the agent - used to reference the agent when running
tasks.
llm_config: LLM configuration for the agent.
"""
model_config = {"ignored_types": (CyFunctionDetector,)} name: str = Field(description="Identifier for referencing this operational agent profile.")
name: str
llm_config: LLMConfig = Field(default_factory=get_default_llm_config) llm_config: LLMConfig = Field(default_factory=get_default_llm_config)
@overload @overload
def __init__(self, *, name: str, llm_config: LLMConfig): ... def __init__(self, *, name: str, llm_config: LLMConfig): ...
@overload @overload
def __init__(self, *, name: str, from_file: str): ... def __init__(self, *, name: str, from_file: str | Path): ...
def __init__( def __init__(
self, self,
*, *,
name: str, name: str,
llm_config: LLMConfig | None = None, llm_config: LLMConfig | None = None,
from_file: str | None = None, from_file: str | Path | None = None,
**kwargs, **kwargs: Any,
): ):
kwargs["name"] = name kwargs["name"] = name
if from_file: if from_file:
@@ -79,11 +66,11 @@ class AgentProfile(BaseModel):
elif llm_config: elif llm_config:
kwargs["llm_config"] = llm_config kwargs["llm_config"] = llm_config
else: else:
raise ValueError("Either llm_config or from_file must be provided") raise ValueError("Either llm_config or from_file must be specified.")
super().__init__(**kwargs) super().__init__(**kwargs)
def __str__(self): def __str__(self) -> str:
return f"Profile {self.name}:\n{self.llm_config}" return f"AgentProfile({self.name}): {self.llm_config}"
T = TypeVar("T", bound=BaseModel) T = TypeVar("T", bound=BaseModel)
@@ -91,146 +78,73 @@ TOutput = TypeVar("TOutput", bound=BaseModel | None)
class TaskRequestBase(BaseModel): class TaskRequestBase(BaseModel):
"""Defines common parameters of a mobile automation task request.""" """Baseline runtime limits and telemetry recording settings for automation runs."""
model_config = ConfigDict(extra="ignore", arbitrary_types_allowed=True)
model_config = {"ignored_types": (CyFunctionDetector,)}
max_steps: int = RECURSION_LIMIT max_steps: int = RECURSION_LIMIT
record_trace: bool = True record_trace: bool = True
trace_path: Path = Path("traces") trace_path: Path = Field(default_factory=lambda: Path("traces"))
llm_output_path: Path | None = None llm_output_path: Path | None = None
class TaskRequestCommon(TaskRequestBase): class TaskRequestCommon(TaskRequestBase):
"""Defines common parameters for any task request.""" """Common automation constraints including application isolation and pre-install packages."""
model_config = {"ignored_types": (CyFunctionDetector,)}
max_steps: int = RECURSION_LIMIT
locked_app_package: str | None = None locked_app_package: str | None = None
app_path: Path | None = None app_path: Path | None = None
"""Path to an app to install before running the task.
For Android: Path to an APK file.
The app will be installed automatically before the task starts.
"""
class TaskRequest(TaskRequestCommon, Generic[TOutput]): class TaskRequest(TaskRequestCommon, Generic[TOutput]):
"""Defines the format of a mobile automation task request. """Declarative request specifying the high-level mobile testing or automation objective."""
Attributes: goal: str = Field(description="Natural language instruction outlining target behavior or assertions.")
goal: Natural language description of the goal to achieve profile: str | None = Field(default=None, description="Execution profile (e.g. 'flash' or 'pro').")
profile: Optional agent profile to use for executing the task task_name: str | None = Field(default=None, description="Optional label identifying this task execution.")
task_name: Optional name for the task output_description: str | None = Field(default=None, description="Formatting prompt for expected deliverables.")
output_description: Optional natural language description of expected output_format: type[TOutput] | None = Field(default=None, description="Target Pydantic schema for structured output.")
output format
output_format: Optional pydantic model for the output format of the task
max_steps: Maximum number of steps the agent can take (default: 20)
record_trace: Whether to record a trace (screenshots, actions) of the
execution (default: False)
trace_path: Directory path to save trace data if recording is enabled
llm_output_path: Path to save LLM output data
"""
model_config = {"ignored_types": (CyFunctionDetector,)}
goal: str
profile: str | None = None
task_name: str | None = None
output_description: str | None = None
output_format: type[TOutput] | None = None
enable_remote_tracing: bool = False enable_remote_tracing: bool = False
class TaskResult(BaseModel): class TaskResult(BaseModel):
"""Result of a mobile automation task. """Immutable outcome payload produced upon automation task completion."""
Attributes: model_config = ConfigDict(arbitrary_types_allowed=True)
content: Raw result content (could be text or structured data)
error: Error message if the task failed
execution_time_seconds: How long the task took to execute
steps_taken: Number of steps the agent took to complete the task
"""
model_config = {"ignored_types": (CyFunctionDetector,)}
content: Any = None content: Any = None
error: str | None = None error: str | None = None
execution_time_seconds: float execution_time_seconds: float = 0.0
steps_taken: int steps_taken: int = 0
@property
def succeeded(self) -> bool:
"""True if the task terminated without error."""
return self.error is None
def get_as_model(self, model_class: type[T]) -> T: def get_as_model(self, model_class: type[T]) -> T:
"""Parse the content into a Pydantic model instance. """Parse structured content into the requested Pydantic model instance."""
Args:
model_class: The Pydantic model class to parse the data into
Returns:
An instance of the specified model class
Raises:
ValueError: If content is None or not compatible with the model
"""
if self.content is None: if self.content is None:
raise ValueError("No content available to parse into a model") raise ValueError("No content available to parse into the requested schema.")
if isinstance(self.content, model_class): if isinstance(self.content, model_class):
return self.content return self.content
return model_class.model_validate(self.content) return model_class.model_validate(self.content)
class Task(BaseModel): class Task(BaseModel):
"""A mobile automation task to be executed. """Stateful execution container tracking lifecycle progress, status transitions, and final results."""
Attributes: model_config = ConfigDict(arbitrary_types_allowed=True)
id: Unique identifier for the task
device: Information about the target device
status: Current status of the task execution
request: User task request
created_at: ISO timestamp when the task was created
ended_at: ISO timestamp when the task ended
"""
model_config = {"ignored_types": (CyFunctionDetector,)}
id: str id: str
device: DeviceContext device: DeviceContext
status: TaskRunStatus status: TaskRunStatus = "pending"
status_message: str | None = None status_message: str | None = None
on_status_changed: Callable[[TaskRunStatus, str | None, Any | None], Coroutine] | None = None on_status_changed: Callable[[TaskRunStatus, str | None, Any | None], Coroutine] | None = None
request: TaskRequest request: TaskRequest[Any]
created_at: datetime created_at: datetime = Field(default_factory=datetime.now)
ended_at: datetime | None = None ended_at: datetime | None = None
result: TaskResult | None = None result: TaskResult | None = None
async def finalize(
self,
content: Any | None = None,
state: dict | None = None,
error: str | None = None,
cancelled: bool = False,
):
new_status: TaskRunStatus = "completed" if error is None else "failed"
if new_status == "failed" and cancelled:
new_status = "cancelled"
message = "Task completed successfully"
if new_status == "failed":
message = "Task failed" + (f": {error}" if error else "")
elif new_status == "cancelled":
message = "Task cancelled" + (f": {error}" if error else "")
await self.set_status(status=new_status, message=message, output=content or error)
self.ended_at = datetime.now()
duration = self.ended_at - self.created_at
steps_taken = -1
if state is not None:
metadata = state.get("metadata", None)
if metadata:
steps_taken = metadata.get("step_count", -1)
self.result = TaskResult(
content=content,
error=error,
execution_time_seconds=duration.total_seconds(),
steps_taken=steps_taken,
)
def get_name(self) -> str: def get_name(self) -> str:
return self.request.task_name or self.id return self.request.task_name or self.id
@@ -239,8 +153,42 @@ class Task(BaseModel):
status: TaskRunStatus, status: TaskRunStatus,
message: str | None = None, message: str | None = None,
output: Any | None = None, output: Any | None = None,
): ) -> None:
self.status = status self.status = status
self.status_message = message self.status_message = message
if self.on_status_changed: if self.on_status_changed is not None:
await self.on_status_changed(status, message, output) await self.on_status_changed(status, message, output)
async def finalize(
self,
content: Any | None = None,
state: dict[str, Any] | None = None,
error: str | None = None,
cancelled: bool = False,
) -> None:
if cancelled:
resolved_status: TaskRunStatus = "cancelled"
default_msg = f"Task cancelled: {error}" if error else "Task cancelled."
elif error:
resolved_status = "failed"
default_msg = f"Task failed: {error}"
else:
resolved_status = "completed"
default_msg = "Task completed successfully."
await self.set_status(status=resolved_status, message=default_msg, output=content or error)
self.ended_at = datetime.now()
elapsed = (self.ended_at - self.created_at).total_seconds()
steps = -1
if state is not None:
metadata = state.get("metadata")
if isinstance(metadata, dict):
steps = metadata.get("step_count", -1)
self.result = TaskResult(
content=content,
error=error,
execution_time_seconds=max(0.0, elapsed),
steps_taken=steps,
)
-3
View File
@@ -46,7 +46,6 @@ from artemis.tools.tool_wrapper import (
invoke_tool_with_injection, invoke_tool_with_injection,
tool_result_messages, tool_result_messages,
) )
from artemis.tools.types import CyFunctionDetector
from artemis.tools.video_tool import get_video_analyzer_tool from artemis.tools.video_tool import get_video_analyzer_tool
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
from artemis.utils.notes import get_note_file_path, get_notes_dir from artemis.utils.notes import get_note_file_path, get_notes_dir
@@ -61,8 +60,6 @@ logger = get_logger(__name__)
class AskCommitteeArgs(BaseModel): class AskCommitteeArgs(BaseModel):
"""Arguments schema for invoking the committee tool.""" """Arguments schema for invoking the committee tool."""
model_config = {"ignored_types": (CyFunctionDetector,)}
avatar_directive: str = Field( avatar_directive: str = Field(
..., ...,
description="Directive to frame and steer the debate for your avatar.", description="Directive to frame and steer the debate for your avatar.",
-3
View File
@@ -27,7 +27,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State from artemis.graph.state import State
from artemis.tools.base import ArtemisTool from artemis.tools.base import ArtemisTool
from artemis.tools.tool_wrapper import ToolWrapper from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
logger = get_logger(__name__) logger = get_logger(__name__)
@@ -36,8 +35,6 @@ logger = get_logger(__name__)
# pylint: disable=too-few-public-methods # pylint: disable=too-few-public-methods
class AskDiagnoserArgs(BaseModel): class AskDiagnoserArgs(BaseModel):
"""Arguments schema for invoking the diagnostic subagent.""" """Arguments schema for invoking the diagnostic subagent."""
model_config = {"ignored_types": (CyFunctionDetector,)}
query: str = Field( query: str = Field(
..., ...,
description=( description=(
-3
View File
@@ -58,7 +58,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.base import ArtemisTool, ToolCategory
from artemis.tools.tool_wrapper import ToolWrapper from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.element_hit_test import find_element_at_point from artemis.utils.element_hit_test import find_element_at_point
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
from artemis.utils.visualization import draw_dots from artemis.utils.visualization import draw_dots
@@ -94,8 +93,6 @@ CANDIDATE_DOT_COLOR = "magenta"
class AskExplorerArgs(BaseModel): class AskExplorerArgs(BaseModel):
"""Arguments of ``ask_explorer`` as seen by every calling agent.""" """Arguments of ``ask_explorer`` as seen by every calling agent."""
model_config = {"ignored_types": (CyFunctionDetector,)}
query: str = Field(..., description=ASK_EXPLORER_QUERY_DESCRIPTION) query: str = Field(..., description=ASK_EXPLORER_QUERY_DESCRIPTION)
context_feedback: str = Field("", description=ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION) context_feedback: str = Field("", description=ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION)
-3
View File
@@ -25,7 +25,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State from artemis.graph.state import State
from artemis.tools.base import ArtemisTool from artemis.tools.base import ArtemisTool
from artemis.tools.tool_wrapper import ToolWrapper from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
logger = get_logger(__name__) logger = get_logger(__name__)
@@ -34,8 +33,6 @@ logger = get_logger(__name__)
# pylint: disable=too-few-public-methods # pylint: disable=too-few-public-methods
class AskVisionCoderArgs(BaseModel): class AskVisionCoderArgs(BaseModel):
"""Arguments schema for invoking the Image Processor subagent.""" """Arguments schema for invoking the Image Processor subagent."""
model_config = {"ignored_types": (CyFunctionDetector,)}
instruction: str = Field( instruction: str = Field(
..., ...,
description=( description=(
-3
View File
@@ -27,7 +27,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State from artemis.graph.state import State
from artemis.tools.base import ArtemisTool from artemis.tools.base import ArtemisTool
from artemis.tools.tool_wrapper import ToolWrapper from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
logger = get_logger(__name__) logger = get_logger(__name__)
@@ -36,8 +35,6 @@ logger = get_logger(__name__)
# pylint: disable=too-few-public-methods # pylint: disable=too-few-public-methods
class AnalyzeLogsArgs(BaseModel): class AnalyzeLogsArgs(BaseModel):
"""Arguments schema for invoking the Log Analyzer subagent.""" """Arguments schema for invoking the Log Analyzer subagent."""
model_config = {"ignored_types": (CyFunctionDetector,)}
specific_query: str = Field( specific_query: str = Field(
..., ...,
description=( description=(
+8 -8
View File
@@ -18,7 +18,10 @@ from langchain_core.messages import ToolMessage
from langchain_core.tools import BaseTool from langchain_core.tools import BaseTool
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
from artemis.agents.hopper.hopper import HopperOutput, hopper from artemis.agents.entity_extractor.entity_extractor import (
ExtractionResult,
extract_entity,
)
from artemis.context import ArtemisContext from artemis.context import ArtemisContext
from artemis.controllers.platform_specific_commands_controller import ( from artemis.controllers.platform_specific_commands_controller import (
list_packages_async, list_packages_async,
@@ -28,7 +31,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.base import ArtemisTool, ToolCategory
from artemis.tools.tool_wrapper import ToolWrapper from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.app_launch_utils import launch_app_with_retries from artemis.utils.app_launch_utils import launch_app_with_retries
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
@@ -37,8 +39,6 @@ logger = get_logger(__name__)
class LaunchAppArgs(BaseModel): class LaunchAppArgs(BaseModel):
"""Arguments schema for launching an application.""" """Arguments schema for launching an application."""
model_config = {"ignored_types": (CyFunctionDetector,)}
app_name: str = Field( app_name: str = Field(
..., ...,
description="The natural language name of the application to launch.", description="The natural language name of the application to launch.",
@@ -77,21 +77,21 @@ async def find_package(ctx: ArtemisContext, app_name: str, use_fallback: bool =
package_cache[app_name] = app_name package_cache[app_name] = app_name
return app_name return app_name
hopper_output: HopperOutput = await hopper( extractor_output: ExtractionResult = await extract_entity(
ctx=ctx, ctx=ctx,
request=(f"I'm looking for the package name of the following app: '{app_name}'"), request=(f"I'm looking for the package name of the following app: '{app_name}'"),
data=all_packages, data=all_packages,
use_fallback=use_fallback, use_fallback=use_fallback,
) )
if not hopper_output.found or not hopper_output.output: if not extractor_output.found or not extractor_output.output:
if isinstance(package_cache, dict): if isinstance(package_cache, dict):
package_cache[app_name] = None package_cache[app_name] = None
return None return None
package_name = hopper_output.output.strip() package_name = extractor_output.output.strip()
if package_name not in package_set: if package_name not in package_set:
logger.warning( logger.warning(
f"Hopper returned package '{package_name}' for '{app_name}', " f"Entity extractor returned package '{package_name}' for '{app_name}', "
"but it is NOT physically installed on the device!" "but it is NOT physically installed on the device!"
) )
if isinstance(package_cache, dict): if isinstance(package_cache, dict):
-5
View File
@@ -25,7 +25,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.base import ArtemisTool, ToolCategory
from artemis.tools.tool_wrapper import ToolWrapper from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
logger = get_logger(__name__) logger = get_logger(__name__)
@@ -33,10 +32,6 @@ logger = get_logger(__name__)
class GetUiHierarchyArgs(BaseModel): class GetUiHierarchyArgs(BaseModel):
"""Arguments schema for getting UI hierarchy.""" """Arguments schema for getting UI hierarchy."""
model_config = {"ignored_types": (CyFunctionDetector,)}
GET_UI_HIERARCHY_DOCSTRING = ( GET_UI_HIERARCHY_DOCSTRING = (
"[DIAGNOSTIC] Retrieves the current screen UI hierarchy XML using a persistent" "[DIAGNOSTIC] Retrieves the current screen UI hierarchy XML using a persistent"
" UI Automator client.\n\nUse this instead of running 'uiautomator dump'" " UI Automator client.\n\nUse this instead of running 'uiautomator dump'"
-5
View File
@@ -35,7 +35,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.base import ArtemisTool, ToolCategory
from artemis.tools.tool_wrapper import ToolWrapper from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
logger = get_logger(__name__) logger = get_logger(__name__)
@@ -44,8 +43,6 @@ logger = get_logger(__name__)
# pylint: disable=too-few-public-methods # pylint: disable=too-few-public-methods
class ObjectDetectionArgs(BaseModel): class ObjectDetectionArgs(BaseModel):
"""Arguments schema for object detection with explicit screenshot path.""" """Arguments schema for object detection with explicit screenshot path."""
model_config = {"ignored_types": (CyFunctionDetector,)}
image_path: str = Field(..., description="Absolute path to the screenshot file.") image_path: str = Field(..., description="Absolute path to the screenshot file.")
queries: list[str] = Field(..., description="A list of single query strings to find.") queries: list[str] = Field(..., description="A list of single query strings to find.")
@@ -53,8 +50,6 @@ class ObjectDetectionArgs(BaseModel):
# pylint: disable=too-few-public-methods # pylint: disable=too-few-public-methods
class OperatorObjectDetectionArgs(BaseModel): class OperatorObjectDetectionArgs(BaseModel):
"""Arguments schema for operator object detection using latest screenshot from state.""" """Arguments schema for operator object detection using latest screenshot from state."""
model_config = {"ignored_types": (CyFunctionDetector,)}
queries: list[str] = Field(..., description="A list of single query strings to find.") queries: list[str] = Field(..., description="A list of single query strings to find.")
-13
View File
@@ -25,7 +25,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State from artemis.graph.state import State
from artemis.tools.base import ArtemisTool from artemis.tools.base import ArtemisTool
from artemis.tools.tool_wrapper import ToolWrapper from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
from artemis.utils.notes import ( from artemis.utils.notes import (
APPEND_NOTE_ARG_CONTENT_DESC, APPEND_NOTE_ARG_CONTENT_DESC,
@@ -57,24 +56,18 @@ logger = get_logger(__name__)
class SaveNoteArgs(BaseModel): class SaveNoteArgs(BaseModel):
"""Arguments schema for saving notes.""" """Arguments schema for saving notes."""
model_config = {"ignored_types": (CyFunctionDetector,)}
key: str = Field(..., description=SAVE_NOTE_ARG_KEY_DESC) key: str = Field(..., description=SAVE_NOTE_ARG_KEY_DESC)
content: str = Field(..., description=SAVE_NOTE_ARG_CONTENT_DESC) content: str = Field(..., description=SAVE_NOTE_ARG_CONTENT_DESC)
class AppendNoteArgs(BaseModel): class AppendNoteArgs(BaseModel):
"""Arguments schema for appending notes.""" """Arguments schema for appending notes."""
model_config = {"ignored_types": (CyFunctionDetector,)}
key: str = Field(..., description=APPEND_NOTE_ARG_KEY_DESC) key: str = Field(..., description=APPEND_NOTE_ARG_KEY_DESC)
content: str = Field(..., description=APPEND_NOTE_ARG_CONTENT_DESC) content: str = Field(..., description=APPEND_NOTE_ARG_CONTENT_DESC)
class UpdateNoteArgs(BaseModel): class UpdateNoteArgs(BaseModel):
"""Arguments schema for updating notes.""" """Arguments schema for updating notes."""
model_config = {"ignored_types": (CyFunctionDetector,)}
key: str = Field(..., description=UPDATE_NOTE_ARG_KEY_DESC) key: str = Field(..., description=UPDATE_NOTE_ARG_KEY_DESC)
target: str = Field(..., description=UPDATE_NOTE_ARG_TARGET_DESC) target: str = Field(..., description=UPDATE_NOTE_ARG_TARGET_DESC)
replacement: str = Field(..., description=UPDATE_NOTE_ARG_REPLACEMENT_DESC) replacement: str = Field(..., description=UPDATE_NOTE_ARG_REPLACEMENT_DESC)
@@ -82,8 +75,6 @@ class UpdateNoteArgs(BaseModel):
class ReadNoteArgs(BaseModel): class ReadNoteArgs(BaseModel):
"""Arguments schema for reading notes.""" """Arguments schema for reading notes."""
model_config = {"ignored_types": (CyFunctionDetector,)}
key: str = Field(..., description=READ_NOTE_ARG_KEY_DESC) key: str = Field(..., description=READ_NOTE_ARG_KEY_DESC)
start_line: int | None = Field(None, description="Start line to read (1-indexed, inclusive)") start_line: int | None = Field(None, description="Start line to read (1-indexed, inclusive)")
end_line: int | None = Field(None, description="End line to read (1-indexed, inclusive)") end_line: int | None = Field(None, description="End line to read (1-indexed, inclusive)")
@@ -91,10 +82,6 @@ class ReadNoteArgs(BaseModel):
class ListNotesArgs(BaseModel): class ListNotesArgs(BaseModel):
"""Arguments schema for listing notes.""" """Arguments schema for listing notes."""
model_config = {"ignored_types": (CyFunctionDetector,)}
class SaveNoteTool(ArtemisTool): class SaveNoteTool(ArtemisTool):
"""Universal tool for saving notes to persistent memory.""" """Universal tool for saving notes to persistent memory."""
-5
View File
@@ -31,13 +31,8 @@ from artemis.context import ArtemisContext
from artemis.data_engine import engine as engine_mod from artemis.data_engine import engine as engine_mod
from artemis.data_engine.trace import CURRENT_TRACE_ID, smart_serialize from artemis.data_engine.trace import CURRENT_TRACE_ID, smart_serialize
from artemis.graph.state import State from artemis.graph.state import State
from artemis.tools.types import CyFunctionDetector
class ToolWrapper(BaseModel): class ToolWrapper(BaseModel):
"""Wrapper holding a tool factory and lifecycle callbacks.""" """Wrapper holding a tool factory and lifecycle callbacks."""
model_config = {"ignored_types": (CyFunctionDetector,)}
tool_fn_getter: Callable[[ArtemisContext], BaseTool] tool_fn_getter: Callable[[ArtemisContext], BaseTool]
on_success_fn: Callable[..., str] on_success_fn: Callable[..., str]
on_failure_fn: Callable[..., str] on_failure_fn: Callable[..., str]
+24 -38
View File
@@ -12,61 +12,47 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from pydantic import BaseModel, Field, model_validator """Multi-strategy target element descriptors for ARTEMIS actions.
Encapsulates semantic identifiers, textual queries, and spatial bounding boxes
into a prioritized target resolution contract.
"""
from __future__ import annotations
from pydantic import BaseModel, ConfigDict, Field, model_validator
from artemis.utils.ui_hierarchy import ElementBounds from artemis.utils.ui_hierarchy import ElementBounds
class _CyFunctionDetectorMeta(type):
"""Metaclass for detecting cython compiled functions."""
def __instancecheck__(cls, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
# pylint: disable=too-few-public-methods
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
"""Detector for cython compiled functions."""
class Target(BaseModel): class Target(BaseModel):
"""A comprehensive locator for a UI element, supporting a fallback mechanism.""" """Composite locator descriptor resolving UI nodes across dynamic and visual channels."""
model_config = {"ignored_types": (CyFunctionDetector,)} model_config = ConfigDict(extra="ignore")
resource_id: str | None = Field(None, description="The resource-id of the element.") resource_id: str | None = Field(default=None, description="Android resource identifier token.")
resource_id_index: int | None = Field( resource_id_index: int | None = Field(
None, default=None,
description=("The zero-based index if multiple elements share the same resource-id."), description="Zero-based ordinal index when multiple nodes match resource_id.",
) )
text: str | None = Field( text: str | None = Field(
None, default=None,
description=("The text content of the element (e.g., a label or placeholder)."), description="Exact or case-insensitive element label or content descriptor.",
) )
text_index: int | None = Field( text_index: int | None = Field(
None, default=None,
description=("The zero-based index if multiple elements share the same text."), description="Zero-based ordinal index when multiple nodes match text query.",
) )
bounds: ElementBounds | None = Field( bounds: ElementBounds | None = Field(
None, description="The x, y, width, and height of the element." default=None,
description="Spatial boundary metrics (x, y, width, height) of the element.",
) )
@model_validator(mode="after") @model_validator(mode="after")
def _default_indices(self): def _normalize_indices(self) -> Target:
# Treat empty strings like “not provided” """Coerce missing indices to zero if primary selector string is populated."""
if ( if self.resource_id and self.resource_id.strip() and self.resource_id_index is None:
self.resource_id is not None and self.resource_id != ""
) and self.resource_id_index is None:
self.resource_id_index = 0 self.resource_id_index = 0
if (self.text is not None and self.text != "") and self.text_index is None: if self.text and self.text.strip() and self.text_index is None:
self.text_index = 0 self.text_index = 0
return self return self
+207 -277
View File
@@ -11,25 +11,29 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
"""Spatial positioning, focus resolution, and interaction helpers for ARTEMIS tools.
Coordinates locator resolution across hierarchy trees, validates bounded screen areas,
and routes focus and cursor placement to target elements.
"""
from __future__ import annotations from __future__ import annotations
from typing import Literal from typing import Any, Literal
from pydantic import BaseModel, ConfigDict from pydantic import BaseModel, ConfigDict
from artemis.context import ArtemisContext from artemis.context import ArtemisContext
from artemis.controllers.types import ( from artemis.controllers.types import CoordinatesSelectorRequest, PercentagesSelectorRequest
CoordinatesSelectorRequest,
PercentagesSelectorRequest,
)
from artemis.controllers.unified_controller import UnifiedMobileController from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.graph.state import State from artemis.graph.state import State
from artemis.tools.types import Target from artemis.tools.types import Target
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
from artemis.utils.ui_hierarchy import ( from artemis.utils.ui_hierarchy import (
ElementBounds, ElementBounds,
Point,
find_element_by_resource_id, find_element_by_resource_id,
get_bounds_for_element, get_bounds_for_element,
get_element_text, get_element_text,
@@ -39,293 +43,54 @@ from artemis.utils.ui_hierarchy import (
logger = get_logger(__name__) logger = get_logger(__name__)
def find_element_by_text(
ui_hierarchy: list[dict], text: str, index: int | None = None
) -> dict | None:
"""Find a UI element by its text content (adapted to both flat and rich hierarchy)
This function performs a recursive, case-insensitive search.
Args:
ui_hierarchy: List of UI element dictionaries.
text: The text content to search for.
index: Optional index to select nth matching element.
Returns:
The complete UI element dictionary if found, None otherwise.
"""
matches: list[dict] = []
def search_recursive(elements: list[dict]) -> None:
for element in elements:
if isinstance(element, dict):
src = element.get("attributes", element)
element_text = src.get("text", "")
# Guard against non-string values
if not isinstance(element_text, str):
element_text = ""
text_lower = text.lower()
if element_text and text_lower == element_text.lower():
matches.append(element)
if children := element.get("children", []):
search_recursive(children)
search_recursive(ui_hierarchy)
if not matches:
return None
if index is None:
idx = 0
elif index < 0:
return None
else:
idx = index
if idx < len(matches):
return matches[idx]
return None
async def tap_bottom_right_of_element(bounds: ElementBounds, ctx: ArtemisContext):
"""Tap the bottom-right corner of the specified element bounds."""
bottom_right: Point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99)
await tap(
ctx=ctx,
selector_request=SelectorRequestWithCoordinates(
coordinates=CoordinatesSelectorRequest(
x=bottom_right.x,
y=bottom_right.y,
),
),
)
# pylint: disable=too-many-return-statements
async def move_cursor_to_end_if_bounds(
ctx: ArtemisContext,
state: State,
target: Target,
elt: dict | None = None,
) -> dict | None:
"""Best-effort move of the text cursor near the end of the input by tapping the
bottom-right area of the focused element (if bounds are available).
"""
if target.resource_id:
if not elt:
elt = find_element_by_resource_id(
ui_hierarchy=state.latest_ui_hierarchy or [],
resource_id=target.resource_id,
index=target.resource_id_index,
)
if not elt:
return None
bounds = get_bounds_for_element(elt)
if not bounds:
return elt
logger.debug("Tapping near the end of the input to move the cursor")
await tap_bottom_right_of_element(bounds=bounds, ctx=ctx)
logger.debug(f"Tapped end of input {target.resource_id}")
return elt
if target.bounds:
await tap_bottom_right_of_element(target.bounds, ctx=ctx)
logger.debug("Tapped end of input by coordinates")
return elt
if target.text:
text_elt = find_element_by_text(
state.latest_ui_hierarchy or [],
target.text,
index=target.text_index,
)
if text_elt:
bounds = get_bounds_for_element(text_elt)
if bounds:
await tap_bottom_right_of_element(bounds=bounds, ctx=ctx)
logger.debug(f"Tapped end of input that had text'{target.text}'")
return text_elt
return None
return None
async def focus_element_if_needed(
ctx: ArtemisContext, target: Target
) -> Literal["resource_id", "coordinates", "text", "already_focused"] | None:
"""Ensures the element is focused, with a sanity check to prevent trusting misleading IDs.
If no target locator is provided (empty target), assumes the field is
already focused
and returns "already_focused" to allow typing without re-focusing.
"""
# If no locator is provided, assume already focused (e.g., keyboard is visible)
if not target.resource_id and not target.bounds and not target.text:
logger.debug("No target locator provided, assuming element is already focused")
return "already_focused"
controller = UnifiedMobileController(ctx)
rich_hierarchy = await controller.get_ui_elements()
elt_from_id = None
if target.resource_id:
elt_from_id = find_element_by_resource_id(
ui_hierarchy=rich_hierarchy,
resource_id=target.resource_id,
index=target.resource_id_index,
is_rich_hierarchy=False,
)
if elt_from_id and target.text:
text_from_id_elt = get_element_text(elt_from_id)
if not text_from_id_elt or target.text.lower() != text_from_id_elt.lower():
logger.warning(
f"ID '{target.resource_id}' and text '{target.text}' seem to be"
" on different elements. Ignoring the resource_id and falling"
" back to other locators."
)
elt_from_id = None
if elt_from_id:
if not is_element_focused(elt_from_id):
await tap(
ctx=ctx,
selector_request=IdSelectorRequest(id=target.resource_id), # type: ignore
index=target.resource_id_index,
)
logger.debug(f"Focused (tap) on resource_id={target.resource_id}")
rich_hierarchy = await controller.get_ui_elements()
elt_from_id = find_element_by_resource_id(
ui_hierarchy=rich_hierarchy,
resource_id=target.resource_id, # type: ignore
index=target.resource_id_index,
is_rich_hierarchy=False,
)
if elt_from_id and is_element_focused(elt_from_id):
logger.debug(f"Text input is focused: {target.resource_id}")
return "resource_id"
logger.warning(f"Failed to focus using resource_id='{target.resource_id}'. Fallback...")
if target.bounds:
relative_point = target.bounds.get_center()
await tap(
ctx=ctx,
selector_request=SelectorRequestWithCoordinates(
coordinates=CoordinatesSelectorRequest(x=relative_point.x, y=relative_point.y)
),
)
logger.debug(f"Tapped on coordinates ({relative_point.x}, {relative_point.y}) to focus.")
return "coordinates"
if target.text:
text_elt = find_element_by_text(rich_hierarchy, target.text, index=target.text_index)
if text_elt:
bounds = get_bounds_for_element(text_elt)
if bounds:
relative_point = bounds.get_center()
await tap(
ctx=ctx,
selector_request=SelectorRequestWithCoordinates(
coordinates=CoordinatesSelectorRequest(
x=relative_point.x, y=relative_point.y
)
),
)
logger.debug(f"Tapped on text element '{target.text}' to focus.")
return "text"
logger.error(
"Failed to focus element. No valid locator (resource_id, coordinates, or text) succeeded."
)
return None
def validate_coordinates_bounds(
target: Target, screen_width: int, screen_height: int
) -> str | None:
"""Validate that coordinates are within screen bounds.
Returns error message if invalid, None if valid.
"""
if not target.bounds:
return None
center = target.bounds.get_center()
errors = []
if center.x < 0 or center.x >= screen_width:
errors.append(f"x={center.x} is outside screen width (0-{screen_width})")
if center.y < 0 or center.y >= screen_height:
errors.append(f"y={center.y} is outside screen height (0-{screen_height})")
return "; ".join(errors) if errors else None
def has_valid_selectors(target: Target) -> bool:
"""Check if target has at least one valid selector."""
has_coordinates = target.bounds is not None
has_resource_id = target.resource_id is not None and target.resource_id != ""
has_text = target.text is not None and target.text != ""
return has_coordinates or has_resource_id or has_text
class IdSelectorRequest(BaseModel): class IdSelectorRequest(BaseModel):
"""Selector request using resource id.""" """Target selector driven by Android resource ID."""
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True) model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
id: str id: str
def to_dict(self) -> dict[str, str | int]: def to_dict(self) -> dict[str, str | int]:
"""Convert selector to dictionary."""
return {"id": self.id} return {"id": self.id}
class TextSelectorRequest(BaseModel): class TextSelectorRequest(BaseModel):
"""Selector request using element text.""" """Target selector driven by exact or case-insensitive element label."""
model_config = ConfigDict(extra="forbid") model_config = ConfigDict(extra="forbid")
text: str text: str
def to_dict(self) -> dict[str, str | int]: def to_dict(self) -> dict[str, str | int]:
"""Convert selector to dictionary."""
return {"text": self.text} return {"text": self.text}
class SelectorRequestWithCoordinates(BaseModel): class SelectorRequestWithCoordinates(BaseModel):
"""Selector request using absolute coordinates.""" """Target selector pointing to an absolute pixel coordinate."""
model_config = ConfigDict(extra="forbid") model_config = ConfigDict(extra="forbid")
coordinates: CoordinatesSelectorRequest coordinates: CoordinatesSelectorRequest
def to_dict(self) -> dict[str, str | int]: def to_dict(self) -> dict[str, str | int]:
"""Convert selector to dictionary."""
return {"point": self.coordinates.to_str()} return {"point": self.coordinates.to_str()}
class SelectorRequestWithPercentages(BaseModel): class SelectorRequestWithPercentages(BaseModel):
"""Selector request using percentage coordinates.""" """Target selector pointing to normalized fractional [0.0, 1.0] coordinates."""
model_config = ConfigDict(extra="forbid") model_config = ConfigDict(extra="forbid")
percentages: PercentagesSelectorRequest percentages: PercentagesSelectorRequest
def to_dict(self) -> dict[str, str | int]: def to_dict(self) -> dict[str, str | int]:
"""Convert selector to dictionary."""
return {"point": self.percentages.to_str()} return {"point": self.percentages.to_str()}
class IdWithTextSelectorRequest(BaseModel): class IdWithTextSelectorRequest(BaseModel):
"""Selector request using both resource id and text.""" """Composite selector pairing resource ID with expected text."""
model_config = ConfigDict(extra="forbid") model_config = ConfigDict(extra="forbid")
id: str id: str
text: str text: str
def to_dict(self) -> dict[str, str | int]: def to_dict(self) -> dict[str, str | int]:
"""Convert selector to dictionary."""
return {"id": self.id, "text": self.text} return {"id": self.id, "text": self.text}
@@ -341,37 +106,30 @@ SelectorRequest = (
def _extract_resource_id_and_text_from_selector( def _extract_resource_id_and_text_from_selector(
selector: SelectorRequest, selector: SelectorRequest,
) -> tuple[str | None, str | None]: ) -> tuple[str | None, str | None]:
"""Extract resource_id and text from a selector.""" """Unpack identifier and textual query from any supported SelectorRequest."""
resource_id = None
text = None
if isinstance(selector, IdSelectorRequest): if isinstance(selector, IdSelectorRequest):
resource_id = selector.id return selector.id, None
elif isinstance(selector, TextSelectorRequest): if isinstance(selector, TextSelectorRequest):
text = selector.text return None, selector.text
elif isinstance(selector, IdWithTextSelectorRequest): if isinstance(selector, IdWithTextSelectorRequest):
resource_id = selector.id return selector.id, selector.text
text = selector.text return None, None
return resource_id, text
async def tap( async def tap(
ctx: ArtemisContext, ctx: ArtemisContext,
selector_request: SelectorRequest, selector_request: SelectorRequest,
index: int | None = None, index: int | None = None,
): ) -> Any:
"""Tap on a selector. """Dispatch an on-screen tap to an element or coordinate selector."""
Index is optional and is used when you have multiple views matching the same
selector. ui_hierarchy is optional and used for ADB taps to find elements.
"""
controller = UnifiedMobileController(ctx) controller = UnifiedMobileController(ctx)
if isinstance(selector_request, SelectorRequestWithCoordinates): if isinstance(selector_request, SelectorRequestWithCoordinates):
result = await controller.tap_at( res = await controller.tap_at(
x=selector_request.coordinates.x, y=selector_request.coordinates.y x=selector_request.coordinates.x,
y=selector_request.coordinates.y,
) )
return result.error if result.error else None return res.error if res.error else None
if isinstance(selector_request, SelectorRequestWithPercentages): if isinstance(selector_request, SelectorRequestWithPercentages):
coords = selector_request.percentages.to_coords( coords = selector_request.percentages.to_coords(
width=ctx.device.device_width, width=ctx.device.device_width,
@@ -379,11 +137,183 @@ async def tap(
) )
return await controller.tap_at(coords.x, coords.y) return await controller.tap_at(coords.x, coords.y)
# For other selectors, we need the UI hierarchy res_id, txt = _extract_resource_id_and_text_from_selector(selector_request)
resource_id, text = _extract_resource_id_and_text_from_selector(selector_request)
return await controller.tap_element( return await controller.tap_element(
resource_id=resource_id, resource_id=res_id,
text=text, text=txt,
index=index if index is not None else 0, index=index if index is not None else 0,
) )
def find_element_by_text(
ui_hierarchy: list[dict[str, Any]], text: str, index: int | None = None
) -> dict[str, Any] | None:
"""Search for elements matching label text across nested hierarchy tree nodes."""
target_text = (text or "").strip().lower()
if not target_text:
return None
matched_nodes: list[dict[str, Any]] = []
def _traverse(nodes: list[dict[str, Any]]) -> None:
for node in nodes:
if not isinstance(node, dict):
continue
attrs = node.get("attributes", node)
val = attrs.get("text")
if isinstance(val, str) and val.strip().lower() == target_text:
matched_nodes.append(attrs if "attributes" in node else node)
children = node.get("children")
if isinstance(children, list) and children:
_traverse(children)
_traverse(ui_hierarchy)
if not matched_nodes:
return None
target_idx = max(0, index or 0)
if target_idx < len(matched_nodes):
return matched_nodes[target_idx]
return None
async def tap_bottom_right_of_element(bounds: ElementBounds, ctx: ArtemisContext) -> None:
"""Position pointer near the trailing boundary of an input element."""
corner_point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99)
await tap(
ctx=ctx,
selector_request=SelectorRequestWithCoordinates(
coordinates=CoordinatesSelectorRequest(x=corner_point.x, y=corner_point.y)
),
)
async def move_cursor_to_end_if_bounds(
ctx: ArtemisContext,
state: State,
target: Target,
elt: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Position insertion point at the end of an input element boundary."""
current_tree = state.latest_ui_hierarchy or []
if target.resource_id:
element_node = elt or find_element_by_resource_id(
ui_hierarchy=current_tree,
resource_id=target.resource_id,
index=target.resource_id_index,
)
if not element_node:
return None
box = get_bounds_for_element(element_node)
if not box:
return element_node
await tap_bottom_right_of_element(bounds=box, ctx=ctx)
return element_node
if target.bounds:
await tap_bottom_right_of_element(bounds=target.bounds, ctx=ctx)
return elt
if target.text:
matched = find_element_by_text(current_tree, target.text, index=target.text_index)
if matched:
box = get_bounds_for_element(matched)
if box:
await tap_bottom_right_of_element(bounds=box, ctx=ctx)
return matched
return None
return None
async def focus_element_if_needed(
ctx: ArtemisContext, target: Target
) -> Literal["resource_id", "coordinates", "text", "already_focused"] | None:
"""Ensure element possesses active input focus, falling back through available selectors."""
if not target.resource_id and not target.bounds and not target.text:
return "already_focused"
controller = UnifiedMobileController(ctx)
hierarchy = await controller.get_ui_elements()
element_by_id = None
if target.resource_id:
element_by_id = find_element_by_resource_id(
ui_hierarchy=hierarchy,
resource_id=target.resource_id,
index=target.resource_id_index,
is_rich_hierarchy=False,
)
if element_by_id and target.text:
observed_label = get_element_text(element_by_id)
if not observed_label or target.text.strip().lower() != observed_label.strip().lower():
logger.warning(
f"ID '{target.resource_id}' and text '{target.text}' do not align to the same node."
" Falling back to secondary selectors."
)
element_by_id = None
if element_by_id:
if not is_element_focused(element_by_id):
await tap(
ctx=ctx,
selector_request=IdSelectorRequest(id=target.resource_id), # type: ignore[arg-type]
index=target.resource_id_index,
)
hierarchy = await controller.get_ui_elements()
element_by_id = find_element_by_resource_id(
ui_hierarchy=hierarchy,
resource_id=target.resource_id, # type: ignore[arg-type]
index=target.resource_id_index,
is_rich_hierarchy=False,
)
if element_by_id and is_element_focused(element_by_id):
return "resource_id"
if target.bounds:
centroid = target.bounds.get_center()
await tap(
ctx=ctx,
selector_request=SelectorRequestWithCoordinates(
coordinates=CoordinatesSelectorRequest(x=centroid.x, y=centroid.y)
),
)
return "coordinates"
if target.text:
matched_text_node = find_element_by_text(hierarchy, target.text, index=target.text_index)
if matched_text_node:
text_box = get_bounds_for_element(matched_text_node)
if text_box:
centroid = text_box.get_center()
await tap(
ctx=ctx,
selector_request=SelectorRequestWithCoordinates(
coordinates=CoordinatesSelectorRequest(x=centroid.x, y=centroid.y)
),
)
return "text"
logger.error("Failed to focus element. No valid locator (resource_id, coordinates, or text) succeeded.")
return None
def validate_coordinates_bounds(target: Target, screen_width: int, screen_height: int) -> str | None:
"""Verify target centroid resides within active viewport boundaries."""
if not target.bounds:
return None
c = target.bounds.get_center()
errors: list[str] = []
if not (0 <= c.x < screen_width):
errors.append(f"x={c.x} is outside screen width (0-{screen_width})")
if not (0 <= c.y < screen_height):
errors.append(f"y={c.y} is outside screen height (0-{screen_height})")
return "; ".join(errors) if errors else None
def has_valid_selectors(target: Target) -> bool:
"""Verify target defines at least one valid locator criterion."""
return bool(target.bounds is not None or target.resource_id or target.text)
-3
View File
@@ -34,7 +34,6 @@ from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.base import ArtemisTool, ToolCategory
from artemis.tools.tool_wrapper import ToolWrapper from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
logger = get_logger(__name__) logger = get_logger(__name__)
@@ -42,8 +41,6 @@ logger = get_logger(__name__)
class VideoAnalyzerArgs(BaseModel): class VideoAnalyzerArgs(BaseModel):
"""Arguments schema for video analyzer tool.""" """Arguments schema for video analyzer tool."""
model_config = {"ignored_types": (CyFunctionDetector,)}
time_description: str = Field( time_description: str = Field(
..., ...,
description=( description=(
-3
View File
@@ -23,7 +23,6 @@ from artemis.data_engine.trace import trace_langchain_tool
from artemis.drivers.base import BaseDeviceDriver from artemis.drivers.base import BaseDeviceDriver
from artemis.tools.base import ArtemisTool, ToolCategory from artemis.tools.base import ArtemisTool, ToolCategory
from artemis.tools.tool_wrapper import ToolWrapper from artemis.tools.tool_wrapper import ToolWrapper
from artemis.tools.types import CyFunctionDetector
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
logger = get_logger(__name__) logger = get_logger(__name__)
@@ -31,8 +30,6 @@ logger = get_logger(__name__)
class WaitArgs(BaseModel): class WaitArgs(BaseModel):
"""Arguments schema for wait tool.""" """Arguments schema for wait tool."""
model_config = {"ignored_types": (CyFunctionDetector,)}
seconds: int = Field( seconds: int = Field(
..., ...,
description=("The duration to wait/sleep in seconds. Minimum is 1, maximum is 60."), description=("The duration to wait/sleep in seconds. Minimum is 1, maximum is 60."),
+6 -3
View File
@@ -11,6 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
"""Utilities for handling app locking and initial app launch logic.""" """Utilities for handling app locking and initial app launch logic."""
@@ -216,7 +219,7 @@ async def _poll_for_app_ready(
Only fails if we get a different (non-null) package or timeout. Only fails if we get a different (non-null) package or timeout.
Args: Args:
ctx: Mobile use context ctx: ArtemisContext
app_package: Expected package name app_package: Expected package name
max_poll_seconds: Maximum time to poll (default: 15s) max_poll_seconds: Maximum time to poll (default: 15s)
poll_interval: Time between polls (default: 1s) poll_interval: Time between polls (default: 1s)
@@ -272,7 +275,7 @@ async def launch_app_with_retries(
"""Launch an app with retry logic and smart polling. """Launch an app with retry logic and smart polling.
Args: Args:
ctx: Mobile use context ctx: ArtemisContext
app_package: Package name (Android) to launch app_package: Package name (Android) to launch
max_retries: Maximum number of launch attempts (default: 3) max_retries: Maximum number of launch attempts (default: 3)
max_poll_seconds: Maximum time to wait for app to load per attempt max_poll_seconds: Maximum time to wait for app to load per attempt
@@ -345,7 +348,7 @@ async def _handle_initial_app_launch(
3. Return status with success/error information 3. Return status with success/error information
Args: Args:
ctx: Mobile use context ctx: ArtemisContext
locked_app_package: Package name (Android) to lock to locked_app_package: Package name (Android) to lock to
Returns: Returns:
-38
View File
@@ -1,38 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from adbutils import AdbClient
from rich.console import Console
def display_device_status(console: Console, adb_client: AdbClient | None = None):
"""Checks for connected devices and displays the status."""
console.print("\n[bold]📱 Device Status[/bold]")
devices = None
if adb_client is not None:
devices = adb_client.device_list()
if devices:
console.print("✅ [bold green]Android device(s) connected:[/bold green]")
for device in devices:
console.print(f" - {device.serial}")
else:
console.print("❌ [bold red]No Android device found.[/bold red]")
command = "emulator -avd <avd_name>"
if sys.platform not in ["win32", "darwin"]:
command = f"./{command}"
console.print(
f"You can start an emulator using a command like: [bold]'{command}'[/bold]"
)
-154
View File
@@ -1,154 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import inquirer
from rich.console import Console
from rich.prompt import Prompt
from rich.text import Text
def select_provider_and_model(
console: Console,
available_providers: list[str],
available_models: dict,
default_provider: str,
default_model: str,
provider: str | None = None,
model: str | None = None,
) -> tuple[str, str]:
"""Interactive selection of LLM provider and model with arrow-key dropdowns when available.
Args:
console: Rich console for output
available_providers: List of available provider names
available_models: Dict mapping providers to their available models
default_provider: Default provider to use
default_model: Default model to use
provider: Pre-selected provider (optional)
model: Pre-selected model (optional)
Returns:
Tuple of (selected_provider, selected_model)
"""
final_provider = provider
final_model = model
# Interactive provider selection
if not final_provider:
console.print("\n🤖 [bold cyan]LLM Provider Selection[/bold cyan]")
final_provider = _select_from_list(
console=console,
item_type="provider",
choices=available_providers,
default=default_provider,
message="Select LLM provider",
)
# Interactive model selection
if not final_model:
console.print(f"\n🎯 [bold green]Model Selection for {final_provider}[/bold green]")
available_model_list = (
available_models[final_provider]
if final_provider
else available_models[default_provider]
)
default_model_for_provider = (
default_model if default_model in available_model_list else available_model_list[0]
)
final_model = _select_from_list(
console=console,
item_type="model",
choices=available_model_list,
default=default_model_for_provider,
message=f"Select model for {final_provider}",
)
return final_provider, final_model
def _select_from_list(
console: Console,
item_type: str,
choices: list[str],
default: str,
message: str,
) -> str:
"""Select an item from a list using arrow keys when available, fallback to numbered selection.
Args:
console: Rich console for output
item_type: Type of item being selected (for error messages)
choices: List of choices to select from
default: Default choice
message: Message to display in dropdown
Returns:
Selected choice
"""
# Try arrow-key dropdown if TTY is available, fallback to numbered selection
if sys.stdin.isatty():
try:
questions = [
inquirer.List(
"selection",
message=f"{message} (use arrow keys)",
choices=choices,
default=default,
),
]
answers = inquirer.prompt(questions)
return answers["selection"] if answers else default
except Exception:
# Fallback to numbered selection
return _numbered_selection(console, item_type, choices, default)
else:
return _numbered_selection(console, item_type, choices, default)
def _numbered_selection(console: Console, item_type: str, choices: list[str], default: str) -> str:
"""Fallback numbered selection when arrow keys aren't available."""
choices_text = "\n".join([f" {i + 1}. {choice}" for i, choice in enumerate(choices)])
console.print(f"Available {item_type}s:\n{choices_text}")
default_idx = choices.index(default) + 1 if default in choices else 1
while True:
choice = Prompt.ask(
f"Select {item_type} (1-{len(choices)}) or press Enter for default",
default=str(default_idx),
)
try:
choice_idx = int(choice) - 1
if 0 <= choice_idx < len(choices):
return choices[choice_idx]
else:
console.print("[red]Invalid choice. Please try again.[/red]")
except ValueError:
console.print("[red]Please enter a number.[/red]")
def display_llm_config(console: Console, provider: str, model: str) -> None:
"""Display the selected LLM configuration with colors."""
config_text = Text()
config_text.append("🤖 LLM Configuration: ", style="bold white")
config_text.append("Provider: ", style="white")
config_text.append(f"{provider}", style="bold cyan")
config_text.append(" | Model: ", style="white")
config_text.append(f"{model}", style="bold green")
console.print(config_text)
-45
View File
@@ -1,45 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TypeGuard
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage
def is_ai_message(message: BaseMessage) -> TypeGuard[AIMessage]:
return isinstance(message, AIMessage)
def is_human_message(message: BaseMessage) -> TypeGuard[HumanMessage]:
return isinstance(message, HumanMessage)
def is_tool_message(message: BaseMessage) -> TypeGuard[ToolMessage]:
return isinstance(message, ToolMessage)
def is_tool_for_name(tool_message: ToolMessage, name: str) -> bool:
return tool_message.name == name
def get_screenshot_message_for_llm(screenshot_base64: str):
prefix = "" if screenshot_base64.startswith("data:image") else "data:image/jpeg;base64,"
return HumanMessage(
content=[
{
"type": "image_url",
"image_url": {"url": f"{prefix}{screenshot_base64}"},
}
]
)
+74 -113
View File
@@ -12,128 +12,89 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import asyncio """Lifecycle callback and execution telemetry decorators for ARTEMIS.
from collections.abc import Awaitable, Callable
Provides function interception hooks for pre-execution logging, success verification,
and error capture across synchronous and asynchronous agent invocations.
"""
from __future__ import annotations
from functools import wraps from functools import wraps
import inspect import inspect
from typing import Any, TypeVar, cast, overload from typing import Any, Callable
R = TypeVar("R")
def wrap_with_callbacks_sync(
fn: Callable[..., R],
*,
before: Callable[..., None] | None = None,
on_success: Callable[[R], None] | None = None,
on_failure: Callable[[Exception], None] | None = None,
suppress_exceptions: bool = False,
) -> Callable[..., R]:
@wraps(fn)
def wrapper(*args: Any, **kwargs: Any) -> R:
if before:
before()
try:
result = fn(*args, **kwargs)
if on_success:
on_success(result)
return result
except Exception as e:
if on_failure:
on_failure(e)
if suppress_exceptions:
return None # type: ignore
raise
return wrapper
def wrap_with_callbacks_async(
fn: Callable[..., Awaitable[R]],
*,
before: Callable[..., None] | None = None,
on_success: Callable[[R], None] | None = None,
on_failure: Callable[[Exception], None] | None = None,
suppress_exceptions: bool = False,
) -> Callable[..., Awaitable[R]]:
@wraps(fn)
async def wrapper(*args: Any, **kwargs: Any) -> R:
if before:
before()
try:
result = await fn(*args, **kwargs)
if on_success:
on_success(result)
return result
except Exception as e:
if on_failure:
on_failure(e)
if suppress_exceptions:
return None # type: ignore
raise
return wrapper
@overload
def wrap_with_callbacks(
fn: Callable[..., Awaitable[R]],
*,
before: Callable[[], None] | None = ...,
on_success: Callable[[R], None] | None = ...,
on_failure: Callable[[Exception], None] | None = ...,
suppress_exceptions: bool = ...,
) -> Callable[..., Awaitable[R]]: ...
@overload
def wrap_with_callbacks(
*,
before: Callable[..., None] | None = ...,
on_success: Callable[[Any], None] | None = ...,
on_failure: Callable[[Exception], None] | None = ...,
suppress_exceptions: bool = ...,
) -> Callable[[Callable[..., R]], Callable[..., R]]: ...
@overload
def wrap_with_callbacks(
fn: Callable[..., R],
*,
before: Callable[[], None] | None = ...,
on_success: Callable[[R], None] | None = ...,
on_failure: Callable[[Exception], None] | None = ...,
suppress_exceptions: bool = ...,
) -> Callable[..., R]: ...
def wrap_with_callbacks( def wrap_with_callbacks(
fn: Callable[..., Any] | None = None, fn: Callable[..., Any] | None = None,
*, *,
before: Callable[[], None] | None = None, before: Callable[..., None] | None = None,
on_success: Callable[[Any], None] | None = None, on_success: Callable[[Any], None] | None = None,
on_failure: Callable[[Exception], None] | None = None, on_failure: Callable[[Exception], None] | None = None,
suppress_exceptions: bool = False, suppress_exceptions: bool = False,
) -> Any: ) -> Any:
def decorator(func: Callable[..., Any]) -> Any: """Wrap a callable with pre-invocation, success, and error callback handlers.
if inspect.iscoroutinefunction(func):
return wrap_with_callbacks_async(
cast(Callable[..., Awaitable[Any]], func),
before=before,
on_success=on_success,
on_failure=on_failure,
suppress_exceptions=suppress_exceptions,
)
else:
return wrap_with_callbacks_sync(
cast(Callable[..., Any], func),
before=before,
on_success=on_success,
on_failure=on_failure,
suppress_exceptions=suppress_exceptions,
)
if fn is None: Supports decorating both coroutines and regular synchronous functions.
return decorator """
else:
return decorator(fn) def _decorator(target_fn: Callable[..., Any]) -> Callable[..., Any]:
if inspect.iscoroutinefunction(target_fn):
@wraps(target_fn)
async def _async_wrapped(*args: Any, **kwargs: Any) -> Any:
if before is not None:
try:
before()
except Exception:
pass
try:
result = await target_fn(*args, **kwargs)
if on_success is not None:
try:
on_success(result)
except Exception:
pass
return result
except Exception as exc:
if on_failure is not None:
try:
on_failure(exc)
except Exception:
pass
if suppress_exceptions:
return None
raise
return _async_wrapped
@wraps(target_fn)
def _sync_wrapped(*args: Any, **kwargs: Any) -> Any:
if before is not None:
try:
before()
except Exception:
pass
try:
result = target_fn(*args, **kwargs)
if on_success is not None:
try:
on_success(result)
except Exception:
pass
return result
except Exception as exc:
if on_failure is not None:
try:
on_failure(exc)
except Exception:
pass
if suppress_exceptions:
return None
raise
return _sync_wrapped
if fn is not None:
return _decorator(fn)
return _decorator
+33 -6
View File
@@ -12,16 +12,43 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""File and configuration loading utilities for ARTEMIS."""
from __future__ import annotations
import json import json
from pathlib import Path
import re import re
from typing import IO from typing import IO, Any
def strip_json_comments(text: str) -> str: def strip_json_comments(text: str) -> str:
text = re.sub(r"//.*?$", "", text, flags=re.MULTILINE) """Removes single-line (//) and multi-line (/* ... */) comments from JSONC text.
text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL)
return text Args:
text: Raw JSON with comments string.
Returns:
Clean JSON string with comments stripped.
"""
pattern = r"//.*?$|/\*.*?\*/"
return re.sub(pattern, "", text, flags=re.MULTILINE | re.DOTALL)
def load_jsonc(file: IO) -> dict: def load_jsonc(source: str | Path | IO[str] | IO[bytes]) -> dict[str, Any]:
return json.loads(strip_json_comments(file.read())) """Loads and parses a JSONC (JSON with Comments) document.
Args:
source: File-like object, Path, or string path to parse.
Returns:
Parsed dictionary.
"""
if isinstance(source, (str, Path)):
raw_content = Path(source).read_text(encoding="utf-8")
else:
content = source.read()
raw_content = content.decode("utf-8") if isinstance(content, bytes) else content
cleaned = strip_json_comments(raw_content)
return json.loads(cleaned)
+3
View File
@@ -11,6 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
from enum import Enum from enum import Enum
import logging import logging
+117 -115
View File
@@ -11,23 +11,33 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
"""Media synthesis and trace asset management for ARTEMIS.
Handles chronological trace GIF generation, frame quantization, and step snapshot
compilation while preserving active session recording manifests.
"""
from __future__ import annotations
import json import json
import os import os
from pathlib import Path from pathlib import Path
import shutil
import subprocess import subprocess
import tempfile import tempfile
from typing import Any
from PIL import Image
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
from PIL import Image
logger = get_logger(__name__) logger = get_logger(__name__)
USE_FFMPEG_GIF = os.environ.get("USE_FFMPEG_GIF", "").lower() in ( _PREFER_FFMPEG = os.environ.get("USE_FFMPEG_GIF", "").strip().lower() in ("1", "true", "yes")
"1",
"true",
"yes",
)
def quantize_and_save_gif_from_paths( def quantize_and_save_gif_from_paths(
@@ -36,81 +46,68 @@ def quantize_and_save_gif_from_paths(
colors: int = 128, colors: int = 128,
duration: int = 100, duration: int = 100,
) -> None: ) -> None:
"""Create an optimized GIF from image file paths. """Compile ordered image snapshots into a quantized animation asset."""
By default uses PIL (loads all frames into memory).
Set USE_FFMPEG_GIF=1 env var to use ffmpeg for memory-efficient streaming.
Args:
image_paths: List of paths to image files (must be sorted in desired
order)
output_path: Path where the GIF will be saved
colors: Number of colors to use in quantization (lower = smaller file)
duration: Duration of each frame in milliseconds
Raises:
ValueError: If image_paths list is empty
RuntimeError: If ffmpeg fails (when USE_FFMPEG_GIF is enabled)
"""
if not image_paths: if not image_paths:
raise ValueError("image_paths must not be empty") raise ValueError("Cannot assemble animation from an empty image sequence.")
if USE_FFMPEG_GIF: if _PREFER_FFMPEG and shutil.which("ffmpeg"):
_save_gif_ffmpeg(image_paths, output_path, colors, duration) try:
else: _render_gif_with_ffmpeg(image_paths, output_path, colors=colors, frame_delay_ms=duration)
_save_gif_pillow(image_paths, output_path, colors, duration) return
except Exception as exc:
logger.warning(f"FFmpeg GIF rendering encountered an error ({exc}); falling back to Pillow.")
_render_gif_with_pillow(image_paths, output_path, colors=colors, frame_delay_ms=duration)
def _save_gif_pillow( def _render_gif_with_pillow(
image_paths: list[Path], paths: list[Path],
output_path: Path, output_path: Path,
colors: int, colors: int,
duration: int, frame_delay_ms: int,
) -> None: ) -> None:
"""Create GIF using PIL (loads all frames into memory).""" """Internal Pillow quantization loop."""
frames: list[Image.Image] = []
for p in paths:
try:
with Image.open(p) as img:
rgb_img = img.convert("RGB") if img.mode != "RGB" else img.copy()
frames.append(rgb_img.quantize(colors=colors, method=Image.Quantize.MEDIANCUT))
except Exception as exc:
logger.debug(f"Skipping damaged frame {p}: {exc}")
def frame_generator(): if not frames:
for path in image_paths: return
with Image.open(path) as img:
if img.mode != "RGB":
img = img.convert("RGB")
yield img.quantize(colors=colors, method=2)
frames = frame_generator() first_frame = frames[0]
first_frame = next(frames)
first_frame.save( first_frame.save(
output_path, output_path,
save_all=True, save_all=True,
append_images=frames, append_images=frames[1:],
loop=0, loop=0,
optimize=True, optimize=True,
duration=duration, duration=frame_delay_ms,
) )
def _save_gif_ffmpeg( def _render_gif_with_ffmpeg(
image_paths: list[Path], paths: list[Path],
output_path: Path, output_path: Path,
colors: int, colors: int,
duration: int, frame_delay_ms: int,
) -> None: ) -> None:
"""Create GIF using ffmpeg (memory-efficient streaming).""" """Streamlined two-pass palette-generated GIF rendering via FFmpeg."""
fps = 1000 / duration fps = max(1.0, 1000.0 / max(1, frame_delay_ms))
with tempfile.TemporaryDirectory() as tmpdir:
list_file = Path(tmpdir) / "frames.txt"
palette = Path(tmpdir) / "palette.png"
with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: lines = [f"file '{p.resolve()}'\nduration {frame_delay_ms / 1000.0}" for p in paths]
for i, path in enumerate(image_paths): if paths:
f.write(f"file '{path.absolute()}'\n") lines.append(f"file '{paths[-1].resolve()}'")
# Last file needs no duration (ffmpeg concat demuxer uses it as the final frame) list_file.write_text("\n".join(lines), encoding="utf-8")
if i < len(image_paths) - 1:
f.write(f"duration {duration / 1000}\n")
# Repeat last file to ensure it's included (ffmpeg concat demuxer quirk)
f.write(f"file '{image_paths[-1].absolute()}'\n")
concat_file = Path(f.name)
palette_path = output_path.with_suffix(".palette.png") subprocess.run(
try:
result = subprocess.run(
[ [
"ffmpeg", "ffmpeg",
"-y", "-y",
@@ -119,18 +116,16 @@ def _save_gif_ffmpeg(
"-safe", "-safe",
"0", "0",
"-i", "-i",
str(concat_file), str(list_file),
"-vf", "-vf",
f"palettegen=max_colors={min(colors, 256)}:stats_mode=diff", f"palettegen=max_colors={min(colors, 256)}",
str(palette_path), str(palette),
], ],
capture_output=True, capture_output=True,
text=True, check=True,
) )
if result.returncode != 0:
raise RuntimeError(f"ffmpeg palette generation failed: {result.stderr}")
result = subprocess.run( subprocess.run(
[ [
"ffmpeg", "ffmpeg",
"-y", "-y",
@@ -139,71 +134,78 @@ def _save_gif_ffmpeg(
"-safe", "-safe",
"0", "0",
"-i", "-i",
str(concat_file), str(list_file),
"-i", "-i",
str(palette_path), str(palette),
"-lavfi", "-lavfi",
f"fps={fps},paletteuse=dither=bayer:bayer_scale=5", f"fps={fps},paletteuse",
"-loop",
"0",
str(output_path), str(output_path),
], ],
capture_output=True, capture_output=True,
text=True, check=True,
) )
if result.returncode != 0:
raise RuntimeError(f"ffmpeg GIF creation failed: {result.stderr}")
finally:
concat_file.unlink(missing_ok=True)
if palette_path.exists():
palette_path.unlink()
def create_gif_from_trace_folder(trace_folder_path: Path): def create_gif_from_trace_folder(trace_folder_path: Path) -> None:
image_files: list[Path] = [] """Scan and synthesize sequential JPEG frames in a trace directory to trace.gif."""
if not trace_folder_path.is_dir():
for file in trace_folder_path.iterdir():
if file.suffix == ".jpeg":
image_files.append(file)
image_files.sort(key=lambda f: int(f.stem))
logger.info(f"Found {len(image_files)} images to compile")
if not image_files:
return return
gif_path = trace_folder_path / "trace.gif" frames = sorted(
quantize_and_save_gif_from_paths(image_files, gif_path) [f for f in trace_folder_path.iterdir() if f.suffix.lower() in (".jpeg", ".jpg") and f.stem.isdigit()],
logger.info(f"GIF created at {gif_path}") key=lambda x: int(x.stem),
)
if not frames:
return
target_gif = trace_folder_path / "trace.gif"
quantize_and_save_gif_from_paths(frames, target_gif)
logger.info(f"Assembled trace animation: {target_gif}")
def remove_images_from_trace_folder(trace_folder_path: Path): def remove_images_from_trace_folder(trace_folder_path: Path) -> None:
for file in trace_folder_path.iterdir(): """Purge temporary individual JPEG frame captures once compiled."""
if file.suffix == ".jpeg": if not trace_folder_path.is_dir():
file.unlink() return
for item in trace_folder_path.iterdir():
if item.suffix.lower() in (".jpeg", ".jpg") and item.stem.isdigit():
try:
item.unlink(missing_ok=True)
except OSError:
pass
def create_steps_json_from_trace_folder(trace_folder_path: Path): def create_steps_json_from_trace_folder(trace_folder_path: Path) -> None:
"""Compile legacy timestamp-named step files without consuming manifests.""" """Collate discrete numeric step payload dumps into steps.json preserving metadata manifests."""
steps = [] if not trace_folder_path.is_dir():
for file in trace_folder_path.iterdir(): return
if file.suffix == ".json" and file.stem.isdigit():
with open(file, encoding="utf-8", errors="ignore") as f:
json_content = f.read()
steps.append({"timestamp": int(file.stem), "data": json_content})
steps.sort(key=lambda f: f["timestamp"]) numeric_step_files = sorted(
[f for f in trace_folder_path.iterdir() if f.suffix == ".json" and f.stem.isdigit()],
key=lambda x: int(x.stem),
)
logger.info(f"Found {len(steps)} steps to compile") records: list[dict[str, Any]] = []
for step_file in numeric_step_files:
try:
records.append({
"timestamp": int(step_file.stem),
"data": step_file.read_text(encoding="utf-8", errors="ignore"),
})
except Exception as exc:
logger.debug(f"Unable to read step file {step_file}: {exc}")
with open(trace_folder_path / "steps.json", "w", encoding="utf-8", errors="ignore") as f: output_file = trace_folder_path / "steps.json"
f.write(json.dumps(steps)) output_file.write_text(json.dumps(records, ensure_ascii=False), encoding="utf-8")
def remove_steps_json_from_trace_folder(trace_folder_path: Path): def remove_steps_json_from_trace_folder(trace_folder_path: Path) -> None:
"""Remove only legacy timestamp-named step files, preserving manifests.""" """Clean up standalone numeric step files, leaving steps.json and recording manifests untouched."""
for file in trace_folder_path.iterdir(): if not trace_folder_path.is_dir():
if file.suffix == ".json" and file.stem.isdigit(): return
file.unlink() for item in trace_folder_path.iterdir():
if item.suffix == ".json" and item.stem.isdigit():
try:
item.unlink(missing_ok=True)
except OSError:
pass
-60
View File
@@ -1,60 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import time
from langchain_core.messages import BaseMessage
from artemis.context import ArtemisContext
from artemis.controllers.controller_factory import create_device_controller
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
async def record_interaction(ctx: ArtemisContext, response: BaseMessage):
if not ctx.execution_setup:
raise ValueError("No execution setup found")
if not ctx.execution_setup.traces_path or not ctx.execution_setup.trace_name:
raise ValueError("No traces path or trace name found")
logger.info("Recording interaction")
controller = create_device_controller(ctx)
screenshot_base64 = await controller.screenshot()
logger.info("Screenshot taken")
try:
controller = create_device_controller(ctx)
compressed_screenshot_base64 = controller.get_compressed_b64_screenshot(screenshot_base64)
except Exception as e:
logger.error(f"Error compressing screenshot: {e}")
return "Could not record this interaction"
timestamp = time.time()
folder = ctx.execution_setup.traces_path.joinpath(ctx.execution_setup.trace_name).resolve()
folder.mkdir(parents=True, exist_ok=True)
try:
with open(
folder.joinpath(f"{int(timestamp)}.jpeg").resolve(),
"wb",
) as f:
f.write(base64.b64decode(compressed_screenshot_base64))
with open(
folder.joinpath(f"{int(timestamp)}.json").resolve(),
"w",
encoding="utf-8",
) as f:
f.write(response.model_dump_json())
except Exception as e:
logger.error(f"Error recording interaction: {e}")
return "Screenshot recorded successfully"
-51
View File
@@ -1,51 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from artemis.utils.logger import get_logger
import requests
logger = get_logger(__name__)
def curl_from_request(req: requests.PreparedRequest) -> str:
"""Converts a requests.PreparedRequest object to a valid cURL command string."""
command = ["curl", f"-X {req.method}"]
for key, value in req.headers.items():
command.append(f'-H "{key}: {value}"')
if req.body:
body = req.body
if isinstance(body, bytes):
body = body.decode("utf-8")
# Escape single quotes in the body for shell safety
body = body.replace("'", "'\\''")
command.append(f"-d '{body}'")
command.append(f"'{req.url}'")
return " ".join(command)
def logging_hook(response, *args, **kwargs):
"""Hook to log the request as a cURL command."""
curl_command = curl_from_request(response.request)
logger.debug(f"\n--- cURL Command ---\n{curl_command}\n--------------------")
def get_session_with_curl_logging() -> requests.Session:
"""Returns a requests.Session with cURL logging enabled."""
session = requests.Session()
session.hooks["response"] = [logging_hook]
return session
-35
View File
@@ -1,35 +0,0 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import subprocess
def run_shell_command_on_host(command: str) -> str:
"""Helper to run a shell command on the host and return the output."""
try:
result = subprocess.run(
command,
stdin=subprocess.DEVNULL,
shell=True,
check=True,
capture_output=True,
text=True,
)
return result.stdout.strip()
except subprocess.CalledProcessError as e:
# Log the error and stderr for better debugging
error_message = (
f"Command '{command}' failed with exit code {e.returncode}.\nStderr: {e.stderr.strip()}"
)
raise RuntimeError(error_message) from e
+1 -1
View File
@@ -739,7 +739,7 @@ def _render_step_detailed(
INTERNAL_SYSTEM_TOOLS = { INTERNAL_SYSTEM_TOOLS = {
"safety_net_validation", "safety_net_validation",
"safety_net_pixel_validation", "safety_net_pixel_validation",
"hopper", "entity_extractor",
} }
if interleaved: if interleaved:
+128 -132
View File
@@ -12,155 +12,151 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
"""UI hierarchy traversal and element geometry primitives for ARTEMIS.
Provides structured spatial coordinates, bounding box metrics, and recursive
node querying across Android accessibility hierarchies and UIAutomator tree dumps.
"""
from __future__ import annotations
from typing import Any
from pydantic import BaseModel, ConfigDict, Field
from artemis.utils.logger import get_logger from artemis.utils.logger import get_logger
from pydantic import BaseModel, Field
logger = get_logger(__name__) logger = get_logger(__name__)
def __find_element_by_resource_id_in_rich_hierarchy(
hierarchy: list[dict], resource_id: str
) -> dict | None:
"""Retrieves all the sibling elements for a given resource ID from a nested dictionary.
Args:
hierarchy (dict): The nested dictionary representing the UI hierarchy.
resource_id (str): The resource-id to find.
Returns:
list: A list of the sibling elements, or None if the resource_id is not
found.
"""
if not hierarchy:
return None
for child in hierarchy:
if child.get("attributes", {}).get("resource-id") == resource_id:
return child.get("attributes", {})
for child in hierarchy:
result = __find_element_by_resource_id_in_rich_hierarchy(
child.get("children", []), resource_id
)
if result is not None:
return result
return None
def text_input_is_empty(text: str | None, hint_text: str | None) -> bool:
return not text or text == hint_text
def find_element_by_resource_id(
ui_hierarchy: list[dict],
resource_id: str,
index: int | None = None,
is_rich_hierarchy: bool = False,
) -> dict | None:
"""Find a UI element by its resource-id in the UI hierarchy.
Args:
ui_hierarchy: List of UI element dictionaries
resource_id: The resource-id to search for (e.g.,
"com.google.android.settings.intelligence:id/open_search_view_edit_text")
Returns:
The complete UI element dictionary if found, None otherwise
"""
if is_rich_hierarchy:
return __find_element_by_resource_id_in_rich_hierarchy(ui_hierarchy, resource_id)
target_idx = index or 0
def search_recursive(elements: list[dict]) -> dict | None:
nonlocal target_idx
for element in elements:
if isinstance(element, dict):
if element.get("resourceId") == resource_id:
if target_idx == 0:
return element
target_idx -= 1
continue
children = element.get("children", [])
if children:
result = search_recursive(children)
if result:
return result
return None
return search_recursive(ui_hierarchy)
def is_element_focused(element: dict) -> bool:
return element.get("focused", None) == "true"
def get_element_text(element: dict, hint_text: bool = False) -> str | None:
if hint_text:
return element.get("hintText", None)
return element.get("text", None)
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class Point(BaseModel): class Point(BaseModel):
model_config = {"ignored_types": (CyFunctionDetector,)} """Discrete pixel coordinate on screen."""
model_config = ConfigDict(frozen=True)
x: int x: int
y: int y: int
class ElementBounds(BaseModel): class ElementBounds(BaseModel):
model_config = {"ignored_types": (CyFunctionDetector,)} """Axis-aligned rectangular boundary defining an on-screen element."""
x: int = Field(description="The x coordinate of the top-left corner of the element.")
y: int = Field(description="The y coordinate of the top-left corner of the element.") model_config = ConfigDict(extra="ignore")
width: int = Field(description="The width of the element.")
height: int = Field(description="The height of the element.") x: int = Field(description="Horizontal offset of the top-left corner in pixels.")
y: int = Field(description="Vertical offset of the top-left corner in pixels.")
width: int = Field(description="Horizontal extent of the bounding box.")
height: int = Field(description="Vertical extent of the bounding box.")
def get_center(self) -> Point: def get_center(self) -> Point:
return Point(x=self.x + self.width // 2, y=self.y + self.height // 2) """Compute the centroid of the element boundary."""
return Point(x=self.x + (self.width // 2), y=self.y + (self.height // 2))
def get_relative_point(self, x_percent: float, y_percent: float) -> Point: def get_relative_point(self, x_percent: float, y_percent: float) -> Point:
"""Returns the coordinates of the point at x_percent of the width and y_percent """Compute an internal pixel coordinate by fractional offsets [0.0, 1.0]."""
clamped_x = max(0.0, min(1.0, float(x_percent)))
of the height of the element. clamped_y = max(0.0, min(1.0, float(y_percent)))
Ex if x_percent = 0.95 and y_percent = 0.95,
the point is at the bottom right of the element:
<------>
| |
| x|
<------>
"""
return Point( return Point(
x=int(self.x + self.width * x_percent), x=int(self.x + (self.width * clamped_x)),
y=int(self.y + self.height * y_percent), y=int(self.y + (self.height * clamped_y)),
) )
def get_bounds_for_element(element: dict) -> ElementBounds | None: def get_bounds_for_element(element: dict[str, Any]) -> ElementBounds | None:
bounds = element.get("bounds", None) """Extract and validate the bounding box from a node descriptor."""
if bounds: raw_bounds = element.get("bounds")
try: if not isinstance(raw_bounds, dict):
return ElementBounds(**bounds) return None
except Exception as e:
logger.error(f"Failed to validate bounds: {e}") try:
return None return ElementBounds(
x=int(raw_bounds["x"]),
y=int(raw_bounds["y"]),
width=int(raw_bounds["width"]),
height=int(raw_bounds["height"]),
)
except (KeyError, TypeError, ValueError) as exc:
logger.error(f"Failed to validate element bounds ({raw_bounds}): {exc}")
return None
def text_input_is_empty(text: str | None, hint_text: str | None) -> bool:
"""Determine whether an editable field contains user-entered text."""
if not text:
return True
return text == hint_text
def is_element_focused(element: dict[str, Any]) -> bool:
"""Check if the given node currently holds system focus."""
focused_val = element.get("focused")
if isinstance(focused_val, bool):
return focused_val
return str(focused_val).strip().lower() == "true"
def get_element_text(element: dict[str, Any], hint_text: bool = False) -> str | None:
"""Retrieve textual label or placeholder from an element node."""
if hint_text:
return element.get("hintText")
return element.get("text")
def find_element_by_resource_id(
ui_hierarchy: list[dict[str, Any]],
resource_id: str,
index: int | None = None,
is_rich_hierarchy: bool = False,
) -> dict[str, Any] | None:
"""Locate an element in the hierarchy tree matching the specified resource ID.
Supports both flat attribute mappings and rich accessibility tree hierarchies.
"""
if not ui_hierarchy or not resource_id:
return None
if is_rich_hierarchy:
return _search_rich_tree(ui_hierarchy, resource_id)
target_index = max(0, index or 0)
current_match = 0
def _walk(nodes: list[dict[str, Any]]) -> dict[str, Any] | None:
nonlocal current_match
for node in nodes:
if not isinstance(node, dict):
continue
if node.get("resourceId") == resource_id:
if current_match == target_index:
return node
current_match += 1
children = node.get("children")
if isinstance(children, list) and children:
found = _walk(children)
if found is not None:
return found
return None
return _walk(ui_hierarchy)
def _search_rich_tree(
nodes: list[dict[str, Any]], resource_id: str
) -> dict[str, Any] | None:
"""Recursively traverse a rich XML attribute tree for resource ID match."""
for node in nodes:
if not isinstance(node, dict):
continue
attributes = node.get("attributes")
if isinstance(attributes, dict) and attributes.get("resource-id") == resource_id:
return attributes
children = node.get("children")
if isinstance(children, list) and children:
sub = _search_rich_tree(children, resource_id)
if sub is not None:
return sub
return None return None
+3 -18
View File
@@ -11,6 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
"""Video recording utilities for mobile devices. """Video recording utilities for mobile devices.
@@ -123,24 +126,6 @@ async def await_scrcpy_first_frame(
return max(spawned_at, time.time()) return max(spawned_at, time.time())
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class RecordingSession(BaseModel): class RecordingSession(BaseModel):
"""Tracks an active video recording session.""" """Tracks an active video recording session."""
+2 -2
View File
@@ -93,8 +93,8 @@
// 📋 Summarizer: Aggregates step history and generates structured outcome // 📋 Summarizer: Aggregates step history and generates structured outcome
"summarizer": {}, "summarizer": {},
// 🔽 Hopper: Fast lightweight package name locator // 🔍 Entity Extractor: Fast lightweight package name locator and entity extractor
"hopper": { "entity_extractor": {
"model": "gemini-3.5-flash-lite", "model": "gemini-3.5-flash-lite",
"fallback": { "fallback": {
"provider": "google", "provider": "google",
@@ -14,6 +14,6 @@
"""This file is intentionally empty. """This file is intentionally empty.
The hopper agent does not expose any tools directly. It is invoked The entity_extractor agent does not expose any tools directly. It is invoked
programmatically by other tools (e.g., launch_app_tool) as an LLM utility. programmatically by other tools (e.g., launch_app_tool) as an LLM utility.
""" """
@@ -0,0 +1,76 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Artemis entity extraction and package resolver."""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from artemis.agents.entity_extractor.entity_extractor import (
ExtractionResult,
extract_entity,
)
def test_extraction_result_model():
"""Verify ExtractionResult attributes and validation."""
res = ExtractionResult(found=True, output="com.google.android.apps.nexuslauncher", reason="Exact match")
assert res.found is True
assert res.output == "com.google.android.apps.nexuslauncher"
assert res.reason == "Exact match"
@pytest.mark.asyncio
async def test_extract_entity_success():
"""Verify successful LLM entity extraction invocation."""
mock_ctx = MagicMock()
mock_llm = MagicMock()
expected_result = ExtractionResult(
found=True,
output="com.google.android.calculator",
reason="Found calculator package in list",
)
mock_llm.ainvoke = AsyncMock(return_value=expected_result)
with patch("artemis.agents.entity_extractor.entity_extractor.get_llm") as mock_get_llm:
mock_get_llm.return_value.with_structured_output.return_value = mock_llm
result = await extract_entity(
request="Calculator",
data="com.google.android.calculator\ncom.google.android.calendar",
ctx=mock_ctx,
)
assert result.found is True
assert result.output == "com.google.android.calculator"
assert result.reason == "Found calculator package in list"
@pytest.mark.asyncio
async def test_extract_entity_handles_exception():
"""Verify extraction returns a graceful failure result when LLM throws an exception."""
mock_ctx = MagicMock()
mock_llm = MagicMock()
mock_llm.ainvoke = AsyncMock(side_effect=RuntimeError("LLM connection timed out"))
with patch("artemis.agents.entity_extractor.entity_extractor.get_llm") as mock_get_llm:
mock_get_llm.return_value.with_structured_output.return_value = mock_llm
result = await extract_entity(
request="Settings",
data="com.android.settings",
ctx=mock_ctx,
)
assert result.found is False
assert result.output is None
assert "LLM extraction error" in result.reason
-19
View File
@@ -26,26 +26,7 @@ from artemis.utils.logger import get_logger # noqa: E402
logger = get_logger(__name__) logger = get_logger(__name__)
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class MockPydanticSchema(BaseModel): class MockPydanticSchema(BaseModel):
model_config = {"ignored_types": (CyFunctionDetector,)}
color: str color: str
price: float price: float
currency_symbol: str currency_symbol: str
+4 -2
View File
@@ -9,6 +9,8 @@ from mcp_server.utils import device_utils
def test_ensure_emulator_uses_windows_creation_flags(monkeypatch) -> None: def test_ensure_emulator_uses_windows_creation_flags(monkeypatch) -> None:
popen = MagicMock() popen = MagicMock()
monkeypatch.setattr(device_utils.sys, "platform", "win32") monkeypatch.setattr(device_utils.sys, "platform", "win32")
monkeypatch.setattr(device_utils.subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200, raising=False)
monkeypatch.setattr(device_utils.subprocess, "DETACHED_PROCESS", 0x00000008, raising=False)
monkeypatch.setattr(device_utils, "is_emulator_running", lambda _adb: False) monkeypatch.setattr(device_utils, "is_emulator_running", lambda _adb: False)
monkeypatch.setattr(device_utils.os.path, "exists", lambda _path: True) monkeypatch.setattr(device_utils.os.path, "exists", lambda _path: True)
monkeypatch.setattr(device_utils.subprocess, "Popen", popen) monkeypatch.setattr(device_utils.subprocess, "Popen", popen)
@@ -20,7 +22,7 @@ def test_ensure_emulator_uses_windows_creation_flags(monkeypatch) -> None:
monkeypatch.setattr(device_utils.time, "sleep", lambda _seconds: None) monkeypatch.setattr(device_utils.time, "sleep", lambda _seconds: None)
assert device_utils.ensure_emulator( assert device_utils.ensure_emulator(
adb_path="adb.exe", emulator_path="emulator.exe", timeout_seconds=1 adb_path="adb.exe", emulator_path="emulator.exe", timeout_seconds=10
) )
kwargs = popen.call_args.kwargs kwargs = popen.call_args.kwargs
@@ -43,7 +45,7 @@ def test_ensure_emulator_starts_new_session_on_posix(monkeypatch) -> None:
) )
monkeypatch.setattr(device_utils.time, "sleep", lambda _seconds: None) monkeypatch.setattr(device_utils.time, "sleep", lambda _seconds: None)
assert device_utils.ensure_emulator(adb_path="adb", emulator_path="emulator", timeout_seconds=1) assert device_utils.ensure_emulator(adb_path="adb", emulator_path="emulator", timeout_seconds=10)
kwargs = popen.call_args.kwargs kwargs = popen.call_args.kwargs
assert kwargs["start_new_session"] is True assert kwargs["start_new_session"] is True
+5 -12
View File
@@ -24,11 +24,8 @@ import queue
import subprocess import subprocess
import sys import sys
import threading import threading
import time
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
import pytest
from artemis.clients import ui_automator_client from artemis.clients import ui_automator_client
from artemis.runtime.awake_lease import ScreenAwakeLease from artemis.runtime.awake_lease import ScreenAwakeLease
from artemis.runtime.awake_service import _run_awake_adb_command from artemis.runtime.awake_service import _run_awake_adb_command
@@ -204,16 +201,12 @@ def test_device_utils_isolates_stdin():
def test_ui_automator_client_isolates_stdin(): def test_ui_automator_client_isolates_stdin():
"""Verify ui_automator_client helper commands isolate stdin.""" """Verify ui_automator_client screencap commands isolate stdin."""
client = ui_automator_client.UIAutomatorClient("dev-1")
with patch("artemis.clients.ui_automator_client.subprocess.run") as mock_run: with patch("artemis.clients.ui_automator_client.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(returncode=0, stdout="package:com.test\n", stderr="") mock_run.return_value = MagicMock(returncode=0, stdout=b"", stderr="")
ui_automator_client._is_package_installed("dev-1", "com.test") with patch("artemis.clients.ui_automator_client.Image.open"):
assert mock_run.called client.get_screenshot()
assert mock_run.call_args.kwargs.get("stdin") == subprocess.DEVNULL
with patch("artemis.clients.ui_automator_client.subprocess.run") as mock_run:
mock_run.return_value = MagicMock(returncode=0, stdout="", stderr="")
ui_automator_client._uninstall_package("dev-1", "com.test")
assert mock_run.called assert mock_run.called
assert mock_run.call_args.kwargs.get("stdin") == subprocess.DEVNULL assert mock_run.call_args.kwargs.get("stdin") == subprocess.DEVNULL
+63
View File
@@ -0,0 +1,63 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests validating Artemis SDK exception hierarchy and diagnostics."""
from artemis.sdk.types.exceptions import (
AgentError,
AgentNotInitializedError,
AgentProfileNotFoundError,
AgentTaskRequestError,
ArtemisError,
DeviceError,
DeviceNotFoundError,
ExecutableNotFoundError,
ServerError,
ServerStartupError,
)
def test_exception_hierarchy():
"""Verify all custom exceptions inherit properly from ArtemisError."""
assert issubclass(DeviceError, ArtemisError)
assert issubclass(DeviceNotFoundError, DeviceError)
assert issubclass(ServerError, ArtemisError)
assert issubclass(ServerStartupError, ServerError)
assert issubclass(AgentError, ArtemisError)
assert issubclass(AgentNotInitializedError, AgentError)
assert issubclass(AgentTaskRequestError, AgentError)
assert issubclass(AgentProfileNotFoundError, AgentTaskRequestError)
assert issubclass(ExecutableNotFoundError, ArtemisError)
def test_executable_not_found_error_adb_guidance():
"""Verify executable error formats download instructions for known tools."""
err = ExecutableNotFoundError("adb")
assert "adb" in str(err)
assert "developer.android.com" in str(err)
assert err.executable_name == "adb"
def test_server_startup_error_formatting():
"""Verify server startup error encapsulates service identifier."""
err = ServerStartupError(server_name="mcp_daemon")
assert "mcp_daemon" in str(err)
assert err.server_name == "mcp_daemon"
def test_agent_profile_not_found_error():
"""Verify profile error retains profile name."""
err = AgentProfileNotFoundError("nonexistent_profile")
assert "nonexistent_profile" in str(err)
assert err.profile_name == "nonexistent_profile"
+111
View File
@@ -0,0 +1,111 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Artemis SDK TaskRequestBuilder."""
from pathlib import Path
from pydantic import BaseModel
import pytest
from artemis.sdk.builders.task_request_builder import TaskRequestBuilder, TaskRequestCommonBuilder
from artemis.sdk.types.task import TaskRequest
class DummyOutputSchema(BaseModel):
summary: str
item_count: int
def test_task_request_builder_fluent_chaining():
"""Verify basic fluent configuration and build."""
req = (
TaskRequestBuilder(goal="Open Settings and verify battery level")
.with_max_steps(25)
.with_trace_recording(True, path="custom_traces")
.with_locked_app_package("com.android.settings")
.with_app_path("build/app.apk")
.using_profile("flash")
.with_name("Battery Check")
.build()
)
assert isinstance(req, TaskRequest)
assert req.goal == "Open Settings and verify battery level"
assert req.max_steps == 25
assert req.record_trace is True
assert req.trace_path == Path("custom_traces")
assert req.locked_app_package == "com.android.settings"
assert req.app_path == Path("build/app.apk")
assert req.profile == "flash"
assert req.task_name == "Battery Check"
def test_task_request_builder_with_output_format():
"""Verify builder binds structured schema properly."""
req = (
TaskRequestBuilder(goal="Extract items")
.with_output_format(DummyOutputSchema)
.build()
)
assert req.output_format is DummyOutputSchema
assert req.output_description is None
def test_task_request_builder_with_output_description():
"""Verify builder accepts unformatted string deliverables."""
req = (
TaskRequestBuilder(goal="Extract items")
.with_output_description("Return a numbered list")
.build()
)
assert req.output_description == "Return a numbered list"
assert req.output_format is None
def test_task_request_builder_conflicting_outputs_raises():
"""Verify that specifying both output_format and output_description raises ValueError."""
builder = (
TaskRequestBuilder(goal="Extract items")
.with_output_format(DummyOutputSchema)
.with_output_description("Description text")
)
with pytest.raises(ValueError, match="Cannot specify both output_format and output_description"):
builder.build()
def test_task_request_builder_empty_goal_raises():
"""Verify empty or whitespace-only goal raises ValueError."""
with pytest.raises(ValueError, match="Task goal must be a non-empty"):
TaskRequestBuilder(goal=" ").build()
def test_task_request_builder_from_common():
"""Verify task request initialization derived from common base settings."""
common_settings = (
TaskRequestCommonBuilder()
.with_max_steps(42)
.with_trace_recording(False)
.with_locked_app_package("org.artemis.target")
.with_app_path(Path("/tmp/target.apk"))
.build()
)
derived = TaskRequestBuilder.from_common("Derived goal", common_settings).build()
assert derived.goal == "Derived goal"
assert derived.max_steps == 42
assert derived.record_trace is False
assert derived.locked_app_package == "org.artemis.target"
assert derived.app_path == Path("/tmp/target.apk")
+1 -1
View File
@@ -154,7 +154,7 @@ def test_llm_config_parsing_and_merging():
assert isinstance(llm_cfg, LLMConfig) assert isinstance(llm_cfg, LLMConfig)
assert llm_cfg.planner.provider in ("google", "openai", "openrouter", "xai", "vertexai") assert llm_cfg.planner.provider in ("google", "openai", "openrouter", "xai", "vertexai")
assert llm_cfg.get_agent("planner") is not None assert llm_cfg.get_agent("planner") is not None
assert llm_cfg.get_utils("hopper") is not None assert llm_cfg.get_utils("entity_extractor") is not None
# Deep merge overrides # Deep merge overrides
overrides = { overrides = {
+3 -7
View File
@@ -11,16 +11,14 @@ from artemis.drivers.android.adb_driver import AndroidAdbDriver
"artemis.clients.ui_automator_client.ensure_device_awake", "artemis.clients.ui_automator_client.ensure_device_awake",
return_value="host_heartbeat", return_value="host_heartbeat",
) )
@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed")
def test_new_ui_connection_enrolls_device_in_shared_awake_strategy( def test_new_ui_connection_enrolls_device_in_shared_awake_strategy(
mock_remove_maestro, mock_ensure_awake, mock_connect mock_ensure_awake, mock_connect
): ):
client = UIAutomatorClient("device-123") client = UIAutomatorClient("device-123")
client.connect() client.connect()
client.connect() client.connect()
mock_remove_maestro.assert_called_once_with("device-123")
mock_ensure_awake.assert_called_once_with("device-123") mock_ensure_awake.assert_called_once_with("device-123")
mock_connect.assert_called_once_with("device-123") mock_connect.assert_called_once_with("device-123")
@@ -30,9 +28,8 @@ def test_new_ui_connection_enrolls_device_in_shared_awake_strategy(
"artemis.clients.ui_automator_client.ensure_device_awake", "artemis.clients.ui_automator_client.ensure_device_awake",
return_value="host_heartbeat", return_value="host_heartbeat",
) )
@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed")
def test_failed_ui_connection_does_not_stop_process_awake_service( def test_failed_ui_connection_does_not_stop_process_awake_service(
_mock_remove_maestro, mock_ensure_awake, _mock_connect mock_ensure_awake, _mock_connect
): ):
client = UIAutomatorClient("device-123") client = UIAutomatorClient("device-123")
@@ -48,9 +45,8 @@ def test_failed_ui_connection_does_not_stop_process_awake_service(
"artemis.clients.ui_automator_client.ensure_device_awake", "artemis.clients.ui_automator_client.ensure_device_awake",
return_value="host_heartbeat", return_value="host_heartbeat",
) )
@patch("artemis.clients.ui_automator_client._ensure_maestro_not_installed")
def test_client_disconnect_does_not_send_power_cleanup_commands( def test_client_disconnect_does_not_send_power_cleanup_commands(
_mock_remove_maestro, mock_ensure_awake, _mock_connect mock_ensure_awake, _mock_connect
): ):
client = UIAutomatorClient("device-123") client = UIAutomatorClient("device-123")
client.connect() client.connect()
+23 -22
View File
@@ -13,10 +13,11 @@
# limitations under the License. # limitations under the License.
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
from artemis.agents.hopper.hopper import HopperOutput import pytest
from artemis.agents.entity_extractor.entity_extractor import ExtractionResult
from artemis.context import ArtemisContext from artemis.context import ArtemisContext
from artemis.tools.mobile.launch_app import find_package from artemis.tools.mobile.launch_app import find_package
import pytest
@pytest.fixture @pytest.fixture
@@ -28,63 +29,63 @@ def mock_context():
@pytest.mark.asyncio @pytest.mark.asyncio
@patch("artemis.tools.mobile.launch_app.list_packages_async") @patch("artemis.tools.mobile.launch_app.list_packages_async")
@patch("artemis.tools.mobile.launch_app.hopper") @patch("artemis.tools.mobile.launch_app.extract_entity")
async def test_find_package_cache_hit(mock_hopper, mock_list_packages, mock_context): async def test_find_package_cache_hit(mock_extract_entity, mock_list_packages, mock_context):
mock_list_packages.return_value = "com.example.app1\ncom.example.app2" mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
# First call: not in cache, calls hopper # First call: not in cache, calls extract_entity
mock_hopper.return_value = HopperOutput( mock_extract_entity.return_value = ExtractionResult(
found=True, output="com.example.app1", reason="Found it" found=True, output="com.example.app1", reason="Found it"
) )
pkg1 = await find_package(mock_context, "App 1") pkg1 = await find_package(mock_context, "App 1")
assert pkg1 == "com.example.app1" assert pkg1 == "com.example.app1"
assert mock_hopper.call_count == 1 assert mock_extract_entity.call_count == 1
assert mock_context.package_cache["App 1"] == "com.example.app1" assert mock_context.package_cache["App 1"] == "com.example.app1"
# Second call: cache hit, does not call hopper or list_packages again # Second call: cache hit, does not call extract_entity or list_packages again
mock_hopper.reset_mock() mock_extract_entity.reset_mock()
mock_list_packages.reset_mock() mock_list_packages.reset_mock()
pkg2 = await find_package(mock_context, "App 1") pkg2 = await find_package(mock_context, "App 1")
assert pkg2 == "com.example.app1" assert pkg2 == "com.example.app1"
mock_hopper.assert_not_called() mock_extract_entity.assert_not_called()
mock_list_packages.assert_not_called() mock_list_packages.assert_not_called()
@pytest.mark.asyncio @pytest.mark.asyncio
@patch("artemis.tools.mobile.launch_app.list_packages_async") @patch("artemis.tools.mobile.launch_app.list_packages_async")
@patch("artemis.tools.mobile.launch_app.hopper") @patch("artemis.tools.mobile.launch_app.extract_entity")
async def test_find_package_not_found_caches_none(mock_hopper, mock_list_packages, mock_context): async def test_find_package_not_found_caches_none(mock_extract_entity, mock_list_packages, mock_context):
mock_list_packages.return_value = "com.example.app1\ncom.example.app2" mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
# Hopper doesn't find it # Extractor doesn't find it
mock_hopper.return_value = HopperOutput(found=False, output=None, reason="Not found") mock_extract_entity.return_value = ExtractionResult(found=False, output=None, reason="Not found")
pkg1 = await find_package(mock_context, "App 3") pkg1 = await find_package(mock_context, "App 3")
assert pkg1 is None assert pkg1 is None
assert mock_hopper.call_count == 1 assert mock_extract_entity.call_count == 1
assert mock_context.package_cache["App 3"] is None assert mock_context.package_cache["App 3"] is None
# Second call: cache hit (None), does not call hopper again # Second call: cache hit (None), does not call extract_entity again
mock_hopper.reset_mock() mock_extract_entity.reset_mock()
pkg2 = await find_package(mock_context, "App 3") pkg2 = await find_package(mock_context, "App 3")
assert pkg2 is None assert pkg2 is None
mock_hopper.assert_not_called() mock_extract_entity.assert_not_called()
@pytest.mark.asyncio @pytest.mark.asyncio
@patch("artemis.tools.mobile.launch_app.list_packages_async") @patch("artemis.tools.mobile.launch_app.list_packages_async")
@patch("artemis.tools.mobile.launch_app.hopper") @patch("artemis.tools.mobile.launch_app.extract_entity")
async def test_find_package_passes_use_fallback(mock_hopper, mock_list_packages, mock_context): async def test_find_package_passes_use_fallback(mock_extract_entity, mock_list_packages, mock_context):
mock_list_packages.return_value = "com.example.app1\ncom.example.app2" mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
mock_hopper.return_value = HopperOutput( mock_extract_entity.return_value = ExtractionResult(
found=True, output="com.example.app1", reason="Found it" found=True, output="com.example.app1", reason="Found it"
) )
# Pass use_fallback=False # Pass use_fallback=False
await find_package(mock_context, "App 1", use_fallback=False) await find_package(mock_context, "App 1", use_fallback=False)
mock_hopper.assert_called_once_with( mock_extract_entity.assert_called_once_with(
ctx=mock_context, ctx=mock_context,
request=("I'm looking for the package name of the following app: 'App 1'"), request=("I'm looking for the package name of the following app: 'App 1'"),
data="com.example.app1\ncom.example.app2", data="com.example.app1\ncom.example.app2",
+248 -354
View File
@@ -12,446 +12,340 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import asyncio """Unit tests for ARTEMIS tool utility functions and element interaction helpers."""
import sys
from unittest.mock import Mock, patch from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock, patch
import pytest import pytest
# Mock the problematic langgraph import at module level from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
_orig_chat_agent_executor = sys.modules.get("langgraph.prebuilt.chat_agent_executor") from artemis.controllers.types import CoordinatesSelectorRequest
_orig_state = sys.modules.get("artemis.graph.state") from artemis.tools.types import Target
from artemis.tools.utils import (
sys.modules["langgraph.prebuilt.chat_agent_executor"] = Mock()
sys.modules["artemis.graph.state"] = Mock()
from artemis.context import DeviceContext, DevicePlatform, ArtemisContext # noqa: E402
from artemis.tools.types import Target # noqa: E402
from artemis.tools.utils import ( # noqa: E402
IdSelectorRequest, IdSelectorRequest,
IdWithTextSelectorRequest,
SelectorRequestWithCoordinates, SelectorRequestWithCoordinates,
SelectorRequestWithPercentages,
TextSelectorRequest,
_extract_resource_id_and_text_from_selector,
find_element_by_text,
focus_element_if_needed, focus_element_if_needed,
has_valid_selectors,
move_cursor_to_end_if_bounds, move_cursor_to_end_if_bounds,
tap_bottom_right_of_element,
validate_coordinates_bounds,
) )
from artemis.utils.ui_hierarchy import ElementBounds # noqa: E402 from artemis.utils.ui_hierarchy import ElementBounds
# Restore original modules to prevent contaminating other tests
if _orig_chat_agent_executor is not None:
sys.modules["langgraph.prebuilt.chat_agent_executor"] = _orig_chat_agent_executor
else:
del sys.modules["langgraph.prebuilt.chat_agent_executor"]
if _orig_state is not None:
sys.modules["artemis.graph.state"] = _orig_state
else:
del sys.modules["artemis.graph.state"]
@pytest.fixture @pytest.fixture
def mock_context(): def dummy_context() -> MagicMock:
"""Create a mock ArtemisContext for testing.""" """Fixture providing an ARTEMIS runtime context configured for Android."""
ctx = Mock(spec=ArtemisContext) ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock(spec=DeviceContext)
# Create device context with necessary attributes
ctx.device = Mock(spec=DeviceContext)
ctx.device.mobile_platform = DevicePlatform.ANDROID ctx.device.mobile_platform = DevicePlatform.ANDROID
ctx.device.device_id = "test_device_123" ctx.device.device_id = "test-device-pixel8"
ctx.device.device_width = 1080 ctx.device.device_width = 1080
ctx.device.device_height = 2340 ctx.device.device_height = 2400
ctx.device.host_platform = "LINUX"
ctx.ui_adb_client = Mock()
# Mock the ADB client for Android
ctx.adb_client = Mock()
mock_device = Mock()
mock_device.shell = Mock(return_value="")
ctx.adb_client.device = Mock(return_value=mock_device)
# Mock the ADB client for Android
mock_response = Mock()
mock_response.json.return_value = {"elements": []}
ctx.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
ctx.ui_adb_client.get_hierarchy = Mock(
return_value=(
b'<hierarchy><node bounds="[0,0][1000,1000]"><node'
b' bounds="[0,0][100,100]"/></node></hierarchy>'
)
)
return ctx return ctx
@pytest.fixture @pytest.fixture
def mock_state(): def mock_graph_state() -> MagicMock:
"""Create a mock State for testing.""" """Fixture providing a mock LangGraph state container."""
state = Mock() state = MagicMock()
state.latest_ui_hierarchy = [] state.latest_ui_hierarchy = []
return state return state
@pytest.fixture @pytest.fixture
def sample_element(): def sample_hierarchy_node() -> dict:
"""Create a sample UI element for testing.""" """Fixture providing a standard flat hierarchy node."""
return { return {
"resourceId": "com.example:id/text_input", "resourceId": "com.google.android.apps:id/search_query",
"text": "Sample text", "text": "Search Google",
"bounds": {"x": 100, "y": 200, "width": 300, "height": 50}, "bounds": {"x": 120, "y": 250, "width": 400, "height": 80},
"focused": "false", "focused": "false",
} }
@pytest.fixture @pytest.fixture
def sample_rich_element(): def sample_nested_node() -> dict:
"""Create a sample rich UI element for testing.""" """Fixture providing a rich nested hierarchy node."""
return { return {
"attributes": { "attributes": {
"resource-id": "com.example:id/text_input", "resource-id": "com.google.android.apps:id/search_query",
"text": "Search Google",
"bounds": {"x": 120, "y": 250, "width": 400, "height": 80},
"focused": "false", "focused": "false",
"text": "Sample text",
"bounds": {"x": 100, "y": 200, "width": 300, "height": 50},
}, },
"children": [], "children": [],
} }
class TestMoveCursorToEndIfBounds: # ==============================================================================
"""Test cases for move_cursor_to_end_if_bounds function.""" # Cursor Placement & Boundary Alignment Tests
# ==============================================================================
@patch("artemis.tools.utils.tap")
class TestCursorAlignment:
"""Validates precise cursor placement against interactive targets."""
@pytest.mark.asyncio
@patch("artemis.tools.utils.tap", new_callable=AsyncMock)
@patch("artemis.tools.utils.find_element_by_resource_id") @patch("artemis.tools.utils.find_element_by_resource_id")
def test_move_cursor_with_resource_id( async def test_cursor_routes_to_end_via_resource_id(
self, self,
mock_find_element, mock_find_elem: MagicMock,
mock_tap, mock_tap_fn: AsyncMock,
mock_context, dummy_context: MagicMock,
mock_state, mock_graph_state: MagicMock,
sample_element, sample_hierarchy_node: dict,
): ):
"""Test moving cursor using resource_id (highest priority).""" """Resource ID locator should calculate end-boundary tap (99% width/height offset)."""
mock_state.latest_ui_hierarchy = [sample_element] mock_graph_state.latest_ui_hierarchy = [sample_hierarchy_node]
mock_find_element.return_value = sample_element mock_find_elem.return_value = sample_hierarchy_node
target = Target( target = Target(resource_id="com.google.android.apps:id/search_query")
resource_id="com.example:id/text_input", elem = await move_cursor_to_end_if_bounds(
resource_id_index=None, ctx=dummy_context, state=mock_graph_state, target=target
text=None,
text_index=None,
bounds=None,
)
result = asyncio.run(
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
) )
mock_find_element.assert_called_once_with( mock_find_elem.assert_called_once_with(
ui_hierarchy=[sample_element], ui_hierarchy=[sample_hierarchy_node],
resource_id="com.example:id/text_input", resource_id="com.google.android.apps:id/search_query",
index=0, index=0,
) )
mock_tap.assert_called_once() mock_tap_fn.assert_awaited_once()
call_args = mock_tap.call_args[1] kwargs = mock_tap_fn.await_args.kwargs
selector_request = call_args["selector_request"] selector_req = kwargs["selector_request"]
assert isinstance(selector_request, SelectorRequestWithCoordinates) assert isinstance(selector_req, SelectorRequestWithCoordinates)
coords = selector_request.coordinates assert selector_req.coordinates.x == 516
assert coords.x == 397 # 100 + 300 * 0.99 assert selector_req.coordinates.y == 329
assert coords.y == 249 # 200 + 50 * 0.99 assert elem == sample_hierarchy_node
assert result == sample_element
@patch("artemis.tools.utils.tap") @pytest.mark.asyncio
@patch("artemis.tools.utils.tap", new_callable=AsyncMock)
@patch("artemis.tools.utils.find_element_by_resource_id") @patch("artemis.tools.utils.find_element_by_resource_id")
def test_move_cursor_with_coordinates_only( async def test_cursor_routes_using_direct_bounds(
self, mock_find_element, mock_tap, mock_context, mock_state self,
mock_find_elem: MagicMock,
mock_tap_fn: AsyncMock,
dummy_context: MagicMock,
mock_graph_state: MagicMock,
): ):
"""Test moving cursor when only coordinates are provided.""" """Direct bounding box coordinates should bypass hierarchy inspection."""
bounds = ElementBounds(x=50, y=150, width=200, height=40) bounds = ElementBounds(x=80, y=100, width=200, height=60)
target = Target( target = Target(bounds=bounds)
resource_id=None,
resource_id_index=None, result = await move_cursor_to_end_if_bounds(
text=None, ctx=dummy_context, state=mock_graph_state, target=target
text_index=None,
bounds=bounds,
) )
result = asyncio.run( mock_find_elem.assert_not_called()
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target) mock_tap_fn.assert_awaited_once()
) kwargs = mock_tap_fn.await_args.kwargs
selector_req = kwargs["selector_request"]
assert selector_req.coordinates.x == 278
assert selector_req.coordinates.y == 159
assert result is None
mock_find_element.assert_not_called() @pytest.mark.asyncio
mock_tap.assert_called_once() @patch("artemis.tools.utils.tap", new_callable=AsyncMock)
call_args = mock_tap.call_args[1]
selector_request = call_args["selector_request"]
coords = selector_request.coordinates
assert coords.x == 248 # 50 + 200 * 0.99
assert coords.y == 189 # 150 + 40 * 0.99
assert result is None # No element is returned when using coords directly
@patch("artemis.tools.utils.tap")
@patch("artemis.tools.utils.find_element_by_text") @patch("artemis.tools.utils.find_element_by_text")
def test_move_cursor_with_text_only_success( async def test_cursor_routes_by_matched_text(
self, mock_find_text, mock_tap, mock_context, mock_state, sample_element self,
mock_find_text: MagicMock,
mock_tap_fn: AsyncMock,
dummy_context: MagicMock,
mock_graph_state: MagicMock,
sample_hierarchy_node: dict,
): ):
"""Test moving cursor when only text is provided and succeeds.""" """Visible text locator should successfully resolve bounds and trigger cursor placement."""
mock_state.latest_ui_hierarchy = [sample_element] mock_graph_state.latest_ui_hierarchy = [sample_hierarchy_node]
mock_find_text.return_value = sample_element mock_find_text.return_value = sample_hierarchy_node
target = Target( target = Target(text="Search Google", text_index=0)
resource_id=None, result = await move_cursor_to_end_if_bounds(
resource_id_index=None, ctx=dummy_context, state=mock_graph_state, target=target
text="Sample text",
text_index=0,
bounds=None,
)
result = asyncio.run(
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
) )
mock_find_text.assert_called_once_with([sample_element], "Sample text", index=0) mock_find_text.assert_called_once_with([sample_hierarchy_node], "Search Google", index=0)
mock_tap.assert_called_once() mock_tap_fn.assert_awaited_once()
assert result == sample_element assert result == sample_hierarchy_node
@patch("artemis.tools.utils.tap") @pytest.mark.asyncio
@patch("artemis.tools.utils.tap", new_callable=AsyncMock)
@patch("artemis.tools.utils.find_element_by_text") @patch("artemis.tools.utils.find_element_by_text")
def test_move_cursor_with_text_only_element_not_found( async def test_cursor_skips_when_text_unmatched(
self, mock_find_text, mock_tap, mock_context, mock_state self,
mock_find_text: MagicMock,
mock_tap_fn: AsyncMock,
dummy_context: MagicMock,
mock_graph_state: MagicMock,
): ):
"""Test when searching by text finds no element.""" """Unresolved text target returns None without tapping."""
mock_state.latest_ui_hierarchy = []
mock_find_text.return_value = None mock_find_text.return_value = None
target = Target(text="Unmatched Query")
target = Target( result = await move_cursor_to_end_if_bounds(
resource_id=None, ctx=dummy_context, state=mock_graph_state, target=target
resource_id_index=None,
text="Nonexistent text",
text_index=None,
bounds=None,
) )
result = asyncio.run( mock_tap_fn.assert_not_awaited()
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
)
mock_tap.assert_not_called()
assert result is None
@patch("artemis.tools.utils.tap")
@patch("artemis.tools.utils.find_element_by_text")
def test_move_cursor_with_text_only_no_bounds(
self, mock_find_text, mock_tap, mock_context, mock_state
):
"""Test when element is found by text but has no bounds."""
element_no_bounds = {"text": "Text without bounds"}
mock_state.latest_ui_hierarchy = [element_no_bounds]
mock_find_text.return_value = element_no_bounds
target = Target(
resource_id=None,
resource_id_index=None,
text="Text without bounds",
text_index=None,
bounds=None,
)
result = asyncio.run(
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
)
mock_tap.assert_not_called()
assert result is None # Should return None as no action was taken
@patch("artemis.tools.utils.find_element_by_resource_id")
def test_move_cursor_element_not_found_by_id(self, mock_find_element, mock_context, mock_state):
"""Test when element is not found by resource_id."""
mock_find_element.return_value = None
target = Target(
resource_id="com.example:id/nonexistent",
resource_id_index=None,
text=None,
text_index=None,
bounds=None,
)
result = asyncio.run(
move_cursor_to_end_if_bounds(ctx=mock_context, state=mock_state, target=target)
)
assert result is None assert result is None
class TestFocusElementIfNeeded: # ==============================================================================
"""Test cases for focus_element_if_needed function.""" # Element Focus Resolution Tests
# ==============================================================================
@patch("artemis.tools.utils.tap")
@patch("artemis.tools.utils.find_element_by_resource_id") class TestElementFocusing:
def test_focus_element_already_focused( """Validates conditional focusing behavior prior to keyboard inputs."""
self, mock_find_element, mock_tap, mock_context, sample_rich_element
@pytest.mark.asyncio
@patch("artemis.tools.utils.tap", new_callable=AsyncMock)
@patch("artemis.tools.utils.UnifiedMobileController")
async def test_skips_tap_when_already_focused(
self,
mock_controller_cls: MagicMock,
mock_tap_fn: AsyncMock,
dummy_context: MagicMock,
sample_hierarchy_node: dict,
): ):
"""Test when element is already focused.""" """Elements reporting focused state true should avoid redundant tap interactions."""
focused_element = sample_rich_element.copy() focused_elem = sample_hierarchy_node.copy()
focused_element["attributes"]["focused"] = "true" focused_elem["focused"] = "true"
mock_response = Mock() mock_ctrl = MagicMock()
mock_response.json.return_value = {"elements": [focused_element]} mock_ctrl.get_ui_elements = AsyncMock(return_value=[focused_elem])
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response) mock_controller_cls.return_value = mock_ctrl
mock_find_element.return_value = focused_element["attributes"]
target = Target( target = Target(resource_id="com.google.android.apps:id/search_query")
resource_id="com.example:id/text_input", focus_strategy = await focus_element_if_needed(ctx=dummy_context, target=target)
resource_id_index=None,
text=None,
text_index=None,
bounds=None,
)
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
mock_tap.assert_not_called() mock_tap_fn.assert_not_awaited()
assert result == "resource_id" assert focus_strategy == "resource_id"
mock_context.ui_adb_client.get_hierarchy.assert_called_once()
@patch("artemis.tools.utils.tap") @pytest.mark.asyncio
@patch("artemis.tools.utils.find_element_by_resource_id") @patch("artemis.tools.utils.tap", new_callable=AsyncMock)
def test_focus_element_needs_focus_success( @patch("artemis.tools.utils.UnifiedMobileController")
self, mock_find_element, mock_tap, mock_context, sample_rich_element async def test_focus_executes_tap_for_unfocused_element(
self,
mock_controller_cls: MagicMock,
mock_tap_fn: AsyncMock,
dummy_context: MagicMock,
sample_hierarchy_node: dict,
): ):
"""Test when element needs focus and focusing succeeds.""" """Unfocused element triggers focus tap and confirms state change."""
unfocused_element = sample_rich_element unfocused = sample_hierarchy_node.copy()
focused_element = { unfocused["focused"] = "false"
"attributes": { focused = sample_hierarchy_node.copy()
"resource-id": "com.example:id/text_input", focused["focused"] = "true"
"focused": "true",
},
"children": [],
}
mock_find_element.side_effect = [ mock_ctrl = MagicMock()
unfocused_element["attributes"], mock_ctrl.get_ui_elements = AsyncMock(side_effect=[[unfocused], [focused]])
focused_element["attributes"], mock_controller_cls.return_value = mock_ctrl
]
target = Target( target = Target(resource_id="com.google.android.apps:id/search_query")
resource_id="com.example:id/text_input", strategy = await focus_element_if_needed(ctx=dummy_context, target=target)
resource_id_index=None,
text=None,
text_index=None,
bounds=None,
)
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
mock_tap.assert_called_once_with( mock_tap_fn.assert_awaited_once_with(
ctx=mock_context, ctx=dummy_context,
selector_request=IdSelectorRequest(id="com.example:id/text_input"), selector_request=IdSelectorRequest(id="com.google.android.apps:id/search_query"),
index=0, index=0,
) )
assert mock_context.ui_adb_client.get_hierarchy.call_count == 2 assert strategy == "resource_id"
assert result == "resource_id"
@patch("artemis.tools.utils.tap") @pytest.mark.asyncio
@patch("artemis.tools.utils.logger") @patch("artemis.tools.utils.tap", new_callable=AsyncMock)
@patch("artemis.tools.utils.find_element_by_resource_id") @patch("artemis.tools.utils.UnifiedMobileController")
def test_focus_id_and_text_mismatch_fallback_to_text( async def test_focus_falls_back_to_text_coordinates(
self, self,
mock_find_id, mock_controller_cls: MagicMock,
mock_logger, mock_tap_fn: AsyncMock,
mock_tap, dummy_context: MagicMock,
mock_context, sample_hierarchy_node: dict,
sample_rich_element,
): ):
"""Test fallback when resource_id and text point to different elements.""" """When ID is unavailable, focusing falls back to text center tap coordinates."""
element_from_id = sample_rich_element["attributes"].copy() elem = sample_hierarchy_node.copy()
element_from_id["text"] = "Different text" elem["bounds"] = {"x": 20, "y": 40, "width": 200, "height": 60}
element_from_text = sample_rich_element.copy() mock_ctrl = MagicMock()
element_from_text["attributes"]["bounds"] = { mock_ctrl.get_ui_elements = AsyncMock(return_value=[elem])
"x": 10, mock_controller_cls.return_value = mock_ctrl
"y": 20,
"width": 100,
"height": 30,
}
mock_response = Mock() target = Target(text="Search Google")
mock_response.json.return_value = {"elements": [element_from_text]} strategy = await focus_element_if_needed(ctx=dummy_context, target=target)
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
mock_find_id.return_value = element_from_id
with patch("artemis.tools.utils.find_element_by_text") as mock_find_text: mock_tap_fn.assert_awaited_once()
mock_find_text.return_value = element_from_text["attributes"] req = mock_tap_fn.await_args.kwargs["selector_request"]
assert isinstance(req, SelectorRequestWithCoordinates)
target = Target( assert req.coordinates.x == 120 # 20 + 200/2
resource_id="com.example:id/text_input", assert req.coordinates.y == 70 # 40 + 60/2
resource_id_index=None, assert strategy == "text"
text="Sample text",
text_index=None,
bounds=None,
)
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
mock_logger.warning.assert_called_once()
mock_tap.assert_called_once()
assert result == "text"
@patch("artemis.tools.utils.tap")
@patch("artemis.tools.utils.find_element_by_text")
def test_focus_fallback_to_text(
self, mock_find_text, mock_tap, mock_context, sample_rich_element
):
"""Test fallback to focusing using text."""
element_with_bounds = sample_rich_element.copy()
element_with_bounds["attributes"]["bounds"] = {
"x": 10,
"y": 20,
"width": 100,
"height": 30,
}
mock_response = Mock()
mock_response.json.return_value = {"elements": [element_with_bounds]}
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
mock_find_text.return_value = element_with_bounds["attributes"]
target = Target(
resource_id=None,
resource_id_index=None,
text="Sample text",
text_index=None,
bounds=None,
)
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
mock_find_text.assert_called_once()
mock_tap.assert_called_once()
call_args = mock_tap.call_args[1]
selector = call_args["selector_request"]
assert isinstance(selector, SelectorRequestWithCoordinates)
assert selector.coordinates.x == 60 # 10 + 100/2
assert selector.coordinates.y == 35 # 20 + 30/2
assert result == "text"
@pytest.mark.asyncio
@patch("artemis.tools.utils.logger") @patch("artemis.tools.utils.logger")
def test_focus_all_locators_fail(self, mock_logger, mock_context): @patch("artemis.tools.utils.UnifiedMobileController")
"""Test failure when no locator can find an element.""" async def test_focus_aborts_when_no_locators_resolve(
self,
mock_controller_cls: MagicMock,
mock_logger: MagicMock,
dummy_context: MagicMock,
):
"""When none of the declared locators match an active element, returns None and logs error."""
mock_ctrl = MagicMock()
mock_ctrl.get_ui_elements = AsyncMock(return_value=[])
mock_controller_cls.return_value = mock_ctrl
mock_response = Mock() target = Target(resource_id="unknown_id", text="unknown_text")
mock_response.json.return_value = {"elements": []} strategy = await focus_element_if_needed(ctx=dummy_context, target=target)
mock_context.ui_adb_client.get_screen_data = Mock(return_value=mock_response)
with (
patch("artemis.tools.utils.find_element_by_resource_id") as mock_find_id,
patch("artemis.tools.utils.find_element_by_text") as mock_find_text,
):
mock_find_id.return_value = None
mock_find_text.return_value = None
target = Target( mock_logger.error.assert_called_once()
resource_id="nonexistent", assert strategy is None
resource_id_index=None,
text="nonexistent",
text_index=None,
bounds=None,
)
result = asyncio.run(focus_element_if_needed(ctx=mock_context, target=target))
mock_logger.error.assert_called_once_with(
"Failed to focus element." # ==============================================================================
+ " No valid locator (resource_id, coordinates, or text) succeeded." # Helper Verification & Selector Validation Tests
# ==============================================================================
class TestSelectorValidation:
"""Verifies target completeness and boundary constraint checks."""
def test_has_valid_selectors_matrix(self):
"""Verifies selector detection across valid and invalid combinations."""
assert has_valid_selectors(Target(resource_id="elem_id"))
assert has_valid_selectors(Target(text="Label"))
assert has_valid_selectors(Target(bounds=ElementBounds(x=0, y=0, width=10, height=10)))
assert not has_valid_selectors(Target())
def test_validate_coordinates_bounds(self):
"""Checks bounds containment against physical device display limits."""
valid_target = Target(bounds=ElementBounds(x=100, y=200, width=100, height=100))
assert validate_coordinates_bounds(valid_target, 1080, 2400) is None
out_width = Target(bounds=ElementBounds(x=1050, y=200, width=100, height=100))
assert validate_coordinates_bounds(out_width, 1080, 2400) is not None
out_height = Target(bounds=ElementBounds(x=100, y=2380, width=100, height=100))
assert validate_coordinates_bounds(out_height, 1080, 2400) is not None
def test_extract_selector_attributes(self):
"""Extracts resource ID and text from heterogeneous selector requests."""
req_id = IdSelectorRequest(id="view_item")
assert _extract_resource_id_and_text_from_selector(req_id) == ("view_item", None)
req_text = TextSelectorRequest(text="Confirm")
assert _extract_resource_id_and_text_from_selector(req_text) == (None, "Confirm")
req_both = IdWithTextSelectorRequest(id="submit_btn", text="Submit")
assert _extract_resource_id_and_text_from_selector(req_both) == ("submit_btn", "Submit")
req_coords = SelectorRequestWithCoordinates(
coordinates=CoordinatesSelectorRequest(x=50, y=50)
) )
assert result is None assert _extract_resource_id_and_text_from_selector(req_coords) == (None, None)
if __name__ == "__main__":
pytest.main([__file__])
+92 -142
View File
@@ -11,6 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
#
# Portions of this file are derived from mobile-use (https://github.com/minitap-ai/mobile-use)
# Copyright 2025-2026 Minitap, Inc. Licensed under the Apache License 2.0.
"""Unit tests validating Artemis UI hierarchy parsing and geometry computation."""
from unittest.mock import patch from unittest.mock import patch
@@ -29,215 +34,160 @@ def test_text_input_is_empty():
assert text_input_is_empty(text=None, hint_text=None) assert text_input_is_empty(text=None, hint_text=None)
assert text_input_is_empty(text="", hint_text=None) assert text_input_is_empty(text="", hint_text=None)
assert text_input_is_empty(text="", hint_text="") assert text_input_is_empty(text="", hint_text="")
assert text_input_is_empty(text="text", hint_text="text") assert text_input_is_empty(text="search", hint_text="search")
assert not text_input_is_empty(text="text", hint_text=None) assert not text_input_is_empty(text="search", hint_text=None)
assert not text_input_is_empty(text="text", hint_text="") assert not text_input_is_empty(text="search", hint_text="")
def test_find_element_by_resource_id(): def test_find_element_by_resource_id():
ui_hierarchy = [ tree = [
{ {
"resourceId": "com.example:id/button1", "resourceId": "com.google.android.settings:id/search_bar",
"text": "Button 1", "text": "Search Settings",
"children": [], "children": [],
}, },
{ {
"resourceId": "com.example:id/container", "resourceId": "com.google.android.settings:id/content_frame",
"children": [ "children": [
{ {
"resourceId": "com.example:id/nested_button", "resourceId": "com.google.android.settings:id/network_item",
"text": "Nested Button", "text": "Network & Internet",
"children": [], "children": [],
} }
], ],
}, },
] ]
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button1") target = find_element_by_resource_id(tree, "com.google.android.settings:id/search_bar")
assert result is not None assert target is not None
assert result["resourceId"] == "com.example:id/button1" assert target["resourceId"] == "com.google.android.settings:id/search_bar"
assert result["text"] == "Button 1" assert target["text"] == "Search Settings"
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nested_button") nested = find_element_by_resource_id(tree, "com.google.android.settings:id/network_item")
assert result is not None assert nested is not None
assert result["resourceId"] == "com.example:id/nested_button" assert nested["resourceId"] == "com.google.android.settings:id/network_item"
assert result["text"] == "Nested Button"
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/nonexistent") assert find_element_by_resource_id(tree, "com.google.android.settings:id/missing") is None
assert result is None assert find_element_by_resource_id([], "com.google.android.settings:id/search_bar") is None
result = find_element_by_resource_id([], "com.example:id/button1")
assert result is None
def test_find_element_by_resource_id_with_index(): def test_find_element_by_resource_id_with_index():
ui_hierarchy = [ duplicate_items = [
{"resourceId": "com.artemis.ui:id/row_item", "text": "Item Alpha", "children": []},
{"resourceId": "com.artemis.ui:id/row_item", "text": "Item Beta", "children": []},
{ {
"resourceId": "com.example:id/button", "resourceId": "com.artemis.ui:id/section_group",
"text": "Button 1",
"children": [],
},
{
"resourceId": "com.example:id/button",
"text": "Button 2",
"children": [],
},
{
"resourceId": "com.example:id/container",
"children": [ "children": [
{ {"resourceId": "com.artemis.ui:id/row_item", "text": "Item Gamma", "children": []}
"resourceId": "com.example:id/button",
"text": "Button 3",
"children": [],
}
], ],
}, },
] ]
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=0) first = find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=0)
assert result is not None assert first is not None
assert result["text"] == "Button 1" assert first["text"] == "Item Alpha"
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=1) second = find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=1)
assert result is not None assert second is not None
assert result["text"] == "Button 2" assert second["text"] == "Item Beta"
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=2) third = find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=2)
assert result is not None assert third is not None
assert result["text"] == "Button 3" assert third["text"] == "Item Gamma"
result = find_element_by_resource_id(ui_hierarchy, "com.example:id/button", index=3) assert find_element_by_resource_id(duplicate_items, "com.artemis.ui:id/row_item", index=3) is None
assert result is None
def test_find_element_by_resource_id_rich_hierarchy(): def test_find_element_by_resource_id_rich_hierarchy():
rich_hierarchy = [ rich_nodes = [
{ {
"attributes": {"resource-id": "com.example:id/button1"}, "attributes": {"resource-id": "com.android.systemui:id/clock"},
"children": [], "children": [],
}, },
{ {
"attributes": {"resource-id": "com.example:id/container"}, "attributes": {"resource-id": "com.android.systemui:id/status_bar"},
"children": [ "children": [
{ {
"attributes": {"resource-id": "com.example:id/nested_button"}, "attributes": {"resource-id": "com.android.systemui:id/battery_meter"},
"children": [], "children": [],
} }
], ],
}, },
] ]
result = find_element_by_resource_id( match = find_element_by_resource_id(
rich_hierarchy, "com.example:id/button1", is_rich_hierarchy=True rich_nodes, "com.android.systemui:id/clock", is_rich_hierarchy=True
) )
assert result is not None assert match is not None
assert result["resource-id"] == "com.example:id/button1" assert match["resource-id"] == "com.android.systemui:id/clock"
result = find_element_by_resource_id( nested_match = find_element_by_resource_id(
rich_hierarchy, "com.example:id/nested_button", is_rich_hierarchy=True rich_nodes, "com.android.systemui:id/battery_meter", is_rich_hierarchy=True
) )
assert result is not None assert nested_match is not None
assert result["resource-id"] == "com.example:id/nested_button" assert nested_match["resource-id"] == "com.android.systemui:id/battery_meter"
result = find_element_by_resource_id( assert find_element_by_resource_id(
rich_hierarchy, "com.example:id/nonexistent", is_rich_hierarchy=True rich_nodes, "com.android.systemui:id/unknown", is_rich_hierarchy=True
) ) is None
assert result is None
def test_is_element_focused(): def test_is_element_focused():
focused_element = {"focused": "true"} assert is_element_focused({"focused": "true"})
assert is_element_focused(focused_element) assert is_element_focused({"focused": True})
assert not is_element_focused({"focused": "false"})
non_focused_element = {"focused": "false"} assert not is_element_focused({"focused": False})
assert not is_element_focused(non_focused_element) assert not is_element_focused({"text": "sample"})
assert not is_element_focused({"focused": None})
no_focused_element = {"text": "some text"}
assert not is_element_focused(no_focused_element)
none_focused_element = {"focused": None}
assert not is_element_focused(none_focused_element)
def test_get_element_text(): def test_get_element_text():
element = {"text": "Button Text", "hintText": "Hint Text"} node = {"text": "Battery", "hintText": "Search settings"}
assert get_element_text(element) == "Button Text" assert get_element_text(node) == "Battery"
assert get_element_text(element, hint_text=False) == "Button Text" assert get_element_text(node, hint_text=False) == "Battery"
assert get_element_text(element, hint_text=True) == "Hint Text" assert get_element_text(node, hint_text=True) == "Search settings"
element_no_text = {"hintText": "Hint Text"} node_no_text = {"hintText": "Placeholder"}
assert get_element_text(element_no_text) is None assert get_element_text(node_no_text) is None
assert get_element_text(element_no_text, hint_text=True) == "Hint Text" assert get_element_text(node_no_text, hint_text=True) == "Placeholder"
element_no_hint = {"text": "Button Text"}
assert get_element_text(element_no_hint) == "Button Text"
assert get_element_text(element_no_hint, hint_text=True) is None
empty_element = {} assert get_element_text({}) is None
assert get_element_text(empty_element) is None
assert get_element_text(empty_element, hint_text=True) is None
def test_get_bounds_for_element(): def test_get_bounds_for_element():
element_with_bounds = {"bounds": {"x": 10, "y": 20, "width": 100, "height": 50}} node_with_box = {"bounds": {"x": 24, "y": 48, "width": 400, "height": 80}}
bounds = get_bounds_for_element(element_with_bounds) box = get_bounds_for_element(node_with_box)
assert bounds is not None assert box is not None
assert isinstance(bounds, ElementBounds) assert isinstance(box, ElementBounds)
assert bounds.x == 10 assert box.x == 24
assert bounds.y == 20 assert box.y == 48
assert bounds.width == 100 assert box.width == 400
assert bounds.height == 50 assert box.height == 80
element_no_bounds = {"text": "Button"} assert get_bounds_for_element({"text": "Click"}) is None
bounds = get_bounds_for_element(element_no_bounds)
assert bounds is None
# Suppress logger output for the invalid bounds test case
with patch("artemis.utils.ui_hierarchy.logger.error"): with patch("artemis.utils.ui_hierarchy.logger.error"):
element_invalid_bounds = { bad_box = {"bounds": {"x": "corrupted", "y": 48, "width": 400, "height": 80}}
"bounds": { assert get_bounds_for_element(bad_box) is None
"x": "invalid", # Should be int
"y": 20,
"width": 100,
"height": 50,
}
}
bounds = get_bounds_for_element(element_invalid_bounds)
assert bounds is None
def test_element_bounds(): def test_element_bounds_geometry():
bounds = ElementBounds(x=10, y=20, width=100, height=50) box = ElementBounds(x=100, y=200, width=500, height=100)
center = bounds.get_center() center = box.get_center()
assert isinstance(center, Point) assert isinstance(center, Point)
assert center.x == 60 assert center.x == 350
assert center.y == 45 assert center.y == 250
center_point = bounds.get_relative_point(0.5, 0.5) mid = box.get_relative_point(0.5, 0.5)
assert isinstance(center_point, Point) assert mid.x == 350
assert center_point.x == 60 assert mid.y == 250
assert center_point.y == 45
top_left = bounds.get_relative_point(0.0, 0.0) top_left = box.get_relative_point(0.0, 0.0)
assert top_left.x == 10 assert top_left.x == 100
assert top_left.y == 20 assert top_left.y == 200
bottom_right = bounds.get_relative_point(1.0, 1.0) bottom_right = box.get_relative_point(1.0, 1.0)
assert bottom_right.x == 110 assert bottom_right.x == 600
assert bottom_right.y == 70 assert bottom_right.y == 300
custom_point = bounds.get_relative_point(0.95, 0.95)
assert custom_point.x == 105
assert custom_point.y == 67
if __name__ == "__main__":
test_text_input_is_empty()
test_find_element_by_resource_id()
test_find_element_by_resource_id_with_index()
test_find_element_by_resource_id_rich_hierarchy()
test_is_element_focused()
test_get_element_text()
test_get_bounds_for_element()
test_element_bounds()
print("All tests passed")