feat: initial release of ARTEMIS Android Agent & Automation Platform
ARTEMIS is an autonomous multimodal Android agent and mobile UI automation platform engineered for physical devices and emulators. It combines state-of-the-art multimodal Vision-Language Models (VLMs) with hierarchical closed-loop planning, resilient self-healing, and deep Android subsystem integration. Key Features & Architectural Highlights: - Dual-Engine Orchestration: * Pro Engine: Multi-agent closed-loop (Planner -> Operator -> Validator/Failure Analyzer -> Summarizer) powered by LangGraph. * Flash Engine: Ultra-fast reactive loop for deterministic, low-latency UI execution (<35 steps). - Multimodal Perception & Grounding: * Hybrid visual screenshot grounding, real-time OCR extraction, and XML accessibility hierarchy parsing. * Dynamic-First, Coordinate-Fallback locating strategy for maximum resilience against UI drift. - Autonomous Self-Healing & Diagnostics: * Failure analyzer and safety net for popup dismissal, crash recovery, and auto-retry. * Diagnostic sub-agents for log analysis, video recording inspection, and object detection. - Unified CLI & Python SDK: * Rich CLI with commands: init, doctor, run, trace, batch, bench, server, and mcp. * Fluent Python SDK with Pydantic structured output models and package locking. - Multi-Provider LLM Architecture: * Support for Google Gemini, OpenAI, Anthropic Claude, Vertex AI, OpenRouter, xAI Grok, and local endpoints (Ollama/vLLM). * Centralized configuration in config/artemis.jsonc. - Native MCP & Observability: * Model Context Protocol (MCP) server endpoints for AI IDEs. * Full trace recording with before/after screenshots, action overlays, and execution replays. Change-Id: I430b93ee2b76b54cc5a3c6eeccd15d2cf29830a1
This commit is contained in:
somew1nd committed
commit
206a9ce921
229 files changed
+49653
No files matched your search
@@ -0,0 +1,45 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Universal device action tools."""
|
||||
|
||||
from artemis.tools.actions.device_actions import (
|
||||
ClickArgs,
|
||||
InputTextArgs,
|
||||
LongPressArgs,
|
||||
PressKeyArgs,
|
||||
SwipeArgs,
|
||||
WaitForDelayArgs,
|
||||
click,
|
||||
input_text,
|
||||
long_press,
|
||||
press_key,
|
||||
swipe,
|
||||
wait_for_delay,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"click",
|
||||
"long_press",
|
||||
"input_text",
|
||||
"swipe",
|
||||
"press_key",
|
||||
"wait_for_delay",
|
||||
"ClickArgs",
|
||||
"LongPressArgs",
|
||||
"InputTextArgs",
|
||||
"SwipeArgs",
|
||||
"PressKeyArgs",
|
||||
"WaitForDelayArgs",
|
||||
]
|
||||
@@ -0,0 +1,237 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Universal device action tools implemented on top of BaseDeviceDriver."""
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Literal
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.tools.base import artemis_tool
|
||||
from artemis.utils.coordinates import parse_swipe_parameters
|
||||
from artemis.utils.logger import get_logger
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
# Schemas
|
||||
class ClickArgs(BaseModel):
|
||||
target: list[int] = Field(
|
||||
..., description="Click target normalized coordinates [x, y] in 0-1000 scale."
|
||||
)
|
||||
|
||||
|
||||
class LongPressArgs(BaseModel):
|
||||
target: list[int] = Field(
|
||||
..., description="Long press target normalized coordinates [x, y] in 0-1000 scale."
|
||||
)
|
||||
duration: int = Field(1000, description="Long press duration in milliseconds (default 1000).")
|
||||
|
||||
|
||||
class InputTextArgs(BaseModel):
|
||||
text: str = Field(..., description="The text content to input.")
|
||||
target: list[int] = Field(
|
||||
..., description="Input target field normalized coordinates [x, y] in 0-1000 scale."
|
||||
)
|
||||
clear_exist: bool = Field(
|
||||
True, description="Whether to clear existing text in the input box before typing."
|
||||
)
|
||||
|
||||
|
||||
class SwipeArgs(BaseModel):
|
||||
direction: Literal["up", "down", "left", "right"] | None = Field(
|
||||
None,
|
||||
description=(
|
||||
"Smart directional swipe ('up', 'down', 'left', 'right') for general page"
|
||||
" scrolling/browsing. Automatically applies safe vectors and a ~40% visual"
|
||||
" overlap anchor: 'up' (drags bottom-to-top, scrolling down to reveal content below),"
|
||||
" 'down' (drags top-to-bottom, scrolling up to reveal content above),"
|
||||
" 'left' (drags right-to-left, scrolling right),"
|
||||
" 'right' (drags left-to-right, scrolling left)."
|
||||
),
|
||||
)
|
||||
start: list[int] | None = Field(
|
||||
None,
|
||||
description=(
|
||||
"Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise,"
|
||||
" local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)."
|
||||
),
|
||||
)
|
||||
end: list[int] | None = Field(
|
||||
None,
|
||||
description=(
|
||||
"End normalized coordinates [end_x, end_y] in 0-1000 scale for precise,"
|
||||
" local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)."
|
||||
),
|
||||
)
|
||||
action: Literal["up", "down", "left", "right"] | list[int] | None = Field(
|
||||
None,
|
||||
description=(
|
||||
"Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right')"
|
||||
" OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale."
|
||||
),
|
||||
)
|
||||
duration: int = Field(
|
||||
400,
|
||||
description=(
|
||||
"Swipe/drag duration in milliseconds (default 400). For drag-and-drop, "
|
||||
"list reordering, or sliding/adjusting sliders (e.g. volume, brightness, SeekBars), "
|
||||
"set duration >= 1000 (e.g. 1500)."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class PressKeyArgs(BaseModel):
|
||||
key: str = Field(
|
||||
...,
|
||||
description="Key name to press (e.g. 'home', 'back', 'enter', 'delete', 'power').",
|
||||
)
|
||||
|
||||
|
||||
class WaitForDelayArgs(BaseModel):
|
||||
seconds: float = Field(
|
||||
..., description="Duration in seconds to wait for animations, pages, or state updates."
|
||||
)
|
||||
|
||||
|
||||
# Universal Tool Definitions
|
||||
@artemis_tool(
|
||||
name="click",
|
||||
description="[ACTION] Clicks on the specified screen coordinate [x, y] in normalized 0-1000 scale.",
|
||||
args_schema=ClickArgs,
|
||||
category="action",
|
||||
)
|
||||
async def click(target: list[int], driver: BaseDeviceDriver) -> str:
|
||||
width, height = driver.screen_size
|
||||
abs_x = int(target[0] * width / 1000.0)
|
||||
abs_y = int(target[1] * height / 1000.0)
|
||||
success = await driver.tap(abs_x, abs_y)
|
||||
return (
|
||||
f"Click executed at normalized {target} (pixels: {abs_x}, {abs_y})"
|
||||
if success
|
||||
else f"Click failed at {target}"
|
||||
)
|
||||
|
||||
|
||||
@artemis_tool(
|
||||
name="long_press",
|
||||
description="[ACTION] Long presses on the specified screen coordinate [x, y] in normalized 0-1000 scale.",
|
||||
args_schema=LongPressArgs,
|
||||
category="action",
|
||||
)
|
||||
async def long_press(
|
||||
target: list[int], duration: int = 1000, driver: BaseDeviceDriver = None
|
||||
) -> str:
|
||||
width, height = driver.screen_size
|
||||
abs_x = int(target[0] * width / 1000.0)
|
||||
abs_y = int(target[1] * height / 1000.0)
|
||||
success = await driver.long_press(abs_x, abs_y, duration_ms=duration)
|
||||
return (
|
||||
f"Long press executed at normalized {target} for {duration}ms"
|
||||
if success
|
||||
else f"Long press failed at {target}"
|
||||
)
|
||||
|
||||
|
||||
@artemis_tool(
|
||||
name="input_text",
|
||||
description="[ACTION] Clicks an input field target [x, y] and types text into it.",
|
||||
args_schema=InputTextArgs,
|
||||
category="action",
|
||||
)
|
||||
async def input_text(
|
||||
text: str, target: list[int], clear_exist: bool = True, driver: BaseDeviceDriver = None
|
||||
) -> str:
|
||||
# First tap to focus
|
||||
width, height = driver.screen_size
|
||||
abs_x = int(target[0] * width / 1000.0)
|
||||
abs_y = int(target[1] * height / 1000.0)
|
||||
await driver.tap(abs_x, abs_y)
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
# Then input text
|
||||
success = await driver.input_text(text, clear_existing=clear_exist)
|
||||
return (
|
||||
f"Input text '{text}' into target {target}"
|
||||
if success
|
||||
else f"Failed to input text into {target}"
|
||||
)
|
||||
|
||||
|
||||
@artemis_tool(
|
||||
name="swipe",
|
||||
description=(
|
||||
"[ACTION] Performs smart directional swipe for browsing, or precise coordinate drag/slider gestures."
|
||||
),
|
||||
args_schema=SwipeArgs,
|
||||
category="action",
|
||||
)
|
||||
async def swipe(
|
||||
action: Any = None,
|
||||
direction: str | None = None,
|
||||
start: list[int] | None = None,
|
||||
end: list[int] | None = None,
|
||||
duration: int = 400,
|
||||
driver: BaseDeviceDriver = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
params = dict(kwargs)
|
||||
if action is not None:
|
||||
params["action"] = action
|
||||
if direction is not None:
|
||||
params["direction"] = direction
|
||||
if start is not None:
|
||||
params["start"] = start
|
||||
if end is not None:
|
||||
params["end"] = end
|
||||
params["duration"] = duration
|
||||
|
||||
kind, target, dur = parse_swipe_parameters(params, default_duration=duration)
|
||||
|
||||
if kind == "direction" and isinstance(target, str):
|
||||
success = await driver.swipe_direction(target, duration_ms=dur)
|
||||
return f"Swiped {target}" if success else f"Swipe {target} failed"
|
||||
elif kind == "coords" and isinstance(target, list) and len(target) == 4:
|
||||
width, height = driver.screen_size
|
||||
start_x = int(target[0] * width / 1000.0)
|
||||
start_y = int(target[1] * height / 1000.0)
|
||||
end_x = int(target[2] * width / 1000.0)
|
||||
end_y = int(target[3] * height / 1000.0)
|
||||
success = await driver.swipe(start_x, start_y, end_x, end_y, duration_ms=dur)
|
||||
return (
|
||||
f"Swiped from {target[:2]} to {target[2:]}" if success else "Swipe coordinates failed"
|
||||
)
|
||||
return f"Invalid swipe action parameter: {action or direction or (start, end)}"
|
||||
|
||||
|
||||
@artemis_tool(
|
||||
name="press_key",
|
||||
description="[ACTION] Presses a hardware or system key on the device ('home', 'back', 'enter', 'delete', 'power').",
|
||||
args_schema=PressKeyArgs,
|
||||
category="action",
|
||||
)
|
||||
async def press_key(key: str, driver: BaseDeviceDriver) -> str:
|
||||
success = await driver.press_key(key)
|
||||
return f"Key '{key}' pressed" if success else f"Failed to press key '{key}'"
|
||||
|
||||
|
||||
@artemis_tool(
|
||||
name="wait_for_delay",
|
||||
description="[ACTION] Pauses execution for a specified number of seconds to allow UI loading or transition.",
|
||||
args_schema=WaitForDelayArgs,
|
||||
category="action",
|
||||
)
|
||||
async def wait_for_delay(seconds: float, driver: BaseDeviceDriver = None) -> str:
|
||||
await asyncio.sleep(seconds)
|
||||
return f"Waited for {seconds} seconds."
|
||||
@@ -0,0 +1,274 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Universal Tool Protocol and Registry for ARTEMIS.
|
||||
|
||||
Enables tools to be authored once using standard Pydantic schemas and async handlers,
|
||||
then automatically exported to LangChain Tools (Pro Graph), Google GenAI FunctionDeclarations
|
||||
(FlashRunner), and MCP Tools (FastMCP) with zero code duplication.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Callable
|
||||
import concurrent.futures
|
||||
import inspect
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from google.genai import types as genai_types
|
||||
from langchain_core.tools import BaseTool, StructuredTool
|
||||
from langgraph.prebuilt import InjectedState
|
||||
from pydantic import BaseModel
|
||||
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.drivers.factory import get_driver
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
ToolCategory = Literal["action", "perception", "system", "memory", "custom"]
|
||||
|
||||
|
||||
class ArtemisTool:
|
||||
"""Unified tool wrapper encapsulating schema, execution logic, and multi-protocol export."""
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
description: str,
|
||||
args_schema: type[BaseModel],
|
||||
handler: Callable[..., Any] | None = None,
|
||||
category: ToolCategory = "action",
|
||||
):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.args_schema = args_schema
|
||||
self.handler = handler
|
||||
self.category = category
|
||||
|
||||
async def execute(self, driver: BaseDeviceDriver, ctx: Any, **kwargs: Any) -> Any:
|
||||
"""Executes the tool handler with injected driver and context."""
|
||||
# Inspect handler signature to only pass supported arguments
|
||||
if self.handler is not None:
|
||||
sig = inspect.signature(self.handler)
|
||||
call_kwargs = {}
|
||||
for param_name in sig.parameters:
|
||||
if param_name == "driver":
|
||||
call_kwargs["driver"] = driver
|
||||
elif param_name in ("ctx", "context"):
|
||||
call_kwargs[param_name] = ctx
|
||||
elif param_name in kwargs:
|
||||
call_kwargs[param_name] = kwargs[param_name]
|
||||
|
||||
if inspect.iscoroutinefunction(self.handler):
|
||||
return await self.handler(**call_kwargs)
|
||||
return self.handler(**call_kwargs)
|
||||
raise NotImplementedError("Subclasses must implement execute() or provide a handler.")
|
||||
|
||||
async def __call__(self, *args: Any, **kwargs: Any) -> Any:
|
||||
"""Allows ArtemisTool instances to be invoked directly as callables."""
|
||||
driver = kwargs.pop("driver", None)
|
||||
ctx = kwargs.pop("ctx", None) or kwargs.pop("context", None)
|
||||
return await self.execute(driver=driver, ctx=ctx, *args, **kwargs)
|
||||
|
||||
def to_langchain_tool(self, ctx: Any, name: str | None = None) -> BaseTool:
|
||||
"""Exports this tool to a LangChain StructuredTool."""
|
||||
|
||||
def _get_context_driver() -> BaseDeviceDriver | None:
|
||||
if (
|
||||
ctx is not None
|
||||
and hasattr(ctx, "device")
|
||||
and ctx.device
|
||||
and getattr(ctx.device, "device_id", None)
|
||||
):
|
||||
try:
|
||||
return get_driver(ctx)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.debug(f"Failed to get driver: {e}")
|
||||
return None
|
||||
return getattr(ctx, "_active_driver", None) if ctx is not None else None
|
||||
|
||||
def _func(state: Annotated[Any, InjectedState] = None, **kwargs):
|
||||
driver = _get_context_driver()
|
||||
if state is not None:
|
||||
kwargs["state"] = state
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
|
||||
if loop and loop.is_running():
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
return pool.submit(
|
||||
asyncio.run, self.execute(driver=driver, ctx=ctx, **kwargs)
|
||||
).result()
|
||||
return asyncio.run(self.execute(driver=driver, ctx=ctx, **kwargs))
|
||||
|
||||
async def _coroutine(state: Annotated[Any, InjectedState] = None, **kwargs):
|
||||
driver = _get_context_driver()
|
||||
if state is not None:
|
||||
kwargs["state"] = state
|
||||
return await self.execute(driver=driver, ctx=ctx, **kwargs)
|
||||
|
||||
return StructuredTool.from_function(
|
||||
func=_func,
|
||||
coroutine=_coroutine,
|
||||
name=name or self.name,
|
||||
description=self.description,
|
||||
args_schema=self.args_schema,
|
||||
)
|
||||
|
||||
def to_genai_declaration(self) -> genai_types.FunctionDeclaration:
|
||||
"""Exports this tool to a Google GenAI FunctionDeclaration schema."""
|
||||
properties = {}
|
||||
required = []
|
||||
|
||||
if self.args_schema:
|
||||
schema = self.args_schema.model_json_schema()
|
||||
props = schema.get("properties", {})
|
||||
required = schema.get("required", [])
|
||||
|
||||
for prop_name, prop_data in props.items():
|
||||
p_type_str = prop_data.get("type")
|
||||
items_data = prop_data.get("items")
|
||||
if not p_type_str and "anyOf" in prop_data:
|
||||
for sub_schema in prop_data["anyOf"]:
|
||||
sub_type = sub_schema.get("type")
|
||||
if sub_type and sub_type != "null":
|
||||
p_type_str = sub_type
|
||||
if "items" in sub_schema and not items_data:
|
||||
items_data = sub_schema["items"]
|
||||
break
|
||||
if not p_type_str:
|
||||
p_type_str = "string"
|
||||
|
||||
p_desc = prop_data.get("description", "")
|
||||
|
||||
genai_type = genai_types.Type.STRING
|
||||
if p_type_str == "integer":
|
||||
genai_type = genai_types.Type.INTEGER
|
||||
elif p_type_str == "number":
|
||||
genai_type = genai_types.Type.NUMBER
|
||||
elif p_type_str == "boolean":
|
||||
genai_type = genai_types.Type.BOOLEAN
|
||||
elif p_type_str == "array":
|
||||
genai_type = genai_types.Type.ARRAY
|
||||
elif p_type_str == "object":
|
||||
genai_type = genai_types.Type.OBJECT
|
||||
|
||||
# Handle array items if present
|
||||
items_schema = None
|
||||
if genai_type == genai_types.Type.ARRAY:
|
||||
item_type_str = (items_data or {}).get("type", "string")
|
||||
item_genai_type = (
|
||||
genai_types.Type.INTEGER
|
||||
if item_type_str == "integer"
|
||||
else genai_types.Type.STRING
|
||||
)
|
||||
items_schema = genai_types.Schema(type=item_genai_type)
|
||||
|
||||
properties[prop_name] = genai_types.Schema(
|
||||
type=genai_type,
|
||||
description=p_desc,
|
||||
items=items_schema,
|
||||
)
|
||||
|
||||
return genai_types.FunctionDeclaration(
|
||||
name=self.name,
|
||||
description=self.description,
|
||||
parameters=genai_types.Schema(
|
||||
type=genai_types.Type.OBJECT,
|
||||
properties=properties,
|
||||
required=required,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class ToolRegistry:
|
||||
"""Global registry managing universal tools."""
|
||||
|
||||
_tools: dict[str, ArtemisTool] = {}
|
||||
|
||||
@classmethod
|
||||
def register(cls, tool: ArtemisTool) -> ArtemisTool:
|
||||
"""Registers a tool in the global registry."""
|
||||
cls._tools[tool.name] = tool
|
||||
return tool
|
||||
|
||||
@classmethod
|
||||
def get(cls, name: str) -> ArtemisTool | None:
|
||||
"""Retrieves a tool by name from the registry."""
|
||||
normalized = name.split(":")[-1] if ":" in name else name
|
||||
return cls._tools.get(normalized)
|
||||
|
||||
@classmethod
|
||||
def list_tools(cls, category: ToolCategory | None = None) -> list[ArtemisTool]:
|
||||
"""Lists all registered tools, optionally filtered by category."""
|
||||
if category:
|
||||
return [t for t in cls._tools.values() if t.category == category]
|
||||
return list(cls._tools.values())
|
||||
|
||||
@classmethod
|
||||
def get_langchain_tools(cls, ctx: Any, names: list[str] | None = None) -> list[BaseTool]:
|
||||
"""Returns LangChain tool wrappers for the requested or all registered tools."""
|
||||
tools = (
|
||||
[cls._tools[n] for n in names if n in cls._tools]
|
||||
if names
|
||||
else list(cls._tools.values())
|
||||
)
|
||||
return [t.to_langchain_tool(ctx) for t in tools]
|
||||
|
||||
@classmethod
|
||||
def get_genai_declarations(
|
||||
cls, names: list[str] | None = None
|
||||
) -> list[genai_types.FunctionDeclaration]:
|
||||
"""Returns Google GenAI function declarations for registered tools."""
|
||||
tools = (
|
||||
[cls._tools[n] for n in names if n in cls._tools]
|
||||
if names
|
||||
else list(cls._tools.values())
|
||||
)
|
||||
return [t.to_genai_declaration() for t in tools]
|
||||
|
||||
@classmethod
|
||||
async def execute(
|
||||
cls, name: str, args: dict[str, Any], driver: BaseDeviceDriver, ctx: Any
|
||||
) -> Any:
|
||||
"""Executes a registered tool by name with provided arguments, driver, and context."""
|
||||
tool = cls.get(name)
|
||||
if not tool:
|
||||
raise ValueError(f"Tool '{name}' not found in ToolRegistry.")
|
||||
return await tool.execute(driver=driver, ctx=ctx, **args)
|
||||
|
||||
|
||||
def artemis_tool(
|
||||
name: str,
|
||||
description: str,
|
||||
args_schema: type[BaseModel],
|
||||
category: ToolCategory = "action",
|
||||
):
|
||||
"""Decorator to define and register a universal ARTEMIS tool."""
|
||||
|
||||
def decorator(fn: Callable[..., Any]) -> ArtemisTool:
|
||||
tool_obj = ArtemisTool(
|
||||
name=name,
|
||||
description=description,
|
||||
args_schema=args_schema,
|
||||
handler=fn,
|
||||
category=category,
|
||||
)
|
||||
ToolRegistry.register(tool_obj)
|
||||
return tool_obj
|
||||
|
||||
return decorator
|
||||
@@ -0,0 +1,836 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Universal ADB command and background task management tools for ARTEMIS."""
|
||||
|
||||
import asyncio
|
||||
from asyncio.subprocess import Process as AsyncProcess
|
||||
import os
|
||||
from typing import Any, Literal
|
||||
import uuid
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.agents.log_analyzer.output_analyzer import TaskOutputAnalyzerNode
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.controllers.platform_specific_commands_controller import (
|
||||
get_adb_device,
|
||||
)
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.tool_wrapper import ToolWrapper
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Global persistent Android shell environments
|
||||
_PERSISTENT_ENVIRONMENTS: dict[str, dict[str, str]] = {}
|
||||
# Global background tasks
|
||||
_BACKGROUND_TASKS: dict[str, "BackgroundTask"] = {}
|
||||
# Finished tasks history logs cache (to support Action='analyze' even after tasks are unregistered)
|
||||
_FINISHED_TASKS_LOGS: dict[str, dict[str, Any]] = {}
|
||||
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
def _register_finished_task(
|
||||
task_id: str,
|
||||
command: str,
|
||||
cwd: str | None,
|
||||
terminal_id: str | None,
|
||||
status: str,
|
||||
exit_code: int | None,
|
||||
output: str,
|
||||
):
|
||||
"""Caches execution details for finished tasks."""
|
||||
_FINISHED_TASKS_LOGS[task_id] = {
|
||||
"task_id": task_id,
|
||||
"command": command,
|
||||
"cwd": cwd,
|
||||
"terminal_id": terminal_id,
|
||||
"status": status,
|
||||
"exit_code": exit_code,
|
||||
"output": output,
|
||||
"notified": False,
|
||||
}
|
||||
# Keep memory bounded
|
||||
if len(_FINISHED_TASKS_LOGS) > 10:
|
||||
oldest_key = next(iter(_FINISHED_TASKS_LOGS))
|
||||
del _FINISHED_TASKS_LOGS[oldest_key]
|
||||
|
||||
|
||||
def _is_output_long(output: str) -> bool:
|
||||
"""Checks if command output exceeds the max threshold for inline display."""
|
||||
return len(output.splitlines()) > 500 or len(output) > 50000
|
||||
|
||||
|
||||
def _format_long_output_response(_task_id: str, output: str, intro: str) -> str:
|
||||
"""Formats truncated output with guidance to use the analyzer tool."""
|
||||
lines = output.splitlines()
|
||||
last_200 = "\n".join(lines[-200:])
|
||||
return (
|
||||
f"{intro}\nWarning: Output is too long ({len(lines)} lines,"
|
||||
f" {len(output)} chars) and has been truncated.\n--- Last 200 lines of"
|
||||
f" output ---\n{last_200}\n------------------------\nYou can use the"
|
||||
" 'analyze_task_output' tool to get more information."
|
||||
)
|
||||
|
||||
|
||||
def _get_task_info(task_id: str, ctx: ArtemisContext) -> dict[str, Any] | None:
|
||||
"""Retrieves metadata and logs for a running or finished task."""
|
||||
if task_id in _BACKGROUND_TASKS:
|
||||
t = _BACKGROUND_TASKS[task_id]
|
||||
return {
|
||||
"command": t.command,
|
||||
"cwd": t.cwd,
|
||||
"terminal_id": t.terminal_id,
|
||||
"status": t.status,
|
||||
"output": "".join(t.stdout_log),
|
||||
}
|
||||
if task_id in _FINISHED_TASKS_LOGS:
|
||||
return _FINISHED_TASKS_LOGS[task_id]
|
||||
if ctx and ctx.data_engine:
|
||||
db_tasks = ctx.data_engine.get_all_background_tasks()
|
||||
for db_t in db_tasks:
|
||||
if db_t.get("task_id") == task_id:
|
||||
return {
|
||||
"command": db_t.get("summary") or "",
|
||||
"cwd": None,
|
||||
"terminal_id": None,
|
||||
"status": db_t.get("status") or "completed",
|
||||
"output": db_t.get("logs") or "",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
# pylint: disable=too-many-instance-attributes
|
||||
class BackgroundTask:
|
||||
"""Represents an active or completing background ADB shell task."""
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
def __init__(
|
||||
self,
|
||||
task_id: str,
|
||||
command: str,
|
||||
process: AsyncProcess,
|
||||
terminal_id: str | None = None,
|
||||
cwd: str | None = None,
|
||||
):
|
||||
self.task_id = task_id
|
||||
self.command = command
|
||||
self.process = process
|
||||
self.terminal_id = terminal_id
|
||||
self.cwd = cwd
|
||||
self.stdout_log: list[str] = []
|
||||
self.status = "running"
|
||||
self.exit_code: int | None = None
|
||||
self.trace_id: uuid.UUID | None = None
|
||||
self.listener_task: asyncio.Task | None = None
|
||||
|
||||
async def start(self, ctx: ArtemisContext):
|
||||
"""Starts the background output listener task."""
|
||||
self.listener_task = asyncio.create_task(self._listen(ctx))
|
||||
|
||||
async def _listen(self, ctx: ArtemisContext):
|
||||
"""Reads process output streams and updates lifecycle status."""
|
||||
try:
|
||||
# Read stdout/stderr merged
|
||||
while True:
|
||||
line = await self.process.stdout.readline()
|
||||
if not line:
|
||||
break
|
||||
decoded = line.decode()
|
||||
self.stdout_log.append(decoded)
|
||||
if ctx and ctx.data_engine and getattr(self, "trace_id", None):
|
||||
ctx.data_engine.stream_output(self.trace_id, decoded)
|
||||
|
||||
self.exit_code = await self.process.wait()
|
||||
self.status = "completed" if self.exit_code == 0 else "failed"
|
||||
logger.info(
|
||||
f"Background ADB task {self.task_id} completed with exit code {self.exit_code}"
|
||||
)
|
||||
|
||||
# Post-process output to extract env if persistent
|
||||
output_text = "".join(self.stdout_log)
|
||||
_, parsed_env = self.parse_output(output_text)
|
||||
|
||||
if self.terminal_id and parsed_env:
|
||||
_PERSISTENT_ENVIRONMENTS[self.terminal_id] = parsed_env
|
||||
logger.info(
|
||||
"Updated persistent env for Android terminal"
|
||||
f" '{self.terminal_id}' from background task."
|
||||
)
|
||||
|
||||
# Update status in DataEngine
|
||||
if ctx and ctx.data_engine:
|
||||
ctx.data_engine.unregister_background_task(
|
||||
task_id=self.task_id,
|
||||
status=self.status,
|
||||
)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Error in background task {self.task_id} listener: {e}")
|
||||
self.status = "error"
|
||||
if ctx and ctx.data_engine:
|
||||
ctx.data_engine.unregister_background_task(self.task_id, "failed")
|
||||
finally:
|
||||
# Cache logs in memory before removing task from active tasks
|
||||
full_output = "".join(self.stdout_log)
|
||||
_register_finished_task(
|
||||
task_id=self.task_id,
|
||||
command=self.command,
|
||||
cwd=self.cwd,
|
||||
terminal_id=self.terminal_id,
|
||||
status=self.status,
|
||||
exit_code=self.exit_code,
|
||||
output=full_output,
|
||||
)
|
||||
if self.task_id in _BACKGROUND_TASKS:
|
||||
del _BACKGROUND_TASKS[self.task_id]
|
||||
|
||||
@staticmethod
|
||||
def parse_output(output: str) -> tuple[str, dict[str, str] | None]:
|
||||
"""Separates stdout lines from exported environment variables."""
|
||||
clean_lines = []
|
||||
env_lines = []
|
||||
in_env = False
|
||||
|
||||
for line in output.splitlines():
|
||||
if "===EXIT_CODE===" in line:
|
||||
continue
|
||||
if "===ENV_START===" in line:
|
||||
in_env = True
|
||||
continue
|
||||
if in_env:
|
||||
env_lines.append(line)
|
||||
else:
|
||||
clean_lines.append(line)
|
||||
|
||||
clean_output = "\n".join(clean_lines)
|
||||
|
||||
parsed_env = None
|
||||
if env_lines:
|
||||
parsed_env = {}
|
||||
for el in env_lines:
|
||||
if "=" in el:
|
||||
k, v = el.split("=", 1)
|
||||
parsed_env[k] = v
|
||||
|
||||
return clean_output, parsed_env
|
||||
|
||||
# Backwards compatibility alias
|
||||
_parse_output = parse_output
|
||||
|
||||
|
||||
class RunAdbCommandArgs(BaseModel):
|
||||
"""Arguments schema for executing ADB shell commands."""
|
||||
|
||||
CommandLine: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"The exact shell command line string to execute inside the Android device shell."
|
||||
),
|
||||
)
|
||||
Cwd: str = Field(
|
||||
default="/data/local/tmp",
|
||||
description="The directory on the Android device to run the command in.",
|
||||
)
|
||||
RunPersistent: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"Set to true to run this command in a persistent terminal session"
|
||||
" that preserves environment variables between invocations on the"
|
||||
" phone."
|
||||
),
|
||||
)
|
||||
RequestedTerminalID: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Optional ID of a persistent terminal to reuse. Reuses environment"
|
||||
" variables if provided."
|
||||
),
|
||||
)
|
||||
WaitMsBeforeAsync: int = Field(
|
||||
default=5000,
|
||||
description=(
|
||||
"Number of milliseconds to wait for the command to finish"
|
||||
" synchronously before sending it to the background."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class RunAdbCommandTool(ArtemisTool):
|
||||
"""Universal tool for executing ADB shell commands on an Android device."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="run_adb_command",
|
||||
description=(
|
||||
"[SHELL] Executes a shell command directly on the Android mobile "
|
||||
"device via ADB shell. Can run synchronously or transition to a "
|
||||
"background task if execution takes longer than WaitMsBeforeAsync. "
|
||||
"Supports persistent environments (environment variables) on the "
|
||||
"phone across invocations."
|
||||
),
|
||||
args_schema=RunAdbCommandArgs,
|
||||
category="system",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements,too-many-return-statements,too-many-positional-arguments,too-many-boolean-expressions
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
CommandLine: str | None = None,
|
||||
Cwd: str = "/data/local/tmp",
|
||||
RunPersistent: bool = False,
|
||||
RequestedTerminalID: str | None = None,
|
||||
WaitMsBeforeAsync: int = 5000,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
# Extract command line supporting alternative key formats
|
||||
cmd_line = (
|
||||
CommandLine
|
||||
if CommandLine is not None
|
||||
else (kwargs.get("command_line") or kwargs.get("command") or kwargs.get("cmd") or "")
|
||||
)
|
||||
cwd = Cwd or kwargs.get("cwd") or "/data/local/tmp"
|
||||
run_persistent = (
|
||||
RunPersistent if RunPersistent is not False else kwargs.get("run_persistent", False)
|
||||
)
|
||||
requested_terminal_id = (
|
||||
RequestedTerminalID
|
||||
if RequestedTerminalID is not None
|
||||
else kwargs.get("requested_terminal_id")
|
||||
)
|
||||
wait_ms = (
|
||||
WaitMsBeforeAsync
|
||||
if WaitMsBeforeAsync != 5000
|
||||
else kwargs.get("wait_ms_before_async", WaitMsBeforeAsync)
|
||||
)
|
||||
|
||||
# Fallback to driver's execute_shell if no full ArtemisContext is available
|
||||
if (
|
||||
(ctx is None or not hasattr(ctx, "device") or not ctx.device)
|
||||
and driver is not None
|
||||
and hasattr(driver, "execute_shell")
|
||||
and not run_persistent
|
||||
):
|
||||
return await driver.execute_shell(cmd_line)
|
||||
|
||||
device_id = None
|
||||
if ctx and hasattr(ctx, "device") and ctx.device:
|
||||
device_id = getattr(ctx.device, "device_id", None)
|
||||
elif driver and hasattr(driver, "device_id"):
|
||||
device_id = getattr(driver, "device_id", None)
|
||||
if not device_id:
|
||||
device_id = "default_device"
|
||||
|
||||
# Prepare environment
|
||||
terminal_id = None
|
||||
run_env = os.environ.copy() # Local host env to run adb command
|
||||
android_env_vars = {}
|
||||
|
||||
if run_persistent:
|
||||
terminal_id = requested_terminal_id or f"term_{uuid.uuid4().hex[:8]}"
|
||||
if terminal_id in _PERSISTENT_ENVIRONMENTS:
|
||||
android_env_vars = _PERSISTENT_ENVIRONMENTS[terminal_id]
|
||||
else:
|
||||
_PERSISTENT_ENVIRONMENTS[terminal_id] = android_env_vars
|
||||
|
||||
# Construct command to run inside adb shell
|
||||
# We inject the cached environment variables by prepending 'export K=V' declarations
|
||||
env_prepends = ""
|
||||
for k, v in android_env_vars.items():
|
||||
env_prepends += f"export {k}='{v}'\n"
|
||||
|
||||
# Construct final command script
|
||||
phone_script = f"cd '{cwd}'\n" + env_prepends + f"{cmd_line}\n"
|
||||
if run_persistent:
|
||||
phone_script += (
|
||||
'_exit_code=$?\necho "===EXIT_CODE===$_exit_code"\necho "===ENV_START==="\nenv'
|
||||
)
|
||||
|
||||
# In Cloud Mode or when adb client is virtualized
|
||||
adb_client = getattr(ctx, "adb_client", None) if ctx else None
|
||||
if os.environ.get("ARTEMIS_CLOUD_MODE") == "1" or (
|
||||
adb_client is not None and hasattr(adb_client, "_bridge")
|
||||
):
|
||||
try:
|
||||
device = get_adb_device(ctx)
|
||||
output = device.shell(phone_script)
|
||||
clean_output, new_env = BackgroundTask.parse_output(output)
|
||||
if run_persistent and new_env:
|
||||
android_env_vars.update(new_env)
|
||||
return (
|
||||
clean_output
|
||||
if clean_output.strip()
|
||||
else "Command executed successfully with empty output."
|
||||
)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to execute virtual adb shell for '{cmd_line}': {e}")
|
||||
return f"Failed to execute adb command: {e}"
|
||||
|
||||
# We spawn the host subprocess: adb -s <serial> shell
|
||||
# Pass phone_script as an argument so it doesn't hold the shell open if we keep stdin open.
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
"adb",
|
||||
"-s",
|
||||
device_id,
|
||||
"shell",
|
||||
phone_script,
|
||||
cwd=os.getcwd(),
|
||||
env=run_env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
)
|
||||
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to spawn adb shell process for command '{cmd_line}': {e}")
|
||||
return f"Failed to spawn adb process: {e}"
|
||||
|
||||
# Wait for wait_ms milliseconds
|
||||
wait_seconds = wait_ms / 1000.0
|
||||
try:
|
||||
stdout_data = []
|
||||
|
||||
async def read_limit():
|
||||
while True:
|
||||
line = await process.stdout.readline()
|
||||
if not line:
|
||||
break
|
||||
stdout_data.append(line.decode())
|
||||
await process.wait()
|
||||
|
||||
await asyncio.wait_for(read_limit(), timeout=wait_seconds)
|
||||
|
||||
# If we reached here, the process completed within the wait window
|
||||
exit_code = process.returncode
|
||||
output_text = "".join(stdout_data)
|
||||
|
||||
# Clean output and extract environment
|
||||
clean_output = output_text
|
||||
parsed_env = None
|
||||
if run_persistent:
|
||||
clean_output, parsed_env = BackgroundTask.parse_output(output_text)
|
||||
if parsed_env:
|
||||
_PERSISTENT_ENVIRONMENTS[terminal_id] = parsed_env
|
||||
|
||||
intro = f"ADB command completed with exit code {exit_code}."
|
||||
if run_persistent:
|
||||
intro += f" TerminalID: {terminal_id}."
|
||||
|
||||
if _is_output_long(clean_output):
|
||||
task_id = f"task_sync_{uuid.uuid4().hex[:8]}"
|
||||
intro += f" TaskId: {task_id}."
|
||||
_register_finished_task(
|
||||
task_id=task_id,
|
||||
command=cmd_line,
|
||||
cwd=cwd,
|
||||
terminal_id=terminal_id,
|
||||
status="completed" if exit_code == 0 else "failed",
|
||||
exit_code=exit_code,
|
||||
output=clean_output,
|
||||
)
|
||||
return _format_long_output_response(task_id, clean_output, intro)
|
||||
|
||||
return f"{intro}\nOutput:\n{clean_output}"
|
||||
|
||||
except TimeoutError:
|
||||
# Process did not finish in time, move to background
|
||||
task_id = f"task_{process.pid}"
|
||||
trace_id = uuid.uuid4()
|
||||
|
||||
bg_task = BackgroundTask(
|
||||
task_id=task_id,
|
||||
command=cmd_line,
|
||||
process=process,
|
||||
terminal_id=terminal_id,
|
||||
cwd=cwd,
|
||||
)
|
||||
bg_task.stdout_log = list(stdout_data)
|
||||
bg_task.trace_id = trace_id
|
||||
_BACKGROUND_TASKS[task_id] = bg_task
|
||||
|
||||
# Register in DataEngine
|
||||
if ctx and ctx.data_engine:
|
||||
ctx.data_engine.register_background_task(
|
||||
task_id=task_id,
|
||||
summary=f"ADB Command: {cmd_line[:50]}...",
|
||||
trace_id=trace_id,
|
||||
)
|
||||
if stdout_data:
|
||||
ctx.data_engine.stream_output(trace_id, "".join(stdout_data))
|
||||
|
||||
if ctx:
|
||||
await bg_task.start(ctx)
|
||||
|
||||
response_msg = f"ADB command was sent to the background as a task.\nTaskId: {task_id}\n"
|
||||
if run_persistent:
|
||||
response_msg += f"TerminalID: {terminal_id}\n"
|
||||
response_msg += "Use the 'manage_task' tool to check its status or terminate it."
|
||||
return response_msg
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
run_adb_command = RunAdbCommandTool()
|
||||
RunAdbCommand = RunAdbCommandTool
|
||||
ToolRegistry.register(run_adb_command)
|
||||
|
||||
|
||||
def get_run_adb_command_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports run_adb_command as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(run_adb_command.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
class ManageTaskArgs(BaseModel):
|
||||
"""Arguments schema for managing background ADB tasks."""
|
||||
|
||||
Action: Literal["list", "list_finished", "kill", "status", "send_input"] = Field(
|
||||
...,
|
||||
description=(
|
||||
"The action to perform: 'list' (list running tasks),"
|
||||
" 'list_finished' (list completed tasks), 'kill', 'status',"
|
||||
" 'send_input'."
|
||||
),
|
||||
)
|
||||
TaskId: str | None = Field(
|
||||
default=None,
|
||||
description=("The task ID to manage (required for kill, status, and send_input)."),
|
||||
)
|
||||
Input: str | None = Field(
|
||||
default=None,
|
||||
description=("The input to send to the task (required when Action is 'send_input')."),
|
||||
)
|
||||
|
||||
|
||||
class ManageTaskTool(ArtemisTool):
|
||||
"""Universal tool for managing background ADB shell tasks."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="manage_task",
|
||||
description=("[SHELL] Manage background ADB shell tasks launched via run_adb_command."),
|
||||
args_schema=ManageTaskArgs,
|
||||
category="system",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements,too-many-return-statements,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
Action: Literal["list", "list_finished", "kill", "status", "send_input"] = "list",
|
||||
TaskId: str | None = None,
|
||||
Input: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
action = Action or kwargs.get("action", "list")
|
||||
task_id = TaskId if TaskId is not None else kwargs.get("task_id")
|
||||
input_str = Input if Input is not None else kwargs.get("input")
|
||||
|
||||
if action == "list":
|
||||
active_tasks = []
|
||||
for tid, t in _BACKGROUND_TASKS.items():
|
||||
active_tasks.append(
|
||||
f"- {tid}: Command='{t.command[:40]}...', Phone"
|
||||
f" Cwd='{t.cwd}', TerminalID='{t.terminal_id}'"
|
||||
)
|
||||
if not active_tasks:
|
||||
return "No active background tasks running."
|
||||
return "Active background tasks:\n" + "\n".join(active_tasks)
|
||||
|
||||
if action == "list_finished":
|
||||
finished_tasks = []
|
||||
for tid, t in _FINISHED_TASKS_LOGS.items():
|
||||
finished_tasks.append(
|
||||
f"- {tid}: Command='{t['command'][:40]}...', Status='{t['status']}'"
|
||||
)
|
||||
if not finished_tasks:
|
||||
return "No recently finished tasks."
|
||||
return "Recently finished tasks (up to 10):\n" + "\n".join(finished_tasks)
|
||||
|
||||
if not task_id:
|
||||
return "Error: TaskId is required for status, kill, or send_input actions."
|
||||
|
||||
task = _BACKGROUND_TASKS.get(task_id)
|
||||
|
||||
if action == "status":
|
||||
task_info = _get_task_info(task_id, ctx)
|
||||
if not task_info:
|
||||
return f"Task {task_id} not found."
|
||||
|
||||
output_text = task_info.get("output", "")
|
||||
if "===ENV_START===" in output_text:
|
||||
output_text = output_text.split("===ENV_START===")[0]
|
||||
|
||||
intro = (
|
||||
f"Task: {task_id}\n"
|
||||
f"Status: {task_info.get('status')}\n"
|
||||
f"Command: {task_info.get('command')}\n"
|
||||
f"Cwd: {task_info.get('cwd')}\n"
|
||||
f"TerminalID: {task_info.get('terminal_id')}"
|
||||
)
|
||||
|
||||
if _is_output_long(output_text):
|
||||
return _format_long_output_response(task_id, output_text, intro)
|
||||
|
||||
return f"{intro}\nAccumulated output:\n{output_text}"
|
||||
|
||||
if action == "kill":
|
||||
if not task:
|
||||
return f"Task {task_id} is not active or already finished."
|
||||
try:
|
||||
task.process.terminate()
|
||||
await task.process.wait()
|
||||
task.status = "killed"
|
||||
if ctx and ctx.data_engine:
|
||||
ctx.data_engine.unregister_background_task(task_id, "killed")
|
||||
if task_id in _BACKGROUND_TASKS:
|
||||
del _BACKGROUND_TASKS[task_id]
|
||||
return f"Task {task_id} successfully terminated."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return f"Failed to terminate task {task_id}: {e}"
|
||||
|
||||
if action == "send_input":
|
||||
if not task:
|
||||
return f"Task {task_id} is not active."
|
||||
if not input_str:
|
||||
return "Error: Input is required for send_input action."
|
||||
try:
|
||||
task.process.stdin.write(input_str.encode())
|
||||
await task.process.stdin.drain()
|
||||
return f"Input successfully sent to task {task_id}."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return f"Failed to send input: {e}"
|
||||
|
||||
return "Unsupported action."
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
manage_task = ManageTaskTool()
|
||||
ManageTask = ManageTaskTool
|
||||
ToolRegistry.register(manage_task)
|
||||
|
||||
|
||||
def get_manage_task_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports manage_task as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(manage_task.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
class RunShortAdbCommandArgs(BaseModel):
|
||||
"""Arguments schema for short ADB shell commands."""
|
||||
|
||||
CommandLine: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"The exact shell command line string to execute inside the Android device shell."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class RunShortAdbCommandTool(ArtemisTool):
|
||||
"""Universal tool for executing short, synchronous ADB shell commands."""
|
||||
|
||||
def __init__(self, name: str = "run_short_adb_command"):
|
||||
super().__init__(
|
||||
name=name,
|
||||
description=(
|
||||
"[SHELL] Executes a short, synchronous shell command directly on the Android "
|
||||
"mobile device via ADB shell. Returns the command output. Times out after 3 "
|
||||
"seconds. Do not use this tool to execute actions that change the phone state "
|
||||
"such as clicking, swiping, or navigating back."
|
||||
),
|
||||
args_schema=RunShortAdbCommandArgs,
|
||||
category="system",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-locals,too-many-return-statements
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
CommandLine: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
cmd_line = (
|
||||
CommandLine
|
||||
if CommandLine is not None
|
||||
else (kwargs.get("command_line") or kwargs.get("command") or kwargs.get("cmd") or "")
|
||||
)
|
||||
|
||||
# Fallback to driver's execute_shell if no full ArtemisContext is available
|
||||
if (
|
||||
(ctx is None or not hasattr(ctx, "device") or not ctx.device)
|
||||
and driver is not None
|
||||
and hasattr(driver, "execute_shell")
|
||||
):
|
||||
return await driver.execute_shell(cmd_line, timeout_seconds=3.0)
|
||||
|
||||
device_id = None
|
||||
if ctx and hasattr(ctx, "device") and ctx.device:
|
||||
device_id = getattr(ctx.device, "device_id", None)
|
||||
elif driver and hasattr(driver, "device_id"):
|
||||
device_id = getattr(driver, "device_id", None)
|
||||
if not device_id:
|
||||
device_id = "default_device"
|
||||
|
||||
run_env = os.environ.copy()
|
||||
phone_script = f"cd /data/local/tmp\n{cmd_line}\n"
|
||||
|
||||
adb_client = getattr(ctx, "adb_client", None) if ctx else None
|
||||
if os.environ.get("ARTEMIS_CLOUD_MODE") == "1" or (
|
||||
adb_client is not None and hasattr(adb_client, "_bridge")
|
||||
):
|
||||
try:
|
||||
device = get_adb_device(ctx)
|
||||
output = device.shell(phone_script)
|
||||
return f"ADB command completed.\nOutput:\n{output}"
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return f"Error running command: {e}"
|
||||
|
||||
try:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
"adb",
|
||||
"-s",
|
||||
device_id,
|
||||
"shell",
|
||||
phone_script,
|
||||
cwd=os.getcwd(),
|
||||
env=run_env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to spawn adb shell process for command '{cmd_line}': {e}")
|
||||
return f"Failed to spawn adb process: {e}"
|
||||
|
||||
try:
|
||||
stdout_data, _ = await asyncio.wait_for(process.communicate(), timeout=3.0)
|
||||
exit_code = process.returncode
|
||||
output_text = stdout_data.decode()
|
||||
|
||||
return f"ADB command completed with exit code {exit_code}.\nOutput:\n{output_text}"
|
||||
except TimeoutError:
|
||||
try:
|
||||
process.terminate()
|
||||
await process.wait()
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
pass
|
||||
return "Error: Command timed out after 3 seconds."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return f"Error running command: {e}"
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
run_short_adb_command = RunShortAdbCommandTool()
|
||||
RunShortAdbCommand = RunShortAdbCommandTool
|
||||
ToolRegistry.register(run_short_adb_command)
|
||||
|
||||
|
||||
def get_run_short_adb_command_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports run_short_adb_command as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(
|
||||
run_short_adb_command.to_langchain_tool(ctx, name="run_adb_command"),
|
||||
ctx,
|
||||
)
|
||||
|
||||
|
||||
# Expose Wrappers
|
||||
run_adb_command_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_run_adb_command_tool,
|
||||
on_success_fn=lambda output: output,
|
||||
on_failure_fn=lambda err: f"ADB command failed: {err}",
|
||||
)
|
||||
|
||||
manage_task_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_manage_task_tool,
|
||||
on_success_fn=lambda output: output,
|
||||
on_failure_fn=lambda err: f"Task management failed: {err}",
|
||||
)
|
||||
|
||||
|
||||
class AnalyzeTaskOutputArgs(BaseModel):
|
||||
"""Arguments schema for output analysis requests."""
|
||||
|
||||
TaskId: str = Field(
|
||||
...,
|
||||
description="The task ID whose output you want to analyze.",
|
||||
)
|
||||
Query: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"The question or query to ask the output analyzer about the task's full output log."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class AnalyzeTaskOutputTool(ArtemisTool):
|
||||
"""Universal tool for analyzing the full, long output of an ADB task."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="analyze_task_output",
|
||||
description=(
|
||||
"[ANALYSIS] Analyzes the full, long output of a completed or running ADB task. "
|
||||
"Use this tool when you receive a message indicating that a task's output "
|
||||
"is too long and was truncated."
|
||||
),
|
||||
args_schema=AnalyzeTaskOutputArgs,
|
||||
category="system",
|
||||
)
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
TaskId: str | None = None,
|
||||
Query: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
task_id = TaskId or kwargs.get("task_id") or ""
|
||||
query = Query or kwargs.get("query") or ""
|
||||
|
||||
task_info = _get_task_info(task_id, ctx)
|
||||
if not task_info:
|
||||
return f"Error: Task {task_id} not found."
|
||||
|
||||
analyzer = TaskOutputAnalyzerNode(ctx)
|
||||
result = await analyzer.run(
|
||||
command=task_info.get("command", ""),
|
||||
output_text=task_info.get("output", ""),
|
||||
query=query,
|
||||
)
|
||||
return f"Analysis result for Task {task_id}:\n{result}"
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
analyze_task_output = AnalyzeTaskOutputTool()
|
||||
AnalyzeTaskOutput = AnalyzeTaskOutputTool
|
||||
ToolRegistry.register(analyze_task_output)
|
||||
|
||||
|
||||
def get_analyze_task_output_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports analyze_task_output as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(analyze_task_output.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
analyze_task_output_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_analyze_task_output_tool,
|
||||
on_success_fn=lambda output: output,
|
||||
on_failure_fn=lambda err: f"Task output analysis failed: {err}",
|
||||
)
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"planner_avatar": "You are the **Planner Avatar** (Meeting Host) in a committee resolving a task plan failure. Your role is to lead the meeting and advocate for a robust and correct high-level plan. You are aware of:\n- **History Analyzer**: The trajectory specialist who can analyze execution history and read notes. They have tools like `get_step_details` and `read_note`.\n- **Diagnoser**: The forensics specialist who can analyze logs and videos to find why things failed. They have tools like `video_analyzer` and `analyze_logs`.\nYour master (Master Planner) has given you a specific directive for this meeting, which is at the top of the blackboard.\n**Your Task**:\n1. In the first round, you must open the meeting, state the problem based on the master's directive, and call on the other experts (Diagnoser or History Analyzer) to provide their analysis.\n2. In subsequent rounds, evaluate their suggestions and push for a concrete, revised plan of subgoals.\n3. In the final round, summarize the discussion and output the final recommended plan revisions and actions.",
|
||||
"history_analyzer_expert": "You are the **History Analyzer** (Trajectory Specialist) in a committee resolving a task failure. Your role is to analyze the execution history of the current session, understand what has been tried, what succeeded, and what failed, and check persistent notes. You are aware of:\n- **Planner Avatar**: The meeting host who needs a concrete plan.\n- **Diagnoser**: The forensics specialist who analyzes logs and videos.\nYou have tools to retrieve full-granularity step details and read notes. Wait for the Planner Avatar to call on you or respond to the current state of the blackboard. Be concise.",
|
||||
"diagnoser_expert": "You are the **Diagnoser** (Forensics Specialist) in a committee resolving a UI automation failure. Your role is to analyze what went wrong in the past, look for error patterns in logs, and understand system state transitions. You are aware of:\n- **Planner Avatar**: The meeting host who needs a concrete plan.\n- **History Analyzer**: The trajectory specialist who analyzes history and notes.\nYou have tools to analyze video and logs. Wait for the Planner Avatar to call on you or respond to the current state of the blackboard. Be concise."
|
||||
}
|
||||
@@ -0,0 +1,427 @@
|
||||
# pylint: disable=too-many-statements
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Committee tool for multi-agent deliberation and failure recovery."""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage
|
||||
from langchain_core.tools import BaseTool
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.agents.history_analyzer.history_analyzer import HistoryAnalyzer
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import (
|
||||
CURRENT_TRACE_ID,
|
||||
TraceSpan,
|
||||
trace_langchain_tool,
|
||||
)
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.services.llm import get_llm, invoke_llm_with_timeout_message
|
||||
from artemis.tools.base import ArtemisTool, ToolCategory, ToolRegistry
|
||||
from artemis.tools.log_tool import get_analyze_logs_tool
|
||||
from artemis.tools.scratchpad import get_list_notes_tool, get_read_note_tool
|
||||
from artemis.tools.tool_wrapper import (
|
||||
ToolWrapper,
|
||||
get_tool_result_content,
|
||||
invoke_tool_with_injection,
|
||||
)
|
||||
from artemis.tools.types import CyFunctionDetector
|
||||
from artemis.tools.video_tool import get_video_analyzer_tool
|
||||
from artemis.utils.logger import get_logger
|
||||
from artemis.utils.notes import get_note_file_path, get_notes_dir
|
||||
from artemis.utils.task_tree import (
|
||||
build_plan_and_history,
|
||||
get_active_subgoal_hashes,
|
||||
get_recent_subgoal_hashes,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class AskCommitteeArgs(BaseModel):
|
||||
"""Arguments schema for invoking the committee tool."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
avatar_directive: str = Field(
|
||||
...,
|
||||
description="Directive to frame and steer the debate for your avatar.",
|
||||
)
|
||||
|
||||
|
||||
ASK_COMMITTEE_DOCSTRING = (
|
||||
"[COMMITTEE] Call this tool to summon a committee of sub-agents to debate"
|
||||
" and resolve complex failures.\n\n"
|
||||
"Your avatar will host the meeting and drive it based on your directive.\n"
|
||||
"The result will be a synthesized recommendation."
|
||||
)
|
||||
|
||||
|
||||
class AskCommitteeTool(ArtemisTool):
|
||||
"""Universal tool for summoning a committee of sub-agents to debate and resolve failures."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "custom"):
|
||||
super().__init__(
|
||||
name="ask_committee",
|
||||
description=ASK_COMMITTEE_DOCSTRING,
|
||||
args_schema=AskCommitteeArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
avatar_directive: str = "",
|
||||
tool_call_id: str | None = None, # pylint: disable=unused-argument
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
directive = avatar_directive or kwargs.get("AvatarDirective") or ""
|
||||
return await _run_committee_logic(
|
||||
ctx=ctx,
|
||||
state=state,
|
||||
avatar_directive=directive,
|
||||
)
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
ask_committee = AskCommitteeTool()
|
||||
AskCommittee = AskCommitteeTool
|
||||
AskCommitteeToolAlias = AskCommitteeTool
|
||||
ToolRegistry.register(ask_committee)
|
||||
|
||||
|
||||
def get_ask_committee_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports ask_committee as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(ask_committee.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
|
||||
async def _execute_committee(
|
||||
ctx: ArtemisContext,
|
||||
state: State | None,
|
||||
avatar_directive: str,
|
||||
) -> str:
|
||||
"""Executes committee debate loop and returns the final conclusion."""
|
||||
with TraceSpan(name="committee", trace_type="agent", ctx=ctx):
|
||||
logger.info(f"Committee invoked with directive: {avatar_directive}")
|
||||
|
||||
# 1. Access or create blackboard file
|
||||
if ctx.data_engine:
|
||||
notes_dir = get_notes_dir(ctx.data_engine.base_dir)
|
||||
else:
|
||||
notes_dir = Path(".") / "notes"
|
||||
notes_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Load latest screenshot as base64 to provide multimodal vision to the committee
|
||||
screenshot_b64 = None
|
||||
screenshot_path = getattr(state, "latest_screenshot", None) if state is not None else None
|
||||
if screenshot_path and Path(screenshot_path).exists():
|
||||
try:
|
||||
with open(screenshot_path, "rb") as f:
|
||||
screenshot_b64 = base64.b64encode(f.read()).decode("utf-8")
|
||||
logger.info(f"Committee successfully loaded screenshot from: {screenshot_path}")
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to read screenshot in committee: {e}")
|
||||
|
||||
trace_id = CURRENT_TRACE_ID.get() or str(uuid4())
|
||||
blackboard_path = notes_dir / f"blackboard_{trace_id}.md"
|
||||
|
||||
# Initialize blackboard with goal and directive
|
||||
initial_goal = (
|
||||
getattr(state, "initial_goal", "Not specified.")
|
||||
if state is not None
|
||||
else "Not specified."
|
||||
)
|
||||
blackboard_content = (
|
||||
"# Committee Discussion Blackboard\n\n"
|
||||
f"## User's Initial Goal\n{initial_goal}\n\n"
|
||||
f"## Master's Directive\n{avatar_directive}\n\n"
|
||||
)
|
||||
blackboard_path.write_text(blackboard_content, encoding="utf-8")
|
||||
|
||||
# 2. Extract history
|
||||
history_summary = "## Previous Context\n"
|
||||
if ctx.data_engine:
|
||||
steps = ctx.data_engine.get_agent_friendly_steps()
|
||||
if steps:
|
||||
active_subgoal_hash = "default"
|
||||
if ctx.data_engine:
|
||||
task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan")
|
||||
else:
|
||||
task_plan_path = notes_dir / "task_plan.md"
|
||||
task_plan = "No task plan yet."
|
||||
keep_hashes = None
|
||||
if task_plan_path.exists():
|
||||
try:
|
||||
task_plan = task_plan_path.read_text(encoding="utf-8")
|
||||
|
||||
active_subgoal_hash, _ = get_active_subgoal_hashes(task_plan)
|
||||
keep_hashes = get_recent_subgoal_hashes(
|
||||
steps,
|
||||
active_subgoal_hash,
|
||||
ctx.data_engine.base_dir,
|
||||
)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to parse active subgoal in committee: {e}")
|
||||
|
||||
try:
|
||||
plan_and_history = build_plan_and_history(
|
||||
task_plan,
|
||||
steps,
|
||||
active_subgoal_hash,
|
||||
keep_subgoal_hashes=keep_hashes,
|
||||
last_n_detailed=3,
|
||||
)
|
||||
history_summary += plan_and_history
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to build plan and history in committee: {e}")
|
||||
history_summary += f"Error building plan and history: {e}"
|
||||
|
||||
# Append immediate failure reason if present in state
|
||||
replan_reason = (
|
||||
getattr(state, "operator_replan_reason", None) if state is not None else None
|
||||
)
|
||||
if replan_reason:
|
||||
history_summary += f"\n\n### Current Failure Reason\n{replan_reason}\n"
|
||||
|
||||
# Read failed plans history to ensure the committee avoids
|
||||
# repeating failed strategies
|
||||
failed_plans_history = ""
|
||||
failed_history_path = notes_dir / "failed_plans_history.md"
|
||||
if failed_history_path.exists():
|
||||
try:
|
||||
failed_plans_history = failed_history_path.read_text(encoding="utf-8")
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to read failed plans history in committee: {e}")
|
||||
if failed_plans_history:
|
||||
history_summary += (
|
||||
f"\n\n### Previous Failed Plan Attempts\n{failed_plans_history}\n"
|
||||
)
|
||||
|
||||
with open(blackboard_path, "a", encoding="utf-8") as f:
|
||||
f.write(history_summary + "\n\n")
|
||||
|
||||
# 3. Load prompts from JSON
|
||||
prompts = {}
|
||||
prompts_path = Path(__file__).parent / "committee_prompts.json"
|
||||
if prompts_path.exists():
|
||||
try:
|
||||
prompts = json.loads(prompts_path.read_text(encoding="utf-8"))
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to load committee prompts: {e}")
|
||||
|
||||
planner_prompt = prompts.get("planner_avatar", "You are the Planner Avatar.")
|
||||
history_analyzer_prompt = prompts.get(
|
||||
"history_analyzer_expert", "You are the History Analyzer."
|
||||
)
|
||||
diagnoser_prompt = prompts.get("diagnoser_expert", "You are the Diagnoser Expert.")
|
||||
|
||||
# 4. Prepare Specialized Tools for Members
|
||||
|
||||
# History Analyzer Tools
|
||||
analyzer_inst = HistoryAnalyzer(ctx)
|
||||
history_steps = ctx.data_engine.get_agent_friendly_steps() if ctx.data_engine else []
|
||||
# pylint: disable=protected-access
|
||||
details_tool = analyzer_inst._get_step_details_tool(history_steps)
|
||||
read_note_tool_hist = get_read_note_tool(ctx)
|
||||
list_notes_tool_hist = get_list_notes_tool(ctx)
|
||||
|
||||
history_analyzer_tools = [
|
||||
trace_langchain_tool(details_tool, ctx),
|
||||
trace_langchain_tool(read_note_tool_hist, ctx),
|
||||
trace_langchain_tool(list_notes_tool_hist, ctx),
|
||||
]
|
||||
|
||||
# Diagnoser Tools
|
||||
video_tool = get_video_analyzer_tool(ctx, role="diagnoser")
|
||||
log_tool = get_analyze_logs_tool(ctx)
|
||||
read_note_tool = get_read_note_tool(ctx)
|
||||
list_notes_tool = get_list_notes_tool(ctx)
|
||||
|
||||
diagnoser_tools = [
|
||||
trace_langchain_tool(video_tool, ctx),
|
||||
trace_langchain_tool(log_tool, ctx),
|
||||
trace_langchain_tool(read_note_tool, ctx),
|
||||
trace_langchain_tool(list_notes_tool, ctx),
|
||||
]
|
||||
|
||||
# 5. Instantiate LLMs
|
||||
llm_pl = get_llm(ctx=ctx, name="planner_avatar")
|
||||
|
||||
llm_hist = get_llm(ctx=ctx, name="history_analyzer_expert")
|
||||
llm_hist = llm_hist.bind_tools(tools=history_analyzer_tools)
|
||||
|
||||
llm_diag = get_llm(ctx=ctx, name="diagnoser_expert")
|
||||
llm_diag = llm_diag.bind_tools(tools=diagnoser_tools)
|
||||
|
||||
# 6. Debate Loop (configured rounds)
|
||||
exec_setup = getattr(ctx, "execution_setup", None)
|
||||
rounds = getattr(exec_setup, "committee_debate_rounds", 2) if exec_setup else 2
|
||||
|
||||
async def run_agent_turn(llm, system_prompt, tools, agent_name):
|
||||
current_blackboard = blackboard_path.read_text(encoding="utf-8")
|
||||
|
||||
human_content = [
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"Current Blackboard:\n{current_blackboard}",
|
||||
}
|
||||
]
|
||||
if screenshot_b64:
|
||||
human_content.append(
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"},
|
||||
}
|
||||
)
|
||||
|
||||
messages = [
|
||||
SystemMessage(content=system_prompt),
|
||||
HumanMessage(content=human_content),
|
||||
]
|
||||
|
||||
max_iterations = 3
|
||||
for _ in range(max_iterations):
|
||||
response = await invoke_llm_with_timeout_message(llm.ainvoke(messages))
|
||||
|
||||
if not response.tool_calls:
|
||||
return response.content
|
||||
|
||||
messages.append(response)
|
||||
|
||||
async def run_tool(tc):
|
||||
tool_name = tc["name"]
|
||||
logger.info(f"{agent_name} requested tool: {tool_name}")
|
||||
tool_to_run = next((t for t in tools if t.name == tool_name), None)
|
||||
if tool_to_run:
|
||||
try:
|
||||
args = dict(tc["args"])
|
||||
result = await invoke_tool_with_injection(
|
||||
tool=tool_to_run,
|
||||
args=args,
|
||||
tool_call_id=tc["id"],
|
||||
state=state,
|
||||
)
|
||||
return get_tool_result_content(result)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return f"Error: {e}"
|
||||
else:
|
||||
return f"Error: Tool {tool_name} not found"
|
||||
|
||||
active_tool_calls = [
|
||||
tc for tc in response.tool_calls if tc["name"] != "google_search"
|
||||
]
|
||||
tool_outputs = await asyncio.gather(*(run_tool(tc) for tc in active_tool_calls))
|
||||
|
||||
for tc, result in zip(active_tool_calls, tool_outputs):
|
||||
messages.append(
|
||||
ToolMessage(
|
||||
tool_call_id=tc["id"],
|
||||
content=result,
|
||||
status="success" if not result.startswith("Error") else "error",
|
||||
)
|
||||
)
|
||||
return "Error: Reached max iterations in turn."
|
||||
|
||||
for r in range(1, rounds + 1):
|
||||
logger.info(f"Committee Round {r}")
|
||||
|
||||
# 1. Planner Avatar speaks first (Host)
|
||||
pl_response = await run_agent_turn(llm_pl, planner_prompt, [], "Planner Avatar")
|
||||
with open(blackboard_path, "a", encoding="utf-8") as f:
|
||||
f.write(f"### Planner Avatar (Round {r})\n{pl_response}\n\n")
|
||||
|
||||
# 2. Diagnoser speaks (Can use tools)
|
||||
diag_response = await run_agent_turn(
|
||||
llm_diag, diagnoser_prompt, diagnoser_tools, "Diagnoser"
|
||||
)
|
||||
with open(blackboard_path, "a", encoding="utf-8") as f:
|
||||
f.write(f"### Diagnoser (Round {r})\n{diag_response}\n\n")
|
||||
|
||||
# 3. History Analyzer speaks (Can use tools)
|
||||
hist_response = await run_agent_turn(
|
||||
llm_hist,
|
||||
history_analyzer_prompt,
|
||||
history_analyzer_tools,
|
||||
"History Analyzer",
|
||||
)
|
||||
with open(blackboard_path, "a", encoding="utf-8") as f:
|
||||
f.write(f"### History Analyzer (Round {r})\n{hist_response}\n\n")
|
||||
|
||||
# Final Synthesis by Planner Avatar
|
||||
current_blackboard = blackboard_path.read_text(encoding="utf-8")
|
||||
|
||||
final_human_content = [
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"Current Blackboard:\n{current_blackboard}",
|
||||
}
|
||||
]
|
||||
if screenshot_b64:
|
||||
final_human_content.append(
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"},
|
||||
}
|
||||
)
|
||||
|
||||
msg_final = [
|
||||
SystemMessage(
|
||||
content=(
|
||||
"You are the Planner Avatar. Summarize the discussion"
|
||||
" and output the final recommended plan revisions and"
|
||||
" action plan based on your master's directive."
|
||||
)
|
||||
),
|
||||
HumanMessage(content=final_human_content),
|
||||
]
|
||||
resp_final = await invoke_llm_with_timeout_message(llm_pl.ainvoke(msg_final))
|
||||
|
||||
with open(blackboard_path, "a", encoding="utf-8") as f:
|
||||
f.write(f"## Final Conclusion\n{resp_final.content}\n")
|
||||
|
||||
return resp_final.content
|
||||
|
||||
|
||||
async def _run_committee_logic(
|
||||
ctx: ArtemisContext | None,
|
||||
state: State | None,
|
||||
avatar_directive: str,
|
||||
) -> str:
|
||||
"""Executes multi-agent committee debate logic."""
|
||||
if ctx is None:
|
||||
return "Error: ArtemisContext is required for ask_committee."
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(_execute_committee(ctx, state, avatar_directive), timeout=300)
|
||||
except TimeoutError:
|
||||
logger.error("Committee timed out after 300 seconds.")
|
||||
return "Error: Committee timed out after 300 seconds."
|
||||
|
||||
|
||||
ask_committee_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_ask_committee_tool,
|
||||
on_success_fn=lambda output: f"Committee concluded:\n{output}",
|
||||
on_failure_fn=lambda error: f"Committee failed: {error}",
|
||||
)
|
||||
@@ -0,0 +1,88 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Universal tool for submitting final diagnosis and actionable recovery advice."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
class SubmitAnswerArgs(BaseModel):
|
||||
"""Arguments schema for submitting final diagnoser answer."""
|
||||
|
||||
analysis: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"A detailed and concise explanation of the root cause of the"
|
||||
" failure or query. Answer in one full paragraph."
|
||||
),
|
||||
)
|
||||
actionable_steps: list[str] = Field(
|
||||
default_factory=list,
|
||||
description=(
|
||||
"A list of concrete recovery instructions or next steps. Each"
|
||||
" step is a short sentence. If no actionable steps are"
|
||||
" available, return an empty list."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class SubmitAnswerTool(ArtemisTool):
|
||||
"""Universal tool for submitting final diagnosis and actionable recovery advice."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="submit_answer",
|
||||
description=(
|
||||
"[Finalization] Submits the final diagnosis and actionable recovery advice. "
|
||||
"Call this tool when you have pinpointed the root cause."
|
||||
),
|
||||
args_schema=SubmitAnswerArgs,
|
||||
category="custom",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None, # pylint: disable=unused-argument
|
||||
analysis: str | None = None, # pylint: disable=unused-argument
|
||||
actionable_steps: list[str] | None = None, # pylint: disable=unused-argument
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
logger.info("Diagnoser submitted answer via tool.")
|
||||
return "Answer submitted successfully."
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
submit_answer = SubmitAnswerTool()
|
||||
SubmitAnswer = SubmitAnswerTool
|
||||
DiagnoserSubmitAnswerTool = SubmitAnswerTool
|
||||
ToolRegistry.register(submit_answer)
|
||||
|
||||
|
||||
def get_submit_answer_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports submit_answer as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(submit_answer.to_langchain_tool(ctx), ctx)
|
||||
@@ -0,0 +1,120 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Universal diagnostic agent tool for deep system logs and video inspection."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.agents.diagnoser.diagnoser import Diagnoser
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.tool_wrapper import ToolWrapper
|
||||
from artemis.tools.types import CyFunctionDetector
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
class AskDiagnoserArgs(BaseModel):
|
||||
"""Arguments schema for invoking the diagnostic subagent."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
query: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"The diagnostic query or what you want to investigate. Please"
|
||||
" provide a specific, narrow time range if applicable."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _run_diagnoser_logic(
|
||||
ctx: ArtemisContext | None,
|
||||
state: State | None,
|
||||
query: str,
|
||||
) -> str:
|
||||
logger.info(f"ask_diagnoser tool called with query: {query}")
|
||||
if ctx is None:
|
||||
return "Error: ArtemisContext is required for Diagnoser."
|
||||
agent = Diagnoser(ctx)
|
||||
try:
|
||||
result = await agent.run(query, state)
|
||||
return f"Diagnoser replied:\n{result}"
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to run diagnoser: {e}")
|
||||
return f"Diagnoser failed: {e}"
|
||||
|
||||
|
||||
class AskDiagnoserTool(ArtemisTool):
|
||||
"""Universal tool for invoking the Diagnoser subagent to inspect logs and videos."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="ask_diagnoser",
|
||||
description=(
|
||||
"[DIAGNOSTIC] Analyzes system logs (ADB) and video recordings to "
|
||||
"pinpoint the root cause of any step failure, screen freeze, or "
|
||||
"unexpected UI drift, and directly executes adb commands. Call this "
|
||||
"tool proactively whenever an action fails, status stagnates, or you "
|
||||
"are unsure why progress is blocked, to diagnose the issue before "
|
||||
"committing to physical actions. Note: When querying video files, "
|
||||
"specify a narrow timeframe (e.g., '[10s, 20s]') for faster analysis."
|
||||
),
|
||||
args_schema=AskDiagnoserArgs,
|
||||
category="custom",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
query: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
q = query or kwargs.get("Query") or ""
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
return await _run_diagnoser_logic(ctx, st, q)
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
ask_diagnoser = AskDiagnoserTool()
|
||||
AskDiagnoser = AskDiagnoserTool
|
||||
DiagnosticTool = AskDiagnoserTool
|
||||
ToolRegistry.register(ask_diagnoser)
|
||||
|
||||
|
||||
def get_ask_diagnostic_agent_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports ask_diagnoser as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(ask_diagnoser.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
def get_ask_diagnoser_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports ask_diagnoser as a LangChain BaseTool (standard naming alias)."""
|
||||
return trace_langchain_tool(ask_diagnoser.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
ask_diagnoser_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_ask_diagnostic_agent_tool,
|
||||
on_success_fn=lambda output: f"Diagnoser replied:\n{output}",
|
||||
on_failure_fn=lambda error: f"Diagnoser failed: {error}",
|
||||
)
|
||||
@@ -0,0 +1,404 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import base64
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
from typing import Any, Literal
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
from pydantic import BaseModel, Field, create_model
|
||||
|
||||
from artemis.agents.explorer.constants import EXPLORE_DESCRIPTIONS
|
||||
from artemis.agents.explorer.explorer import Explorer
|
||||
from artemis.agents.object_detector.object_detector import _run_object_detection
|
||||
from artemis.config import ExplorerVersion, resolve_explorer_version, settings
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import trace, trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolCategory, ToolRegistry
|
||||
from artemis.tools.tool_wrapper import ToolWrapper
|
||||
from artemis.tools.types import CyFunctionDetector
|
||||
from artemis.utils.logger import get_logger
|
||||
from artemis.utils.visualization import draw_dots
|
||||
|
||||
|
||||
class AskExplorerArgs(BaseModel):
|
||||
"""Arguments schema for invoking the explorer subagent."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
query: str = Field(
|
||||
...,
|
||||
description=("The target element or information to search for, including descriptions."),
|
||||
)
|
||||
context_feedback: str = Field(
|
||||
"",
|
||||
description=("Feedback from previous failed attempts or specific instructions"),
|
||||
)
|
||||
version: Literal["flash", "pro", "ultra"] | None = Field(
|
||||
default=None,
|
||||
description="Explicit explorer version to use (flash, pro, or ultra).",
|
||||
)
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class AskExplorerTool(ArtemisTool):
|
||||
"""Universal tool for visually parsing and locating UI elements."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
version: ExplorerVersion | None = None,
|
||||
agent_name: str = "operator",
|
||||
description: str | None = None,
|
||||
category: ToolCategory = "explorer",
|
||||
):
|
||||
self.version = version
|
||||
self.agent_name = agent_name
|
||||
resolved_v = version or "pro"
|
||||
explore_info = EXPLORE_DESCRIPTIONS.get(resolved_v, EXPLORE_DESCRIPTIONS["pro"])
|
||||
desc = description or explore_info["description"]
|
||||
if "{max_iterations}" in desc:
|
||||
max_iterations = 8 if resolved_v == "ultra" else 3
|
||||
desc = desc.format(max_iterations=max_iterations)
|
||||
|
||||
super().__init__(
|
||||
name="ask_explorer",
|
||||
description=desc,
|
||||
args_schema=AskExplorerArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
query: str = "",
|
||||
context_feedback: str = "",
|
||||
version: Literal["flash", "pro", "ultra"] | None = None,
|
||||
tool_call_id: str | None = None, # pylint: disable=unused-argument
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
q = query or kwargs.get("Query") or ""
|
||||
cf = context_feedback or kwargs.get("ContextFeedback") or ""
|
||||
v = version or self.version
|
||||
resolved_v = (
|
||||
resolve_explorer_version(ctx, explicit_version=v, agent_or_profile_name=self.agent_name)
|
||||
if ctx
|
||||
else (v or "pro")
|
||||
)
|
||||
return await _run_explorer_logic(
|
||||
ctx=ctx,
|
||||
state=state,
|
||||
query=q,
|
||||
context_feedback=cf,
|
||||
version=resolved_v,
|
||||
)
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
ask_explorer = AskExplorerTool()
|
||||
AskExplorer = AskExplorerTool
|
||||
AskExplorerToolAlias = AskExplorerTool
|
||||
ToolRegistry.register(ask_explorer)
|
||||
|
||||
|
||||
def get_ask_explorer_tool(
|
||||
ctx: ArtemisContext,
|
||||
version: ExplorerVersion | None = None,
|
||||
agent_name: str = "operator",
|
||||
) -> BaseTool:
|
||||
"""Exports ask_explorer as a LangChain BaseTool with dynamic schema."""
|
||||
resolved_version = resolve_explorer_version(
|
||||
ctx, explicit_version=version, agent_or_profile_name=agent_name
|
||||
)
|
||||
|
||||
explore_info = EXPLORE_DESCRIPTIONS.get(resolved_version, EXPLORE_DESCRIPTIONS["pro"])
|
||||
description = explore_info["description"]
|
||||
if "{max_iterations}" in description:
|
||||
max_iterations = 8 if resolved_version == "ultra" else 3
|
||||
description = description.format(max_iterations=max_iterations)
|
||||
query_description = explore_info["query_description"]
|
||||
|
||||
# Dynamically build argument schema so Operator does not see
|
||||
# context_feedback when flash is active
|
||||
fields = {
|
||||
"query": (str, Field(..., description=query_description)),
|
||||
}
|
||||
if resolved_version != "flash":
|
||||
fields["context_feedback"] = (
|
||||
str,
|
||||
Field(
|
||||
"",
|
||||
description=("Feedback from previous failed attempts or specific instructions"),
|
||||
),
|
||||
)
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
DynamicAskExplorerArgs = create_model("AskExplorerArgs", **fields)
|
||||
|
||||
tool_instance = AskExplorerTool(
|
||||
version=resolved_version,
|
||||
agent_name=agent_name,
|
||||
description=description,
|
||||
)
|
||||
tool_instance.args_schema = DynamicAskExplorerArgs
|
||||
|
||||
return trace_langchain_tool(tool_instance.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
|
||||
async def _run_explorer_logic(
|
||||
ctx: ArtemisContext,
|
||||
state: State,
|
||||
query: str,
|
||||
context_feedback: str,
|
||||
version: Literal["flash", "pro", "ultra"],
|
||||
) -> Any:
|
||||
if version == "flash":
|
||||
logger.info(
|
||||
"Explorer [version='flash'] intercepted -> Direct one-shot object"
|
||||
" detection (context_feedback ignored)."
|
||||
)
|
||||
screenshot_path = state.latest_screenshot
|
||||
if not screenshot_path:
|
||||
return "Explorer (Flash) error: No latest_screenshot available in state."
|
||||
|
||||
queries = [q.strip() for q in query.split("|") if q.strip()]
|
||||
if not queries:
|
||||
queries = [query.strip()]
|
||||
|
||||
try:
|
||||
prompt_path = Path(__file__).parent.parent.joinpath(
|
||||
"agents", "object_detector", "object_detector.json"
|
||||
)
|
||||
with open(prompt_path, encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
templates = [f"{t}\n\n{cfg.get('instructions', '')}" for t in cfg.get("templates", [])]
|
||||
|
||||
result_dict = await _run_object_detection(
|
||||
ctx, screenshot_path, queries, templates, global_timeout=15.0
|
||||
)
|
||||
detected_items = result_dict.get("detected", [])
|
||||
candidates = []
|
||||
for item in detected_items:
|
||||
if isinstance(item, dict) and "point" in item and len(item["point"]) == 2:
|
||||
nx, ny = item["point"]
|
||||
lbl = item.get("label", query)
|
||||
candidates.append(
|
||||
{
|
||||
"label": lbl,
|
||||
"coords": [int(nx), int(ny)],
|
||||
"description": f"{lbl} (flash detection)",
|
||||
}
|
||||
)
|
||||
|
||||
if not candidates:
|
||||
failed_queries = result_dict.get("failed", queries)
|
||||
explorer_output = json.dumps(
|
||||
{
|
||||
"candidates": [],
|
||||
"fallback_message": (
|
||||
"Flash one-shot detection could not locate:"
|
||||
f" {', '.join(failed_queries)}. Consider checking"
|
||||
" visual visibility or layout."
|
||||
),
|
||||
}
|
||||
)
|
||||
else:
|
||||
explorer_output = json.dumps({"candidates": candidates})
|
||||
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Flash one-shot routing failed: {e}")
|
||||
explorer_output = json.dumps(
|
||||
{
|
||||
"candidates": [],
|
||||
"fallback_message": f"Flash detection error: {e}",
|
||||
}
|
||||
)
|
||||
else:
|
||||
|
||||
@trace(type="agent", name="explorer", ctx=ctx)
|
||||
async def run_explorer_agent(q: str, cf: str, screenshot_path: str, state: State) -> str:
|
||||
agent = Explorer(ctx)
|
||||
env_cache = os.getenv("ARTEMIS_EXPLORER_CACHING", "").lower()
|
||||
if env_cache in ["true", "false"]:
|
||||
enable_caching = env_cache == "true"
|
||||
else:
|
||||
agent_cfg = getattr(ctx, "agent_config", None)
|
||||
if (
|
||||
agent_cfg
|
||||
and hasattr(agent_cfg, "explorer")
|
||||
and hasattr(agent_cfg.explorer, "caching")
|
||||
):
|
||||
enable_caching = agent_cfg.explorer.caching
|
||||
elif (
|
||||
ctx and ctx.execution_setup and hasattr(ctx.execution_setup, "explorer_caching")
|
||||
):
|
||||
enable_caching = ctx.execution_setup.explorer_caching
|
||||
else:
|
||||
enable_caching = getattr(settings, "EXPLORER_CACHING", True)
|
||||
return await agent.run(
|
||||
q,
|
||||
cf,
|
||||
screenshot_path,
|
||||
state,
|
||||
enable_caching=enable_caching,
|
||||
version=version,
|
||||
)
|
||||
|
||||
screenshot_path = state.latest_screenshot
|
||||
explorer_output = await run_explorer_agent(query, context_feedback, screenshot_path, state)
|
||||
|
||||
try:
|
||||
data = json.loads(explorer_output)
|
||||
candidates = data.get("candidates", [])
|
||||
fallback_message = data.get("fallback_message", "")
|
||||
|
||||
if candidates:
|
||||
# 1. Read Operator's active coordinates list
|
||||
indexed_points = getattr(state, "indexed_points", None)
|
||||
if indexed_points is None:
|
||||
indexed_points = []
|
||||
state.indexed_points = indexed_points
|
||||
|
||||
indexed_elements = getattr(state, "indexed_elements", None)
|
||||
if indexed_elements is None:
|
||||
indexed_elements = []
|
||||
state.indexed_elements = indexed_elements
|
||||
|
||||
width = 1080
|
||||
height = 2400
|
||||
operator_raw_data = getattr(state, "operator_raw_data", {}) or {}
|
||||
w_raw = operator_raw_data.get("width")
|
||||
h_raw = operator_raw_data.get("height")
|
||||
if isinstance(w_raw, int) and isinstance(h_raw, int):
|
||||
width = w_raw
|
||||
height = h_raw
|
||||
else:
|
||||
if ctx.device and getattr(ctx.device, "device_width", None):
|
||||
width = ctx.device.device_width
|
||||
if ctx.device and getattr(ctx.device, "device_height", None):
|
||||
height = ctx.device.device_height
|
||||
|
||||
registered_info = []
|
||||
for cand in candidates:
|
||||
coords = cand.get("coords")
|
||||
desc = cand.get("description", "Explorer candidate")
|
||||
if coords and len(coords) == 2:
|
||||
nx, ny = coords
|
||||
pixel_x = int(max(0, min(width, nx * width / 1000)))
|
||||
pixel_y = int(max(0, min(height, ny * height / 1000)))
|
||||
|
||||
indexed_points.append([pixel_x, pixel_y])
|
||||
new_idx = len(indexed_points)
|
||||
|
||||
# Update indexed_elements in sync
|
||||
indexed_elements.append(
|
||||
{
|
||||
"index": new_idx,
|
||||
"center": [pixel_x, pixel_y],
|
||||
"text": desc,
|
||||
"bounds": None,
|
||||
"class": "ExplorerCandidate",
|
||||
"resource_id": None,
|
||||
"is_ocr": False,
|
||||
}
|
||||
)
|
||||
|
||||
registered_info.append(f"- [{new_idx}] '{desc}' at coordinate {coords}")
|
||||
|
||||
response_msg = (
|
||||
"Explorer successfully located the following candidate(s):\n"
|
||||
+ "\n".join(registered_info)
|
||||
)
|
||||
response_msg += (
|
||||
"\nYou can click/act on them directly by calling perform_action"
|
||||
" with their respective index."
|
||||
)
|
||||
|
||||
if fallback_message:
|
||||
response_msg += f"\n\nAdditional Notes from Explorer: {fallback_message}"
|
||||
|
||||
# 2. Generate annotated image with ALL points
|
||||
|
||||
base_dir = (
|
||||
Path(ctx.data_engine.base_dir)
|
||||
if ctx.data_engine and getattr(ctx.data_engine, "base_dir", None)
|
||||
else settings.TRACES_PATH
|
||||
)
|
||||
images_dir = base_dir / "images"
|
||||
explorer_dir = images_dir / "explorer_tool"
|
||||
explorer_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
image_name = "explorer_output"
|
||||
existing_files = glob.glob(str(explorer_dir / f"{image_name}_*.jpg"))
|
||||
max_seq = 0
|
||||
for f in existing_files:
|
||||
match = re.search(r"_(\d+)\.jpg$", f)
|
||||
if match:
|
||||
max_seq = max(max_seq, int(match.group(1)))
|
||||
seq = max_seq + 1
|
||||
output_path = explorer_dir / f"{image_name}_{seq}.jpg"
|
||||
|
||||
all_labels = [str(i) for i in range(1, len(indexed_points) + 1)]
|
||||
|
||||
try:
|
||||
draw_dots(
|
||||
screenshot_path,
|
||||
indexed_points,
|
||||
all_labels,
|
||||
str(output_path),
|
||||
)
|
||||
with open(output_path, "rb") as im_f:
|
||||
img_bytes = im_f.read()
|
||||
img_b64 = base64.b64encode(img_bytes).decode("utf-8")
|
||||
|
||||
return [
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"Explorer replied:\n{response_msg}",
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/jpeg;base64,{img_b64}"},
|
||||
},
|
||||
]
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to draw dots or read image in explorer_tool: {e}")
|
||||
# Fallback to pure text if drawing fails
|
||||
return f"Explorer replied:\n{response_msg}"
|
||||
|
||||
elif fallback_message:
|
||||
return f"Explorer could not locate the element. Message: {fallback_message}"
|
||||
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to parse explorer output: {e}")
|
||||
return f"Explorer failed to provide a valid format. Raw output: {explorer_output}"
|
||||
|
||||
return f"Explorer replied:\n{explorer_output}"
|
||||
|
||||
|
||||
ask_explorer_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_ask_explorer_tool,
|
||||
on_success_fn=lambda output: f"Explorer replied:\n{output}",
|
||||
on_failure_fn=lambda error: f"Explorer failed: {error}",
|
||||
)
|
||||
@@ -0,0 +1,125 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Universal Image Processor tool for executing image transformations."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import trace, trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.tool_wrapper import ToolWrapper
|
||||
from artemis.tools.types import CyFunctionDetector
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
class AskVisionCoderArgs(BaseModel):
|
||||
"""Arguments schema for invoking the Image Processor subagent."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
instruction: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"Step-by-step instructions on what image processing or"
|
||||
" modifications should be performed."
|
||||
),
|
||||
)
|
||||
target_image_id: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"The ID of the image in the Image Pool to start from (e.g. 'img_0', 'img_1')."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# Alias for naming consistency
|
||||
AskImageProcessorArgs = AskVisionCoderArgs
|
||||
|
||||
|
||||
class AskImageProcessorTool(ArtemisTool):
|
||||
"""Universal tool for executing image transformations via ImageProcessor subagent."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="ask_image_processor",
|
||||
description=(
|
||||
"Call this tool to ask the image processor agent to run"
|
||||
" complex image transformations or modifications."
|
||||
),
|
||||
args_schema=AskVisionCoderArgs,
|
||||
category="custom",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
instruction: str | None = None,
|
||||
target_image_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
inst = instruction or kwargs.get("Instruction") or ""
|
||||
tid = target_image_id or kwargs.get("TargetImageId") or kwargs.get("target_image_id") or ""
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
return await _run_image_processor_logic(ctx, st, inst, tid)
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
ask_image_processor = AskImageProcessorTool()
|
||||
AskImageProcessor = AskImageProcessorTool
|
||||
AskVisionCoderTool = AskImageProcessorTool
|
||||
ImageProcessorTool = AskImageProcessorTool
|
||||
ToolRegistry.register(ask_image_processor)
|
||||
|
||||
|
||||
def get_ask_image_processor_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports ask_image_processor as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(ask_image_processor.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
async def _run_image_processor_logic(
|
||||
ctx: ArtemisContext | None,
|
||||
state: State | None,
|
||||
instruction: str,
|
||||
target_image_id: str, # pylint: disable=unused-argument
|
||||
) -> Any:
|
||||
@trace(type="agent", name="image_processor", ctx=ctx)
|
||||
async def run_image_processor_agent(inst: str, img_path: str) -> dict:
|
||||
from artemis.agents.image_processor.image_processor import ( # pylint: disable=import-outside-toplevel
|
||||
ImageProcessor,
|
||||
)
|
||||
|
||||
agent = ImageProcessor(ctx)
|
||||
return await agent.run(inst, img_path)
|
||||
|
||||
screenshot_path = state.latest_screenshot if state else None
|
||||
result = await run_image_processor_agent(instruction, screenshot_path)
|
||||
return result
|
||||
|
||||
|
||||
ask_image_processor_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_ask_image_processor_tool,
|
||||
on_success_fn=lambda output: f"Image Processor replied:\n{output}",
|
||||
on_failure_fn=lambda error: f"Image Processor failed: {error}",
|
||||
)
|
||||
@@ -0,0 +1,52 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.tools.tool_wrapper import CompositeToolWrapper, ToolWrapper
|
||||
|
||||
|
||||
def get_tools_from_wrappers(
|
||||
ctx: "ArtemisContext",
|
||||
wrappers: list[ToolWrapper],
|
||||
) -> list[BaseTool]:
|
||||
"""Instantiate and wrap LangChain tools from a list of ToolWrappers."""
|
||||
tools: list[BaseTool] = []
|
||||
for wrapper in wrappers:
|
||||
if isinstance(wrapper, CompositeToolWrapper):
|
||||
comp_tools = wrapper.composite_tools_fn_getter(ctx)
|
||||
for t in comp_tools:
|
||||
tools.append(trace_langchain_tool(t, ctx))
|
||||
continue
|
||||
|
||||
t = wrapper.tool_fn_getter(ctx)
|
||||
tools.append(trace_langchain_tool(t, ctx))
|
||||
return tools
|
||||
|
||||
|
||||
def get_tool_by_name(name: str, tools: list[BaseTool]) -> BaseTool | None:
|
||||
"""Get a tool by name, stripping any prefixes like 'default_api:'."""
|
||||
normalized_name = name
|
||||
if ":" in normalized_name:
|
||||
normalized_name = normalized_name.split(":")[-1]
|
||||
|
||||
for t in tools:
|
||||
t_name = t.name
|
||||
if ":" in t_name:
|
||||
t_name = t_name.split(":")[-1]
|
||||
if t_name == normalized_name:
|
||||
return t
|
||||
return None
|
||||
@@ -0,0 +1,108 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Universal Log Analyzer tool for inspecting and analyzing Android system logs."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.agents.log_analyzer.log_analyzer import LogAnalyzerNode
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.tool_wrapper import ToolWrapper
|
||||
from artemis.tools.types import CyFunctionDetector
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
class AnalyzeLogsArgs(BaseModel):
|
||||
"""Arguments schema for invoking the Log Analyzer subagent."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
specific_query: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"Instructions for the log analyzer, specifying what to search for or analyze."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class AnalyzeLogsTool(ArtemisTool):
|
||||
"""Universal tool for analyzing Android logs using a dedicated hybrid log analyzer agent."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="analyze_logs",
|
||||
description=(
|
||||
"[DIAGNOSTIC] Analyzes Android logs using a dedicated hybrid log analyzer agent. "
|
||||
"Use this when you need to investigate failures, check for specific error "
|
||||
"logs, or understand system events."
|
||||
),
|
||||
args_schema=AnalyzeLogsArgs,
|
||||
category="custom",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
specific_query: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
query = (
|
||||
specific_query
|
||||
if specific_query is not None
|
||||
else (
|
||||
kwargs.get("specific_query")
|
||||
or kwargs.get("SpecificQuery")
|
||||
or kwargs.get("query")
|
||||
or kwargs.get("Query")
|
||||
or ""
|
||||
)
|
||||
)
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
logger.info(f"analyze_logs tool called with query: '{query}'")
|
||||
if ctx is None:
|
||||
return "Error: ArtemisContext is required for LogAnalyzer."
|
||||
analyst = LogAnalyzerNode(ctx)
|
||||
result = await analyst.run(query, st)
|
||||
return result
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
analyze_logs = AnalyzeLogsTool()
|
||||
AnalyzeLogs = AnalyzeLogsTool
|
||||
LogTool = AnalyzeLogsTool
|
||||
ToolRegistry.register(analyze_logs)
|
||||
|
||||
|
||||
def get_analyze_logs_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports analyze_logs as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(analyze_logs.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
analyze_logs_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_analyze_logs_tool,
|
||||
on_success_fn=lambda output: f"Log Analyzer replied:\n{output}",
|
||||
on_failure_fn=lambda error: f"Log Analyzer failed: {error}",
|
||||
)
|
||||
@@ -0,0 +1,66 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Memory and Scratchpad Note Management tools."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.tools.base import artemis_tool
|
||||
|
||||
_IN_MEMORY_NOTES: dict[str, str] = {}
|
||||
|
||||
|
||||
class ReadNoteArgs(BaseModel):
|
||||
"""Arguments schema for reading a note."""
|
||||
|
||||
title: str = Field(..., description="Note title to read.")
|
||||
|
||||
|
||||
class SaveNoteArgs(BaseModel):
|
||||
"""Arguments schema for saving a note."""
|
||||
|
||||
title: str = Field(..., description="Note title to create/save.")
|
||||
content: str = Field(..., description="Content text of the note.")
|
||||
|
||||
|
||||
@artemis_tool(
|
||||
name="read_note",
|
||||
description="[MEMORY] Reads content of an existing scratchpad note.",
|
||||
args_schema=ReadNoteArgs,
|
||||
category="memory",
|
||||
)
|
||||
async def read_note(
|
||||
title: str,
|
||||
ctx: Any = None, # pylint: disable=unused-argument
|
||||
) -> str:
|
||||
"""Reads content of an existing scratchpad note."""
|
||||
return _IN_MEMORY_NOTES.get(title, f"Note '{title}' not found.")
|
||||
|
||||
|
||||
@artemis_tool(
|
||||
name="save_note",
|
||||
description="[MEMORY] Saves content into the scratchpad notes system.",
|
||||
args_schema=SaveNoteArgs,
|
||||
category="memory",
|
||||
)
|
||||
async def save_note(
|
||||
title: str,
|
||||
content: str,
|
||||
ctx: Any = None, # pylint: disable=unused-argument
|
||||
) -> str:
|
||||
"""Saves content into the scratchpad notes system."""
|
||||
_IN_MEMORY_NOTES[title] = content
|
||||
return f"Note '{title}' saved successfully."
|
||||
@@ -0,0 +1,730 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Literal
|
||||
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langchain_core.tools import BaseTool
|
||||
from langgraph.types import Command
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.controllers.unified_controller import UnifiedMobileController
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolCategory, ToolRegistry
|
||||
from artemis.tools.tool_wrapper import ToolWrapper
|
||||
from artemis.tools.types import CyFunctionDetector
|
||||
from artemis.utils.coordinates import parse_swipe_parameters
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class ClickArgs(BaseModel):
|
||||
"""Arguments schema for clicking on normalized coordinates."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
target: list[int] = Field(
|
||||
...,
|
||||
description=("Click target normalized coordinates [x, y] in 0-1000 scale."),
|
||||
)
|
||||
|
||||
|
||||
class LongPressArgs(BaseModel):
|
||||
"""Arguments schema for long pressing on normalized coordinates."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
target: list[int] = Field(
|
||||
...,
|
||||
description=("Long press target normalized coordinates [x, y] in 0-1000 scale."),
|
||||
)
|
||||
duration: int = Field(1000, description="Long press duration in milliseconds (default 1000).")
|
||||
|
||||
|
||||
class InputTextArgs(BaseModel):
|
||||
"""Arguments schema for inputting text into normalized coordinates."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
text: str = Field(
|
||||
...,
|
||||
description="The text content to input. Supports multi-line strings with '\\n'.",
|
||||
)
|
||||
target: list[int] = Field(
|
||||
...,
|
||||
description=("Input target field normalized coordinates [x, y] in 0-1000 scale."),
|
||||
)
|
||||
clear_exist: bool = Field(
|
||||
True,
|
||||
description=(
|
||||
"Whether to clear existing text before typing (True to replace all"
|
||||
" text, False to append at the end of existing content)."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class SwipeArgs(BaseModel):
|
||||
"""Arguments schema for performing swipe gestures."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
direction: Literal["up", "down", "left", "right"] | None = Field(
|
||||
None,
|
||||
description=(
|
||||
"Smart directional swipe ('up', 'down', 'left', 'right') for general page"
|
||||
" scrolling/browsing. Automatically applies safe vectors and a ~40% visual"
|
||||
" overlap anchor: 'up' (drags bottom-to-top, scrolling down to reveal content below),"
|
||||
" 'down' (drags top-to-bottom, scrolling up to reveal content above),"
|
||||
" 'left' (drags right-to-left, scrolling right),"
|
||||
" 'right' (drags left-to-right, scrolling left)."
|
||||
),
|
||||
)
|
||||
start: list[int] | None = Field(
|
||||
None,
|
||||
description=(
|
||||
"Start normalized coordinates [start_x, start_y] in 0-1000 scale for precise,"
|
||||
" local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)."
|
||||
),
|
||||
)
|
||||
end: list[int] | None = Field(
|
||||
None,
|
||||
description=(
|
||||
"End normalized coordinates [end_x, end_y] in 0-1000 scale for precise,"
|
||||
" local interactions (e.g. adjusting sliders, SeekBars, fine range selection, or drag-and-drop)."
|
||||
),
|
||||
)
|
||||
action: Literal["up", "down", "left", "right"] | list[int] | None = Field(
|
||||
None,
|
||||
description=(
|
||||
"Backward-compatible swipe gesture: smart direction string ('up', 'down', 'left', 'right')"
|
||||
" OR precise custom coordinates [start_x, start_y, end_x, end_y] in 0-1000 scale."
|
||||
),
|
||||
)
|
||||
duration: int = Field(
|
||||
400,
|
||||
description=(
|
||||
"Optional swipe/drag duration in milliseconds (default 400). For drag-and-drop,"
|
||||
" list reordering, or sliding/adjusting sliders (e.g., volume, brightness, SeekBars),"
|
||||
" set duration >= 1000 (e.g. 1500). If omitted for directional swipe, duration is computed automatically."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class PressKeyArgs(BaseModel):
|
||||
"""Arguments schema for pressing system keys."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
key: Literal["ENTER", "BACK", "HOME", "APP_SWITCH"] = Field(
|
||||
...,
|
||||
description=("Standard Android system button name (ENTER, BACK, HOME, APP_SWITCH)."),
|
||||
)
|
||||
|
||||
|
||||
CLICK_DOCSTRING = "[ACTION] Click on the target normalized coordinates on the screen."
|
||||
|
||||
|
||||
class ClickTool(ArtemisTool):
|
||||
"""Universal tool for clicking on target normalized coordinates on the screen."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "action"):
|
||||
super().__init__(
|
||||
name="click",
|
||||
description=CLICK_DOCSTRING,
|
||||
args_schema=ClickArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
target: list[int] | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
tgt = target if target is not None else (kwargs.get("target") or kwargs.get("Target") or [])
|
||||
tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
width = 1080
|
||||
height = 2400
|
||||
if ctx and hasattr(ctx, "device") and ctx.device:
|
||||
width = getattr(ctx.device, "device_width", 1080)
|
||||
height = getattr(ctx.device, "device_height", 2400)
|
||||
elif driver and hasattr(driver, "screen_size"):
|
||||
width, height = driver.screen_size
|
||||
|
||||
try:
|
||||
if not tgt or len(tgt) < 2:
|
||||
raise ValueError(f"Invalid target coordinates: {tgt}")
|
||||
nx, ny = tgt[0], tgt[1]
|
||||
x = int(max(0, min(width - 1, nx * width / 1000)))
|
||||
y = int(max(0, min(height - 1, ny * height / 1000)))
|
||||
|
||||
if driver is not None and hasattr(driver, "tap"):
|
||||
success = await driver.tap(x, y)
|
||||
outcome = (
|
||||
f"Clicked at [{nx}, {ny}] (normalized) successfully."
|
||||
if success
|
||||
else f"Click failed at {tgt}"
|
||||
)
|
||||
elif ctx is not None:
|
||||
controller = UnifiedMobileController(ctx)
|
||||
result = await controller.tap_at(x, y)
|
||||
success = result.error is None
|
||||
outcome = (
|
||||
f"Clicked at [{nx}, {ny}] (normalized) successfully."
|
||||
if success
|
||||
else f"Failed to click: {result.error}"
|
||||
)
|
||||
else:
|
||||
success = False
|
||||
outcome = "Error during click: No driver or context provided."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
success = False
|
||||
outcome = f"Error during click: {e}"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid or "",
|
||||
content=outcome,
|
||||
status="success" if success else "error",
|
||||
)
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
|
||||
agent="validator",
|
||||
)
|
||||
)
|
||||
|
||||
return outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
click = ClickTool()
|
||||
Click = ClickTool
|
||||
ToolRegistry.register(click)
|
||||
|
||||
|
||||
def get_click_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports click as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(click.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
LONG_PRESS_DOCSTRING = "[ACTION] Long press on the target normalized coordinates on the screen."
|
||||
|
||||
|
||||
class LongPressTool(ArtemisTool):
|
||||
"""Universal tool for long pressing on target normalized coordinates on the screen."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "action"):
|
||||
super().__init__(
|
||||
name="long_press",
|
||||
description=LONG_PRESS_DOCSTRING,
|
||||
args_schema=LongPressArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
target: list[int] | None = None,
|
||||
duration: int = 1000,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
tgt = target if target is not None else (kwargs.get("target") or kwargs.get("Target") or [])
|
||||
dur = (
|
||||
duration
|
||||
if duration is not None
|
||||
else (kwargs.get("duration") or kwargs.get("Duration") or 1000)
|
||||
)
|
||||
tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
width = 1080
|
||||
height = 2400
|
||||
if ctx and hasattr(ctx, "device") and ctx.device:
|
||||
width = getattr(ctx.device, "device_width", 1080)
|
||||
height = getattr(ctx.device, "device_height", 2400)
|
||||
elif driver and hasattr(driver, "screen_size"):
|
||||
width, height = driver.screen_size
|
||||
|
||||
try:
|
||||
if not tgt or len(tgt) < 2:
|
||||
raise ValueError(f"Invalid target coordinates: {tgt}")
|
||||
nx, ny = tgt[0], tgt[1]
|
||||
x = int(max(0, min(width - 1, nx * width / 1000)))
|
||||
y = int(max(0, min(height - 1, ny * height / 1000)))
|
||||
|
||||
if driver is not None and hasattr(driver, "long_press"):
|
||||
success = await driver.long_press(x, y, duration_ms=dur)
|
||||
outcome = (
|
||||
f"Long pressed at [{nx}, {ny}] (normalized) successfully."
|
||||
if success
|
||||
else f"Failed to long press: Long press failed at {tgt}"
|
||||
)
|
||||
elif ctx is not None:
|
||||
controller = UnifiedMobileController(ctx)
|
||||
result = await controller.tap_at(x, y, long_press=True, long_press_duration=dur)
|
||||
success = result.error is None
|
||||
outcome = (
|
||||
f"Long pressed at [{nx}, {ny}] (normalized) successfully."
|
||||
if success
|
||||
else f"Failed to long press: {result.error}"
|
||||
)
|
||||
else:
|
||||
success = False
|
||||
outcome = "Error during long press: No driver or context provided."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
success = False
|
||||
outcome = f"Error during long press: {e}"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid or "",
|
||||
content=outcome,
|
||||
status="success" if success else "error",
|
||||
)
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
|
||||
agent="validator",
|
||||
)
|
||||
)
|
||||
|
||||
return outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
long_press = LongPressTool()
|
||||
LongPress = LongPressTool
|
||||
ToolRegistry.register(long_press)
|
||||
|
||||
|
||||
def get_long_press_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports long_press as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(long_press.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
INPUT_TEXT_DOCSTRING = "[ACTION] Type text into the target input field coordinates."
|
||||
|
||||
|
||||
class InputTextTool(ArtemisTool):
|
||||
"""Universal tool for typing text into target input field coordinates."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "action"):
|
||||
super().__init__(
|
||||
name="input_text",
|
||||
description=INPUT_TEXT_DOCSTRING,
|
||||
args_schema=InputTextArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
text: str | None = None,
|
||||
target: list[int] | None = None,
|
||||
clear_exist: bool = True,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
txt = text if text is not None else (kwargs.get("text") or kwargs.get("Text") or "")
|
||||
tgt = target if target is not None else (kwargs.get("target") or kwargs.get("Target") or [])
|
||||
clr = (
|
||||
clear_exist
|
||||
if clear_exist is not None
|
||||
else kwargs.get("clear_exist", kwargs.get("ClearExist", True))
|
||||
)
|
||||
tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
width = 1080
|
||||
height = 2400
|
||||
if ctx and hasattr(ctx, "device") and ctx.device:
|
||||
width = getattr(ctx.device, "device_width", 1080)
|
||||
height = getattr(ctx.device, "device_height", 2400)
|
||||
elif driver and hasattr(driver, "screen_size"):
|
||||
width, height = driver.screen_size
|
||||
|
||||
try:
|
||||
if not tgt or len(tgt) < 2:
|
||||
raise ValueError(f"Invalid target coordinates: {tgt}")
|
||||
nx, ny = tgt[0], tgt[1]
|
||||
x = int(max(0, min(width - 1, nx * width / 1000)))
|
||||
y = int(max(0, min(height - 1, ny * height / 1000)))
|
||||
|
||||
if driver is not None and hasattr(driver, "input_text"):
|
||||
await driver.tap(x, y)
|
||||
await asyncio.sleep(0.2)
|
||||
success = await driver.input_text(txt, clear_existing=clr)
|
||||
outcome = (
|
||||
f"Typed '{txt}' successfully."
|
||||
if success
|
||||
else f"Failed to input text into {tgt}"
|
||||
)
|
||||
elif ctx is not None:
|
||||
controller = UnifiedMobileController(ctx)
|
||||
tap_result = await controller.tap_at(x, y)
|
||||
if tap_result.error:
|
||||
raise RuntimeError(f"Failed to focus input field: {tap_result.error}")
|
||||
|
||||
if clr:
|
||||
await controller.erase_text()
|
||||
else:
|
||||
await controller.press_key("123") # Move to end
|
||||
|
||||
success = await controller.type_text(txt, clear_existing=False)
|
||||
outcome = f"Typed '{txt}' successfully." if success else "Failed to type text."
|
||||
else:
|
||||
success = False
|
||||
outcome = "Error during input text: No driver or context provided."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
success = False
|
||||
outcome = f"Error during input text: {e}"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid or "",
|
||||
content=outcome,
|
||||
status="success" if success else "error",
|
||||
)
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
|
||||
agent="validator",
|
||||
)
|
||||
)
|
||||
|
||||
return outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
input_text = InputTextTool()
|
||||
InputText = InputTextTool
|
||||
ToolRegistry.register(input_text)
|
||||
|
||||
|
||||
def get_input_text_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports input_text as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(input_text.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
SWIPE_DOCSTRING = (
|
||||
"[ACTION] Perform a swipe, drag, or slider-adjustment gesture on the"
|
||||
" screen.\n\n"
|
||||
"• Smart Directional Scrolling ('direction'): Best for general browsing"
|
||||
" and page scrolling. Automatically computes safe swipe vectors (locking"
|
||||
" axes to prevent edge/sidebar mis-clicks), retains a ~40% visual overlap"
|
||||
" anchor for zero-omission traversal, and prevents inertial flings.\n"
|
||||
"• Precise Coordinate Gestures ('start', 'end'): Best for local,"
|
||||
" fine-grained interactions such as adjusting sliders/SeekBars (e.g.,"
|
||||
" volume, brightness, progress bars) or drag-and-drop / list reordering."
|
||||
" Always drag slightly PAST the target position to overcome touch slop"
|
||||
" and reliably trigger the update. When setting a slider to Maximum (100%)"
|
||||
" or Minimum (0%), swipe fully to the extreme boundary."
|
||||
)
|
||||
|
||||
|
||||
class SwipeTool(ArtemisTool):
|
||||
"""Universal tool for performing swipe gestures on the screen."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "action"):
|
||||
super().__init__(
|
||||
name="swipe",
|
||||
description=SWIPE_DOCSTRING,
|
||||
args_schema=SwipeArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-branches,too-many-locals,too-many-statements
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
action: Literal["up", "down", "left", "right"] | list[int] | str | None = None,
|
||||
direction: Literal["up", "down", "left", "right"] | str | None = None,
|
||||
start: list[int] | None = None,
|
||||
end: list[int] | None = None,
|
||||
duration: int = 400,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
params = dict(kwargs)
|
||||
if action is not None:
|
||||
params["action"] = action
|
||||
if direction is not None:
|
||||
params["direction"] = direction
|
||||
if start is not None:
|
||||
params["start"] = start
|
||||
if end is not None:
|
||||
params["end"] = end
|
||||
if duration is not None:
|
||||
params["duration"] = duration
|
||||
|
||||
kind, target, dur = parse_swipe_parameters(params, default_duration=duration or 400)
|
||||
|
||||
tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
width = 1080
|
||||
height = 2400
|
||||
if ctx and hasattr(ctx, "device") and ctx.device:
|
||||
width = getattr(ctx.device, "device_width", 1080)
|
||||
height = getattr(ctx.device, "device_height", 2400)
|
||||
elif driver and hasattr(driver, "screen_size"):
|
||||
width, height = driver.screen_size
|
||||
|
||||
try:
|
||||
if kind is None or target is None:
|
||||
raise ValueError(
|
||||
f"Invalid swipe action parameter: {action or direction or (start, end) or kwargs}"
|
||||
)
|
||||
|
||||
if (
|
||||
driver is not None
|
||||
and hasattr(driver, "swipe_direction")
|
||||
and hasattr(driver, "swipe")
|
||||
):
|
||||
if kind == "direction" and isinstance(target, str):
|
||||
success = await driver.swipe_direction(target, duration_ms=dur)
|
||||
outcome = (
|
||||
"Swipe completed successfully." if success else f"Swipe {target} failed"
|
||||
)
|
||||
elif kind == "coords" and isinstance(target, list) and len(target) == 4:
|
||||
start_x = int(max(0, min(width - 1, target[0] * width / 1000)))
|
||||
start_y = int(max(0, min(height - 1, target[1] * height / 1000)))
|
||||
end_x = int(max(0, min(width - 1, target[2] * width / 1000)))
|
||||
end_y = int(max(0, min(height - 1, target[3] * height / 1000)))
|
||||
success = await driver.swipe(start_x, start_y, end_x, end_y, duration_ms=dur)
|
||||
outcome = (
|
||||
"Swipe completed successfully."
|
||||
if success
|
||||
else f"Swipe coordinates {target} failed"
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid swipe action parameter: {target}")
|
||||
elif ctx is not None:
|
||||
controller = UnifiedMobileController(ctx)
|
||||
|
||||
if kind == "direction" and isinstance(target, str):
|
||||
if target == "up":
|
||||
coords = [
|
||||
width // 2,
|
||||
int(height * 0.8),
|
||||
width // 2,
|
||||
int(height * 0.2),
|
||||
]
|
||||
elif target == "down":
|
||||
coords = [
|
||||
width // 2,
|
||||
int(height * 0.2),
|
||||
width // 2,
|
||||
int(height * 0.8),
|
||||
]
|
||||
elif target == "left":
|
||||
coords = [
|
||||
int(width * 0.8),
|
||||
height // 2,
|
||||
int(width * 0.2),
|
||||
height // 2,
|
||||
]
|
||||
elif target == "right":
|
||||
coords = [
|
||||
int(width * 0.2),
|
||||
height // 2,
|
||||
int(width * 0.8),
|
||||
height // 2,
|
||||
]
|
||||
else:
|
||||
raise ValueError(f"Invalid direction: {target}")
|
||||
|
||||
start_x, start_y, end_x, end_y = coords
|
||||
elif kind == "coords" and isinstance(target, list) and len(target) == 4:
|
||||
nx1, ny1, nx2, ny2 = target
|
||||
start_x = int(max(0, min(width - 1, nx1 * width / 1000)))
|
||||
start_y = int(max(0, min(height - 1, ny1 * height / 1000)))
|
||||
end_x = int(max(0, min(width - 1, nx2 * width / 1000)))
|
||||
end_y = int(max(0, min(height - 1, ny2 * height / 1000)))
|
||||
else:
|
||||
raise ValueError(f"Invalid swipe action parameter: {target}")
|
||||
|
||||
error = await controller.swipe_coords(start_x, start_y, end_x, end_y, dur)
|
||||
success = error is None
|
||||
outcome = "Swipe completed successfully." if success else f"Swipe failed: {error}"
|
||||
else:
|
||||
success = False
|
||||
outcome = "Error during swipe: No driver or context provided."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
success = False
|
||||
outcome = f"Error during swipe: {e}"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid or "",
|
||||
content=outcome,
|
||||
status="success" if success else "error",
|
||||
)
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
|
||||
agent="validator",
|
||||
)
|
||||
)
|
||||
|
||||
return outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
swipe = SwipeTool()
|
||||
Swipe = SwipeTool
|
||||
ToolRegistry.register(swipe)
|
||||
|
||||
|
||||
def get_swipe_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports swipe as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(swipe.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
PRESS_KEY_DOCSTRING = "[ACTION] Press a physical or virtual system button."
|
||||
|
||||
|
||||
class PressKeyTool(ArtemisTool):
|
||||
"""Universal tool for pressing a physical or virtual system button."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "action"):
|
||||
super().__init__(
|
||||
name="press_key",
|
||||
description=PRESS_KEY_DOCSTRING,
|
||||
args_schema=PressKeyArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
key: Literal["ENTER", "BACK", "HOME", "APP_SWITCH"] | str | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
k = key if key is not None else (kwargs.get("key") or kwargs.get("Key") or "")
|
||||
tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
try:
|
||||
if not k:
|
||||
raise ValueError("Key parameter is required.")
|
||||
|
||||
if driver is not None and hasattr(driver, "press_key"):
|
||||
success = await driver.press_key(k)
|
||||
outcome = (
|
||||
f"Pressed key '{k}' successfully." if success else f"Failed to press key '{k}'."
|
||||
)
|
||||
elif ctx is not None:
|
||||
controller = UnifiedMobileController(ctx)
|
||||
success = False
|
||||
if k == "ENTER":
|
||||
success = await controller.press_enter()
|
||||
elif k == "BACK":
|
||||
success = await controller.go_back()
|
||||
elif k == "HOME":
|
||||
success = await controller.go_home()
|
||||
elif k == "APP_SWITCH":
|
||||
success = await controller.press_key("KEYCODE_APP_SWITCH")
|
||||
else:
|
||||
success = await controller.press_key(k)
|
||||
|
||||
outcome = (
|
||||
f"Pressed key '{k}' successfully." if success else f"Failed to press key '{k}'."
|
||||
)
|
||||
else:
|
||||
success = False
|
||||
outcome = "Error during press key: No driver or context provided."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
success = False
|
||||
outcome = f"Error during press key: {e}"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid or "",
|
||||
content=outcome,
|
||||
status="success" if success else "error",
|
||||
)
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={VALIDATOR_MESSAGES_KEY: [tool_message]},
|
||||
agent="validator",
|
||||
)
|
||||
)
|
||||
|
||||
return outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
press_key = PressKeyTool()
|
||||
PressKey = PressKeyTool
|
||||
ToolRegistry.register(press_key)
|
||||
|
||||
|
||||
def get_press_key_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports press_key as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(press_key.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
# Wrappers to expose
|
||||
click_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_click_tool,
|
||||
on_success_fn=lambda *args, **kwargs: "Success",
|
||||
on_failure_fn=lambda *args, **kwargs: "Failure",
|
||||
)
|
||||
long_press_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_long_press_tool,
|
||||
on_success_fn=lambda *args, **kwargs: "Success",
|
||||
on_failure_fn=lambda *args, **kwargs: "Failure",
|
||||
)
|
||||
input_text_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_input_text_tool,
|
||||
on_success_fn=lambda *args, **kwargs: "Success",
|
||||
on_failure_fn=lambda *args, **kwargs: "Failure",
|
||||
)
|
||||
swipe_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_swipe_tool,
|
||||
on_success_fn=lambda *args, **kwargs: "Success",
|
||||
on_failure_fn=lambda *args, **kwargs: "Failure",
|
||||
)
|
||||
press_key_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_press_key_tool,
|
||||
on_success_fn=lambda *args, **kwargs: "Success",
|
||||
on_failure_fn=lambda *args, **kwargs: "Failure",
|
||||
)
|
||||
@@ -0,0 +1,215 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langchain_core.tools import BaseTool
|
||||
from langgraph.types import Command
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.agents.hopper.hopper import HopperOutput, hopper
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.controllers.platform_specific_commands_controller import (
|
||||
list_packages_async,
|
||||
)
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolCategory, ToolRegistry
|
||||
from artemis.tools.tool_wrapper import ToolWrapper
|
||||
from artemis.tools.types import CyFunctionDetector
|
||||
from artemis.utils.app_launch_utils import launch_app_with_retries
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class LaunchAppArgs(BaseModel):
|
||||
"""Arguments schema for launching an application."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
app_name: str = Field(
|
||||
...,
|
||||
description="The natural language name of the application to launch.",
|
||||
)
|
||||
|
||||
|
||||
LAUNCH_APP_DOCSTRING = (
|
||||
"[ACTION] Finds and launches an application on the device using its natural language name."
|
||||
)
|
||||
|
||||
|
||||
async def find_package(ctx: ArtemisContext, app_name: str, use_fallback: bool = True) -> str | None:
|
||||
"""Finds the package name for a given application name.
|
||||
|
||||
Returns None if package not found or on error.
|
||||
"""
|
||||
package_cache = getattr(ctx, "package_cache", None)
|
||||
if package_cache is None or not isinstance(package_cache, dict):
|
||||
try:
|
||||
ctx.package_cache = {}
|
||||
package_cache = ctx.package_cache
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
package_cache = {}
|
||||
|
||||
if isinstance(package_cache, dict) and app_name in package_cache:
|
||||
logger.info(f"Cache hit for app '{app_name}': {package_cache[app_name]}")
|
||||
return package_cache[app_name]
|
||||
|
||||
try:
|
||||
all_packages = await list_packages_async(ctx=ctx)
|
||||
package_set = {p.strip() for p in all_packages.split("\n") if p.strip()}
|
||||
|
||||
# Fast path: If app_name is already directly an installed package name
|
||||
if app_name in package_set:
|
||||
if isinstance(package_cache, dict):
|
||||
package_cache[app_name] = app_name
|
||||
return app_name
|
||||
|
||||
hopper_output: HopperOutput = await hopper(
|
||||
ctx=ctx,
|
||||
request=(f"I'm looking for the package name of the following app: '{app_name}'"),
|
||||
data=all_packages,
|
||||
use_fallback=use_fallback,
|
||||
)
|
||||
if not hopper_output.found or not hopper_output.output:
|
||||
if isinstance(package_cache, dict):
|
||||
package_cache[app_name] = None
|
||||
return None
|
||||
|
||||
package_name = hopper_output.output.strip()
|
||||
if package_name not in package_set:
|
||||
logger.warning(
|
||||
f"Hopper returned package '{package_name}' for '{app_name}', "
|
||||
"but it is NOT physically installed on the device!"
|
||||
)
|
||||
if isinstance(package_cache, dict):
|
||||
package_cache[app_name] = None
|
||||
return None
|
||||
|
||||
if isinstance(package_cache, dict):
|
||||
package_cache[app_name] = package_name
|
||||
return package_name
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to find package for '{app_name}': {e}")
|
||||
return None
|
||||
|
||||
|
||||
class LaunchAppTool(ArtemisTool):
|
||||
"""Universal tool for finding and launching an application on the device."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "action"):
|
||||
super().__init__(
|
||||
name="launch_app",
|
||||
description=LAUNCH_APP_DOCSTRING,
|
||||
args_schema=LaunchAppArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
app_name: str | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
app = (
|
||||
app_name
|
||||
if app_name is not None
|
||||
else (kwargs.get("app_name") or kwargs.get("AppName") or "")
|
||||
)
|
||||
tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
success = False
|
||||
error_msg = None
|
||||
|
||||
try:
|
||||
if not app:
|
||||
raise ValueError("app_name parameter is required.")
|
||||
|
||||
if ctx is not None:
|
||||
package_name = await find_package(ctx=ctx, app_name=app)
|
||||
if not package_name:
|
||||
success = False
|
||||
outcome = f"Failed to launch app '{app}': Package not found."
|
||||
error_msg = "Package not found."
|
||||
else:
|
||||
success, error_msg = await launch_app_with_retries(
|
||||
ctx=ctx, app_package=package_name
|
||||
)
|
||||
outcome = (
|
||||
f"App '{app}' launched successfully."
|
||||
if success
|
||||
else f"Failed to launch app '{app}': {error_msg}"
|
||||
)
|
||||
elif driver is not None and hasattr(driver, "launch_app"):
|
||||
success = await driver.launch_app(app)
|
||||
outcome = (
|
||||
f"App '{app}' launched successfully."
|
||||
if success
|
||||
else f"Failed to launch app '{app}'."
|
||||
)
|
||||
error_msg = None if success else "Launch failed."
|
||||
else:
|
||||
success = False
|
||||
outcome = "Error during launch app: No driver or context provided."
|
||||
error_msg = "No driver or context provided."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
success = False
|
||||
outcome = f"Error during launch app: {e}"
|
||||
error_msg = str(e)
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
additional_kwargs = {} if success else ({"error": error_msg} if error_msg else {})
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid or "",
|
||||
content=outcome,
|
||||
additional_kwargs=additional_kwargs,
|
||||
status="success" if success else "error",
|
||||
)
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={
|
||||
VALIDATOR_MESSAGES_KEY: [tool_message],
|
||||
},
|
||||
agent="validator",
|
||||
)
|
||||
)
|
||||
|
||||
return outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
launch_app = LaunchAppTool()
|
||||
LaunchApp = LaunchAppTool
|
||||
ToolRegistry.register(launch_app)
|
||||
|
||||
|
||||
def get_launch_app_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports launch_app as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(launch_app.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
launch_app_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_launch_app_tool,
|
||||
on_success_fn=lambda *args, **kwargs: "Success",
|
||||
on_failure_fn=lambda *args, **kwargs: "Failure",
|
||||
)
|
||||
@@ -0,0 +1,138 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
import time
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.controllers.platform_specific_commands_controller import get_adb_device
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def parse_log_time(time_str: str) -> datetime:
|
||||
"""Parses a log timestamp string into a datetime object."""
|
||||
now = datetime.now()
|
||||
current_year = now.year
|
||||
|
||||
def _try_parse(year: int) -> datetime | None:
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S"):
|
||||
try:
|
||||
return datetime.strptime(f"{year}-{time_str}", fmt)
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
dt = _try_parse(current_year)
|
||||
if dt is None:
|
||||
raise ValueError(f"Unsupported time format: {time_str}")
|
||||
|
||||
# If parsed time is in the future (allowing 1 hour grace for clock drift),
|
||||
# we assume it belongs to the previous year.
|
||||
if dt > now + timedelta(hours=1):
|
||||
prev_dt = _try_parse(current_year - 1)
|
||||
if prev_dt is not None:
|
||||
dt = prev_dt
|
||||
|
||||
return dt
|
||||
|
||||
|
||||
def resolve_time(
|
||||
time_input: str | float | None, session_start: float | None
|
||||
) -> tuple[str | None, datetime | None]:
|
||||
"""Resolves relative or absolute time input into formatted string and datetime."""
|
||||
if time_input is None:
|
||||
return None, None
|
||||
|
||||
if isinstance(time_input, (int, float)) or (
|
||||
isinstance(time_input, str) and time_input.replace(".", "", 1).isdigit()
|
||||
):
|
||||
# It's relative seconds
|
||||
if session_start is None:
|
||||
raise ValueError("Cannot resolve relative time without session start time.")
|
||||
rel_sec = float(time_input)
|
||||
abs_ts = session_start + rel_sec
|
||||
|
||||
t_struct = time.localtime(abs_ts)
|
||||
t_str = time.strftime("%m-%d %H:%M:%S", t_struct)
|
||||
ms = int((abs_ts - int(abs_ts)) * 1000)
|
||||
formatted_str = f"{t_str}.{ms:03d}"
|
||||
|
||||
dt = datetime.fromtimestamp(abs_ts)
|
||||
return formatted_str, dt
|
||||
|
||||
# Assume it's absolute time string "MM-DD HH:MM:SS.ms"
|
||||
try:
|
||||
dt = parse_log_time(str(time_input))
|
||||
except ValueError as e:
|
||||
raise ValueError(f"Unsupported time format: {time_input}") from e
|
||||
return str(time_input), dt
|
||||
|
||||
|
||||
# pylint: disable=too-many-locals
|
||||
def fetch_and_filter_logs(
|
||||
ctx: ArtemisContext,
|
||||
lines: int = 1000,
|
||||
since_time: str | None = None,
|
||||
until_time: str | None = None,
|
||||
) -> str:
|
||||
"""Fetches logs from ADB device and filters them by time if needed."""
|
||||
device = get_adb_device(ctx)
|
||||
|
||||
session_start = ctx.data_engine.session_start_time if ctx.data_engine else None
|
||||
|
||||
since_str, since_dt = resolve_time(since_time, session_start)
|
||||
_, until_dt = resolve_time(until_time, session_start)
|
||||
|
||||
if since_str:
|
||||
try:
|
||||
logger.info(f"Attempting to fetch logs since {since_str}...")
|
||||
logs = device.shell(f'logcat -v threadtime -t "{since_str}"')
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.warning(
|
||||
f"Failed to fetch logs since time directly: {e}. Falling back"
|
||||
" to reading lines and filtering."
|
||||
)
|
||||
logs = device.shell(f"logcat -v threadtime -t {lines}")
|
||||
else:
|
||||
logger.info(f"Reading last {lines} lines of logs...")
|
||||
logs = device.shell(f"logcat -v threadtime -t {lines}")
|
||||
|
||||
if since_dt or until_dt:
|
||||
filtered_lines = []
|
||||
for line in logs.splitlines():
|
||||
if len(line) >= 18:
|
||||
log_time_str = line[:18]
|
||||
try:
|
||||
log_dt = parse_log_time(log_time_str)
|
||||
|
||||
if since_dt and log_dt < since_dt:
|
||||
continue
|
||||
if until_dt and log_dt > until_dt:
|
||||
continue
|
||||
|
||||
filtered_lines.append(line)
|
||||
except ValueError:
|
||||
pass
|
||||
# Cap to the requested line limit so timestamp fetches don't overflow context
|
||||
if 0 < lines < len(filtered_lines):
|
||||
filtered_lines = filtered_lines[-lines:]
|
||||
logs = "\n".join(filtered_lines)
|
||||
else:
|
||||
# Ensure fallback capping even when no datetime parsing occurred
|
||||
log_lines = logs.splitlines()
|
||||
if 0 < lines < len(log_lines):
|
||||
logs = "\n".join(log_lines[-lines:])
|
||||
|
||||
return logs
|
||||
@@ -0,0 +1,210 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langchain_core.tools import BaseTool
|
||||
from langgraph.types import Command
|
||||
from pydantic import BaseModel
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolCategory, ToolRegistry
|
||||
from artemis.tools.tool_wrapper import ToolWrapper
|
||||
from artemis.utils.logger import get_logger
|
||||
from artemis.utils.ocr_api import perform_ocr
|
||||
|
||||
|
||||
class OcrArgs(BaseModel):
|
||||
"""Arguments schema for OCR recognition."""
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
OCR_DOCSTRING = (
|
||||
"[EXPLORER] Performs OCR on the current screen and returns the detected"
|
||||
" text along with their coordinates."
|
||||
)
|
||||
|
||||
|
||||
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
|
||||
async def run_ocr_core(
|
||||
ctx: ArtemisContext | None = None,
|
||||
state: State | None = None,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
) -> str:
|
||||
"""Universal OCR core function: reads screenshot from state or driver and returns results."""
|
||||
screenshot_b64 = None
|
||||
screenshot_path = getattr(state, "latest_screenshot", None) if state else None
|
||||
|
||||
if screenshot_path:
|
||||
if not Path(screenshot_path).exists():
|
||||
logger.error(f"Screenshot file does not exist: {screenshot_path}")
|
||||
raise FileNotFoundError(f"Screenshot file does not exist: {screenshot_path}")
|
||||
|
||||
try:
|
||||
with open(screenshot_path, "rb") as f:
|
||||
screenshot_b64 = base64.b64encode(f.read()).decode("utf-8")
|
||||
logger.info(f"OCR loaded screenshot from file: {screenshot_path}")
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to read screenshot from {screenshot_path}: {e}")
|
||||
raise e
|
||||
elif driver is not None and hasattr(driver, "get_screen_data"):
|
||||
try:
|
||||
screen_data = await driver.get_screen_data()
|
||||
if hasattr(screen_data, "screenshot_base64") and screen_data.screenshot_base64:
|
||||
screenshot_b64 = screen_data.screenshot_base64
|
||||
elif hasattr(screen_data, "screenshot_bytes") and screen_data.screenshot_bytes:
|
||||
screenshot_b64 = base64.b64encode(screen_data.screenshot_bytes).decode("utf-8")
|
||||
elif hasattr(screen_data, "base64") and screen_data.base64:
|
||||
screenshot_b64 = screen_data.base64
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to capture screenshot from driver: {e}")
|
||||
raise e
|
||||
else:
|
||||
logger.error("No screenshot path found in state.latest_screenshot and no driver available")
|
||||
raise ValueError(
|
||||
"No screenshot path found in state.latest_screenshot and no driver available"
|
||||
)
|
||||
|
||||
if not screenshot_b64:
|
||||
raise ValueError("Failed to obtain screenshot for OCR.")
|
||||
|
||||
width = 1080
|
||||
height = 2400
|
||||
try:
|
||||
if ctx and hasattr(ctx, "device") and ctx.device:
|
||||
w = getattr(ctx.device, "device_width", 1080)
|
||||
h = getattr(ctx.device, "device_height", 2400)
|
||||
if isinstance(w, int):
|
||||
width = w
|
||||
if isinstance(h, int):
|
||||
height = h
|
||||
elif driver and hasattr(driver, "screen_size") and driver.screen_size:
|
||||
width, height = driver.screen_size
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.warning(f"Failed to get device resolution for OCR normalization: {e}")
|
||||
|
||||
results = await perform_ocr(screenshot_b64)
|
||||
if results:
|
||||
normalized_results = []
|
||||
for item in results:
|
||||
text = item.get("text")
|
||||
vertices = item.get("position") or []
|
||||
|
||||
# Calculate center point
|
||||
x_coords = [v.get("x", 0) for v in vertices]
|
||||
y_coords = [v.get("y", 0) for v in vertices]
|
||||
|
||||
if x_coords and y_coords:
|
||||
min_x, max_x = min(x_coords), max(x_coords)
|
||||
min_y, max_y = min(y_coords), max(y_coords)
|
||||
|
||||
center_x = (min_x + max_x) / 2
|
||||
center_y = (min_y + max_y) / 2
|
||||
|
||||
# Normalize to 0-1000
|
||||
norm_x = int(max(0, min(1000, center_x * 1000 / width)))
|
||||
norm_y = int(max(0, min(1000, center_y * 1000 / height)))
|
||||
|
||||
normalized_results.append({"text": text, "coordinates": [norm_x, norm_y]})
|
||||
else:
|
||||
normalized_results.append({"text": text, "coordinates": None})
|
||||
|
||||
return json.dumps(normalized_results, ensure_ascii=False, indent=2)
|
||||
return "No text detected on the screen."
|
||||
|
||||
|
||||
class OcrRecognitionTool(ArtemisTool):
|
||||
"""Universal tool for performing OCR on the current screen."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "explorer"):
|
||||
super().__init__(
|
||||
name="ocr_recognition",
|
||||
description=OCR_DOCSTRING,
|
||||
args_schema=OcrArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
has_failed = False
|
||||
output = ""
|
||||
error_message = None
|
||||
|
||||
try:
|
||||
output = await run_ocr_core(ctx=ctx, state=st, driver=driver)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
has_failed = True
|
||||
error_message = f"Failed to perform OCR: {str(e)}"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid or "",
|
||||
content=ocr_recognition_wrapper.on_failure_fn(error_message)
|
||||
if has_failed
|
||||
else ocr_recognition_wrapper.on_success_fn(output),
|
||||
additional_kwargs={"error": error_message} if has_failed else {},
|
||||
status="error" if has_failed else "success",
|
||||
)
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={
|
||||
VALIDATOR_MESSAGES_KEY: [tool_message],
|
||||
},
|
||||
agent="validator",
|
||||
)
|
||||
)
|
||||
|
||||
if has_failed:
|
||||
return ocr_recognition_wrapper.on_failure_fn(error_message)
|
||||
return ocr_recognition_wrapper.on_success_fn(output)
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
ocr_recognition = OcrRecognitionTool()
|
||||
OcrRecognition = OcrRecognitionTool
|
||||
OCRRecognition = OcrRecognitionTool
|
||||
OcrTool = OcrRecognitionTool
|
||||
ToolRegistry.register(ocr_recognition)
|
||||
|
||||
|
||||
def get_ocr_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports ocr_recognition as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(ocr_recognition.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
ocr_recognition_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_ocr_tool,
|
||||
on_success_fn=lambda output: f"OCR Recognition successful. Results:\n{output}",
|
||||
on_failure_fn=lambda error: f"OCR Recognition failed: {error}",
|
||||
)
|
||||
@@ -0,0 +1,106 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import inspect
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
from pydantic import BaseModel
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolCategory, ToolRegistry
|
||||
from artemis.tools.tool_wrapper import ToolWrapper
|
||||
from artemis.tools.types import CyFunctionDetector
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class GetUiHierarchyArgs(BaseModel):
|
||||
"""Arguments schema for getting UI hierarchy."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
|
||||
|
||||
GET_UI_HIERARCHY_DOCSTRING = (
|
||||
"[DIAGNOSTIC] Retrieves the current screen UI hierarchy XML using a persistent"
|
||||
" UI Automator client.\n\nUse this instead of running 'uiautomator dump'"
|
||||
" command via adb shell, as it is much faster and more stable."
|
||||
)
|
||||
|
||||
|
||||
class GetUiHierarchyTool(ArtemisTool):
|
||||
"""Universal tool for retrieving screen UI hierarchy XML."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "diagnostic"):
|
||||
super().__init__(
|
||||
name="get_ui_hierarchy",
|
||||
description=GET_UI_HIERARCHY_DOCSTRING,
|
||||
args_schema=GetUiHierarchyArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None,
|
||||
ctx: ArtemisContext | None = None,
|
||||
tool_call_id: str | None = None, # pylint: disable=unused-argument
|
||||
state: State | None = None, # pylint: disable=unused-argument
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
try:
|
||||
logger.info("get_ui_hierarchy tool called.")
|
||||
if ctx is not None and getattr(ctx, "ui_adb_client", None) is not None:
|
||||
xml_hierarchy = ctx.ui_adb_client.get_hierarchy()
|
||||
if inspect.iscoroutine(xml_hierarchy):
|
||||
xml_hierarchy = await xml_hierarchy
|
||||
return str(xml_hierarchy)
|
||||
if driver is not None and hasattr(driver, "get_screen_data"):
|
||||
screen_data = await driver.get_screen_data()
|
||||
xml_hierarchy = getattr(screen_data, "ui_hierarchy_xml", None)
|
||||
if xml_hierarchy:
|
||||
return str(xml_hierarchy)
|
||||
return "Error retrieving UI hierarchy: No UI hierarchy in screen data."
|
||||
if driver is not None and hasattr(driver, "get_ui_hierarchy"):
|
||||
xml_hierarchy = await driver.get_ui_hierarchy()
|
||||
if inspect.iscoroutine(xml_hierarchy):
|
||||
xml_hierarchy = await xml_hierarchy
|
||||
return str(xml_hierarchy)
|
||||
return "Error retrieving UI hierarchy: No UI Automator client or driver provided."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to get UI hierarchy: {e}")
|
||||
return f"Error retrieving UI hierarchy: {e}"
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
get_ui_hierarchy = GetUiHierarchyTool()
|
||||
GetUiHierarchy = GetUiHierarchyTool
|
||||
GetUIHierarchy = GetUiHierarchyTool
|
||||
ToolRegistry.register(get_ui_hierarchy)
|
||||
|
||||
|
||||
def get_ui_hierarchy_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports get_ui_hierarchy as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(get_ui_hierarchy.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
ui_hierarchy_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_ui_hierarchy_tool,
|
||||
on_success_fn=lambda output: output,
|
||||
on_failure_fn=lambda error: f"Failed to retrieve UI hierarchy: {error}",
|
||||
)
|
||||
@@ -0,0 +1,116 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.tools.base import BaseTool
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolCategory, ToolRegistry
|
||||
from artemis.tools.mobile.log_utils import fetch_and_filter_logs
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class ReadLogsArgs(BaseModel):
|
||||
"""Arguments schema for read_logs tool."""
|
||||
|
||||
lines: int | None = Field(
|
||||
default=200,
|
||||
description=(
|
||||
"Number of recent log lines to read. Default is 200. Use larger"
|
||||
" numbers only if necessary."
|
||||
),
|
||||
)
|
||||
since_time: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Target timestamp in 'MM-DD HH:MM:SS.ms' format OR relative seconds"
|
||||
" (e.g., '15.5') to read logs since. If provided, we will try to"
|
||||
" fetch logs starting from this time."
|
||||
),
|
||||
)
|
||||
until_time: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Target timestamp in 'MM-DD HH:MM:SS.ms' format OR relative seconds"
|
||||
" (e.g., '20.0') to read logs until. If provided, we will filter"
|
||||
" logs up to this time."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
READ_LOGS_DOCSTRING = (
|
||||
"[DIAGNOSTIC] Fetches raw Android logs.\n\n"
|
||||
"- Use when: You need to scan logs for a time window or see the sequence"
|
||||
" of events.\n"
|
||||
"- Input: lines (int, optional): Number of lines to read (default 200)."
|
||||
" since_time (str, optional): Timestamp or relative seconds to read since."
|
||||
" until_time (str, optional): Timestamp or relative seconds to read until.\n"
|
||||
"- Output: Raw log text."
|
||||
)
|
||||
|
||||
|
||||
class ReadLogsTool(ArtemisTool):
|
||||
"""Universal tool for fetching raw Android logs."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "diagnostic"):
|
||||
super().__init__(
|
||||
name="read_logs",
|
||||
description=READ_LOGS_DOCSTRING,
|
||||
args_schema=ReadLogsArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
lines: int | None = 200,
|
||||
since_time: str | None = None,
|
||||
until_time: str | None = None,
|
||||
tool_call_id: str | None = None, # pylint: disable=unused-argument
|
||||
state: State | None = None, # pylint: disable=unused-argument
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
try:
|
||||
num_lines = 200 if lines is None else lines
|
||||
logs = fetch_and_filter_logs(
|
||||
ctx=ctx,
|
||||
lines=num_lines,
|
||||
since_time=since_time,
|
||||
until_time=until_time,
|
||||
)
|
||||
return logs
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to read logs: {e}")
|
||||
return f"Failed to read logs: {e}"
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
read_logs = ReadLogsTool()
|
||||
ReadLogs = ReadLogsTool
|
||||
ReadLogsToolAlias = ReadLogsTool
|
||||
ToolRegistry.register(read_logs)
|
||||
|
||||
|
||||
def get_read_logs_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports read_logs as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(read_logs.to_langchain_tool(ctx), ctx)
|
||||
@@ -0,0 +1,237 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.tools.base import BaseTool
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolCategory, ToolRegistry
|
||||
from artemis.tools.mobile.log_utils import fetch_and_filter_logs
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
|
||||
def search_and_merge_logs(
|
||||
logs: str,
|
||||
keyword: str,
|
||||
context_lines: int = 50,
|
||||
is_regex: bool = False,
|
||||
) -> str:
|
||||
"""Searches for keyword/pattern in logs and merges overlapping context windows."""
|
||||
log_lines = logs.splitlines()
|
||||
|
||||
if is_regex:
|
||||
try:
|
||||
pattern = re.compile(keyword, re.IGNORECASE)
|
||||
except re.error as e:
|
||||
return f"Error: Invalid regex: {e}"
|
||||
else:
|
||||
keyword_lower = keyword.lower()
|
||||
|
||||
# 1. Collect all raw match ranges
|
||||
ranges = []
|
||||
for i, line in enumerate(log_lines):
|
||||
matched = False
|
||||
if is_regex:
|
||||
if pattern.search(line):
|
||||
matched = True
|
||||
else:
|
||||
if keyword_lower in line.lower():
|
||||
matched = True
|
||||
|
||||
if matched:
|
||||
start = max(0, i - context_lines)
|
||||
end = min(len(log_lines), i + context_lines + 1)
|
||||
ranges.append((start, end, i))
|
||||
|
||||
if not ranges:
|
||||
return f"No matches found for '{keyword}'."
|
||||
|
||||
# 2. Merge overlapping/adjacent ranges
|
||||
ranges.sort(key=lambda x: x[0])
|
||||
|
||||
merged_ranges = []
|
||||
curr_start, curr_end, curr_matches = (
|
||||
ranges[0][0],
|
||||
ranges[0][1],
|
||||
[ranges[0][2]],
|
||||
)
|
||||
|
||||
for start, end, match_idx in ranges[1:]:
|
||||
if start <= curr_end: # Overlaps or adjacent
|
||||
curr_end = max(curr_end, end)
|
||||
curr_matches.append(match_idx)
|
||||
else:
|
||||
merged_ranges.append((curr_start, curr_end, curr_matches))
|
||||
curr_start = start
|
||||
curr_end = end
|
||||
curr_matches = [match_idx]
|
||||
merged_ranges.append((curr_start, curr_end, curr_matches))
|
||||
|
||||
# 3. Format output
|
||||
matches = []
|
||||
total_lines = 0
|
||||
max_return_lines = 300
|
||||
|
||||
truncated = False
|
||||
for start, end, match_indices in merged_ranges:
|
||||
if total_lines >= max_return_lines:
|
||||
truncated = True
|
||||
break
|
||||
|
||||
match_str = ", ".join(map(str, match_indices))
|
||||
matches.append(f"--- Match at line(s) {match_str} ---")
|
||||
for idx in range(start, end):
|
||||
line_content = log_lines[idx]
|
||||
if idx in match_indices:
|
||||
matches.append(f"{idx:4d}: [MATCH] {line_content}")
|
||||
else:
|
||||
matches.append(f"{idx:4d}: {line_content}")
|
||||
|
||||
total_lines += 1
|
||||
if total_lines >= max_return_lines:
|
||||
truncated = True
|
||||
break
|
||||
|
||||
matches.append("-" * 20)
|
||||
|
||||
if truncated:
|
||||
matches.append(
|
||||
"\n... [SYSTEM WARNING: Output truncated to protect Context Window."
|
||||
" Too many matches exceeded the 300 lines limit. You MUST use a"
|
||||
" more specific keyword, narrow the time range, or use"
|
||||
" context_lines=0.] ..."
|
||||
)
|
||||
|
||||
return "\n".join(matches)
|
||||
|
||||
|
||||
class SearchLogsArgs(BaseModel):
|
||||
"""Arguments schema for search_logs tool."""
|
||||
|
||||
keyword: str = Field(
|
||||
...,
|
||||
description=("The keyword or regular expression to search for. Case-insensitive."),
|
||||
)
|
||||
is_regex: bool = Field(
|
||||
default=False,
|
||||
description=("Whether to treat keyword as a regular expression. Default is False."),
|
||||
)
|
||||
lines: int | None = Field(
|
||||
default=10000,
|
||||
description=("Number of recent log lines to search in. Default is 10000."),
|
||||
)
|
||||
since_time: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Target timestamp in 'MM-DD HH:MM:SS.ms' format OR relative seconds"
|
||||
" (e.g., '15.5') to search logs since. If provided, we will search"
|
||||
" logs starting from this time."
|
||||
),
|
||||
)
|
||||
until_time: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Target timestamp in 'MM-DD HH:MM:SS.ms' format OR relative seconds"
|
||||
" (e.g., '20.0') to search logs until. If provided, we will filter"
|
||||
" logs up to this time."
|
||||
),
|
||||
)
|
||||
context_lines: int | None = Field(
|
||||
default=0,
|
||||
description=(
|
||||
"Number of lines to include before and after each match for"
|
||||
" context. Default is 0. Only increase if you need more context"
|
||||
" around a very specific match."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
SEARCH_LOGS_DOCSTRING = (
|
||||
"[DIAGNOSTIC] Searches logs for a keyword or regex pattern.\n\n"
|
||||
"- Use when: Looking for specific errors, tags, or keywords without"
|
||||
" reading all logs.\n"
|
||||
"- Input: keyword (str): Search pattern. is_regex (bool, optional):"
|
||||
" Treat keyword as regex. context_lines (int, optional): Lines around"
|
||||
" match. since_time (str, optional): Timestamp or relative seconds to"
|
||||
" search since. until_time (str, optional): Timestamp or relative seconds"
|
||||
" to search until.\n"
|
||||
"- Output: Matching lines with context."
|
||||
)
|
||||
|
||||
|
||||
class SearchLogsTool(ArtemisTool):
|
||||
"""Universal tool for searching logs for keywords or regex patterns."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "diagnostic"):
|
||||
super().__init__(
|
||||
name="search_logs",
|
||||
description=SEARCH_LOGS_DOCSTRING,
|
||||
args_schema=SearchLogsArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
keyword: str = "",
|
||||
is_regex: bool = False,
|
||||
lines: int | None = 10000,
|
||||
since_time: str | None = None,
|
||||
until_time: str | None = None,
|
||||
context_lines: int | None = 0,
|
||||
tool_call_id: str | None = None, # pylint: disable=unused-argument
|
||||
state: State | None = None, # pylint: disable=unused-argument
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
try:
|
||||
num_lines = 10000 if lines is None else lines
|
||||
ctx_lines = 0 if context_lines is None else context_lines
|
||||
logs = fetch_and_filter_logs(
|
||||
ctx=ctx,
|
||||
lines=num_lines,
|
||||
since_time=since_time,
|
||||
until_time=until_time,
|
||||
)
|
||||
return search_and_merge_logs(
|
||||
logs=logs,
|
||||
keyword=keyword,
|
||||
context_lines=ctx_lines,
|
||||
is_regex=is_regex,
|
||||
)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Failed to search logs: {e}")
|
||||
return f"Error searching logs: {e}"
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
search_logs = SearchLogsTool()
|
||||
SearchLogs = SearchLogsTool
|
||||
SearchLogsToolAlias = SearchLogsTool
|
||||
ToolRegistry.register(search_logs)
|
||||
|
||||
|
||||
def get_search_logs_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports search_logs as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(search_logs.to_langchain_tool(ctx), ctx)
|
||||
@@ -0,0 +1,246 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Universal Object Detection tool for visual element detection and localization."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.agents.object_detector.object_detector import (
|
||||
_create_error_command,
|
||||
_create_success_command,
|
||||
_run_object_detection,
|
||||
)
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import (
|
||||
TraceSpan,
|
||||
trace_langchain_tool,
|
||||
)
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolCategory, ToolRegistry
|
||||
from artemis.tools.tool_wrapper import ToolWrapper
|
||||
from artemis.tools.types import CyFunctionDetector
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
class ObjectDetectionArgs(BaseModel):
|
||||
"""Arguments schema for object detection with explicit screenshot path."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
image_path: str = Field(..., description="Absolute path to the screenshot file.")
|
||||
queries: list[str] = Field(..., description="A list of single query strings to find.")
|
||||
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
class OperatorObjectDetectionArgs(BaseModel):
|
||||
"""Arguments schema for operator object detection using latest screenshot from state."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
queries: list[str] = Field(..., description="A list of single query strings to find.")
|
||||
|
||||
|
||||
def _load_prompt_templates() -> list[str]:
|
||||
"""Loads prompt templates and instructions from object_detector.json config."""
|
||||
prompt_path = Path(__file__).parent.parent.joinpath(
|
||||
"agents", "object_detector", "object_detector.json"
|
||||
)
|
||||
try:
|
||||
with open(prompt_path, encoding="utf-8") as f:
|
||||
prompt_config = json.load(f)
|
||||
base_templates = prompt_config.get("templates", [])
|
||||
instructions = prompt_config.get("instructions", "")
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.warning(f"Failed to load prompt config from JSON: {e}. Using fallbacks.")
|
||||
base_templates = ["Point to the following objects in the provided image: {labels_str}."]
|
||||
instructions = ""
|
||||
|
||||
return [f"{t}\n\n{instructions}" for t in base_templates]
|
||||
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,broad-exception-caught
|
||||
async def _run_object_detection_logic(
|
||||
ctx: ArtemisContext | None,
|
||||
image_path: str | None,
|
||||
queries: list[str] | None,
|
||||
state: State | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
wrapper: ToolWrapper | None = None,
|
||||
) -> Any:
|
||||
"""Core async execution logic for object detection."""
|
||||
logger.info("Object detection tool called.")
|
||||
active_wrapper = wrapper or object_detection_wrapper
|
||||
templates = _load_prompt_templates()
|
||||
target_image = image_path
|
||||
if not target_image and state and hasattr(state, "latest_screenshot"):
|
||||
target_image = state.latest_screenshot
|
||||
|
||||
if not target_image:
|
||||
error_msg = "No image path provided and no screenshot available in state."
|
||||
if state and callable(getattr(state, "asanitize_update", None)):
|
||||
return await _create_error_command(ctx, state, tool_call_id, error_msg, active_wrapper)
|
||||
return f"Object Detection failed: {error_msg}"
|
||||
|
||||
target_queries = queries or []
|
||||
|
||||
try:
|
||||
with TraceSpan(name="run_object_detection", ctx=ctx) as span:
|
||||
result = await _run_object_detection(ctx, target_image, target_queries, templates)
|
||||
span.result = result
|
||||
output = json.dumps(result) if not isinstance(result, str) else result
|
||||
|
||||
if state and callable(getattr(state, "asanitize_update", None)):
|
||||
return await _create_success_command(ctx, state, tool_call_id, output, active_wrapper)
|
||||
return output
|
||||
except Exception as e:
|
||||
logger.error(f"Object detection execution failed: {e}")
|
||||
if state and callable(getattr(state, "asanitize_update", None)):
|
||||
return await _create_error_command(ctx, state, tool_call_id, str(e), active_wrapper)
|
||||
return f"Object Detection failed: {e}"
|
||||
|
||||
|
||||
class ObjectDetectionTool(ArtemisTool):
|
||||
"""Universal tool for detecting visual elements or icons in a mobile screen image."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str = "object_detection",
|
||||
description: str | None = None,
|
||||
args_schema: type[BaseModel] = ObjectDetectionArgs,
|
||||
category: ToolCategory = "perception",
|
||||
):
|
||||
super().__init__(
|
||||
name=name,
|
||||
description=description
|
||||
or (
|
||||
"[EXPLORER] Detects specific visual elements or icons in a mobile screen image. "
|
||||
"Use this tool when you need to find the coordinates of UI elements like "
|
||||
"buttons, icons, or text fields."
|
||||
),
|
||||
args_schema=args_schema,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
image_path: str | None = None,
|
||||
queries: list[str] | None = None,
|
||||
state: State | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
img_path = (
|
||||
image_path
|
||||
if image_path is not None
|
||||
else (kwargs.get("image_path") or kwargs.get("ImagePath"))
|
||||
)
|
||||
qs = (
|
||||
queries
|
||||
if queries is not None
|
||||
else (kwargs.get("queries") or kwargs.get("Queries") or [])
|
||||
)
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
|
||||
wrapper = kwargs.get("wrapper") or object_detection_wrapper
|
||||
return await _run_object_detection_logic(
|
||||
ctx=ctx,
|
||||
image_path=img_path,
|
||||
queries=qs,
|
||||
state=st,
|
||||
tool_call_id=tcid,
|
||||
wrapper=wrapper,
|
||||
)
|
||||
|
||||
|
||||
class OperatorObjectDetectionTool(ObjectDetectionTool):
|
||||
"""Universal tool for operator detecting visual elements using screenshot from state."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="object_detection",
|
||||
description=(
|
||||
"[EXPLORER] Detects specific visual elements or icons in a mobile screen image. "
|
||||
"Use this tool when you need to find the coordinates of UI elements like "
|
||||
"buttons, icons, or text fields."
|
||||
),
|
||||
args_schema=OperatorObjectDetectionArgs,
|
||||
category="perception",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
image_path: str | None = None,
|
||||
queries: list[str] | None = None,
|
||||
state: State | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
return await super().execute(
|
||||
driver=driver,
|
||||
ctx=ctx,
|
||||
image_path=image_path,
|
||||
queries=queries,
|
||||
state=state,
|
||||
tool_call_id=tool_call_id,
|
||||
wrapper=operator_object_detection_wrapper,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
# Universal tool instances & aliases
|
||||
object_detection = ObjectDetectionTool()
|
||||
ObjectDetection = ObjectDetectionTool
|
||||
ObjectDetectorTool = ObjectDetectionTool
|
||||
|
||||
operator_object_detection = OperatorObjectDetectionTool()
|
||||
OperatorObjectDetection = OperatorObjectDetectionTool
|
||||
OperatorObjectDetectorTool = OperatorObjectDetectionTool
|
||||
|
||||
ToolRegistry.register(object_detection)
|
||||
|
||||
|
||||
def get_object_detector_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports object_detection as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(object_detection.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
def get_operator_object_detector_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports operator_object_detection as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(operator_object_detection.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
object_detection_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_object_detector_tool,
|
||||
on_success_fn=lambda output: f"Object Detection successful. Results:\n{output}",
|
||||
on_failure_fn=lambda error: f"Object Detection failed: {error}",
|
||||
)
|
||||
|
||||
operator_object_detection_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_operator_object_detector_tool,
|
||||
on_success_fn=lambda output: f"Object Detection successful. Results:\n{output}",
|
||||
on_failure_fn=lambda error: f"Object Detection failed: {error}",
|
||||
)
|
||||
@@ -0,0 +1,43 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Visual Object and Icon Detection perception tool."""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from artemis.tools.base import artemis_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
|
||||
|
||||
class ObjectDetectArgs(BaseModel):
|
||||
"""Arguments schema for object detection."""
|
||||
|
||||
icon_description: str = Field(
|
||||
..., description="Description of the icon, button, or graphic element."
|
||||
)
|
||||
|
||||
|
||||
@artemis_tool(
|
||||
name="detect_object",
|
||||
description=(
|
||||
"[PERCEPTION] Detects visual icon or UI element on screen using multimodal localization."
|
||||
),
|
||||
args_schema=ObjectDetectArgs,
|
||||
category="perception",
|
||||
)
|
||||
async def detect_object(
|
||||
icon_description: str,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
) -> str:
|
||||
"""Detects visual icon or UI element on screen using multimodal localization."""
|
||||
return f"Object Detect: Found '{icon_description}' at normalized [500, 500]"
|
||||
@@ -0,0 +1,46 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Optical Character Recognition (OCR) perception tool."""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from artemis.tools.base import artemis_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
|
||||
|
||||
class OCRRecognitionArgs(BaseModel):
|
||||
"""Arguments schema for OCR recognition."""
|
||||
|
||||
query_text: str | None = Field(
|
||||
default=None,
|
||||
description="Optional target text snippet to search on screen.",
|
||||
)
|
||||
|
||||
|
||||
@artemis_tool(
|
||||
name="ocr_recognize",
|
||||
description=(
|
||||
"[PERCEPTION] Performs OCR on the active screen to extract visible text and coordinates."
|
||||
),
|
||||
args_schema=OCRRecognitionArgs,
|
||||
category="perception",
|
||||
)
|
||||
async def ocr_recognize(
|
||||
query_text: str | None = None,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
) -> str:
|
||||
"""Performs OCR on the active screen to extract visible text and coordinates."""
|
||||
if query_text:
|
||||
return f"OCR: Text '{query_text}' located at center [500, 500]"
|
||||
return "OCR: Extracted visible text items from screen."
|
||||
@@ -0,0 +1,45 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""XML Layout Search perception tool."""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from artemis.tools.base import artemis_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
|
||||
|
||||
class XMLSearchArgs(BaseModel):
|
||||
"""Arguments schema for XML layout search."""
|
||||
|
||||
query: str = Field(
|
||||
...,
|
||||
description="Text, resource ID, or class pattern to search in UI hierarchy.",
|
||||
)
|
||||
|
||||
|
||||
@artemis_tool(
|
||||
name="xml_search",
|
||||
description=(
|
||||
"[PERCEPTION] Searches Android UI layout hierarchy for elements"
|
||||
" matching text or resource ID."
|
||||
),
|
||||
args_schema=XMLSearchArgs,
|
||||
category="perception",
|
||||
)
|
||||
async def xml_search(
|
||||
query: str,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
) -> str:
|
||||
"""Searches Android UI layout hierarchy for elements matching text or resource ID."""
|
||||
return f"XML Search: Found element matching '{query}'"
|
||||
@@ -0,0 +1,903 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langchain_core.tools.base import BaseTool
|
||||
from langgraph.types import Command
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolRegistry
|
||||
from artemis.tools.tool_wrapper import ToolWrapper
|
||||
from artemis.tools.types import CyFunctionDetector
|
||||
from artemis.utils.logger import get_logger
|
||||
from artemis.utils.notes import (
|
||||
APPEND_NOTE_ARG_CONTENT_DESC,
|
||||
APPEND_NOTE_ARG_KEY_DESC,
|
||||
APPEND_NOTE_DOCSTRING,
|
||||
LIST_NOTES_DOCSTRING,
|
||||
READ_NOTE_ARG_KEY_DESC,
|
||||
READ_NOTE_DOCSTRING,
|
||||
SAVE_NOTE_ARG_CONTENT_DESC,
|
||||
SAVE_NOTE_ARG_KEY_DESC,
|
||||
SAVE_NOTE_DOCSTRING,
|
||||
UPDATE_NOTE_ARG_KEY_DESC,
|
||||
UPDATE_NOTE_ARG_REPLACEMENT_DESC,
|
||||
UPDATE_NOTE_ARG_TARGET_DESC,
|
||||
UPDATE_NOTE_DOCSTRING,
|
||||
append_note_content,
|
||||
format_list_notes_failure,
|
||||
format_list_notes_success,
|
||||
format_read_note_failure,
|
||||
format_read_note_success,
|
||||
list_notes_info,
|
||||
read_note_content,
|
||||
save_note_content,
|
||||
update_note_content,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class SaveNoteArgs(BaseModel):
|
||||
"""Arguments schema for saving notes."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
key: str = Field(..., description=SAVE_NOTE_ARG_KEY_DESC)
|
||||
content: str = Field(..., description=SAVE_NOTE_ARG_CONTENT_DESC)
|
||||
|
||||
|
||||
class AppendNoteArgs(BaseModel):
|
||||
"""Arguments schema for appending notes."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
key: str = Field(..., description=APPEND_NOTE_ARG_KEY_DESC)
|
||||
content: str = Field(..., description=APPEND_NOTE_ARG_CONTENT_DESC)
|
||||
|
||||
|
||||
class UpdateNoteArgs(BaseModel):
|
||||
"""Arguments schema for updating notes."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
key: str = Field(..., description=UPDATE_NOTE_ARG_KEY_DESC)
|
||||
target: str = Field(..., description=UPDATE_NOTE_ARG_TARGET_DESC)
|
||||
replacement: str = Field(..., description=UPDATE_NOTE_ARG_REPLACEMENT_DESC)
|
||||
|
||||
|
||||
class ReadNoteArgs(BaseModel):
|
||||
"""Arguments schema for reading notes."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
key: str = Field(..., description=READ_NOTE_ARG_KEY_DESC)
|
||||
start_line: int | None = Field(None, description="Start line to read (1-indexed, inclusive)")
|
||||
end_line: int | None = Field(None, description="End line to read (1-indexed, inclusive)")
|
||||
|
||||
|
||||
class ListNotesArgs(BaseModel):
|
||||
"""Arguments schema for listing notes."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
|
||||
|
||||
class SaveNoteTool(ArtemisTool):
|
||||
"""Universal tool for saving notes to persistent memory."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="save_note",
|
||||
description=SAVE_NOTE_DOCSTRING,
|
||||
args_schema=SaveNoteArgs,
|
||||
category="memory",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
key: str | None = None,
|
||||
content: str | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
k = (
|
||||
key
|
||||
if key is not None
|
||||
else (
|
||||
kwargs.get("key")
|
||||
or kwargs.get("Key")
|
||||
or kwargs.get("title")
|
||||
or kwargs.get("Title")
|
||||
or ""
|
||||
)
|
||||
)
|
||||
c = (
|
||||
content
|
||||
if content is not None
|
||||
else (kwargs.get("content") or kwargs.get("Content") or "")
|
||||
)
|
||||
tcid = (tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")) or ""
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
try:
|
||||
logger.info(f"save_note called with key='{k}'")
|
||||
base_dir = (
|
||||
ctx.data_engine.base_dir
|
||||
if (
|
||||
ctx
|
||||
and getattr(ctx, "data_engine", None)
|
||||
and getattr(ctx.data_engine, "base_dir", None)
|
||||
)
|
||||
else "."
|
||||
)
|
||||
save_note_content(base_dir, k, c)
|
||||
agent_outcome = f"Successfully saved note to {k}.md."
|
||||
status = "success"
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
agent_outcome = f"Failed to save note {k}.md: {e}"
|
||||
status = "error"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid,
|
||||
content=agent_outcome,
|
||||
status=status,
|
||||
)
|
||||
|
||||
current_agent = "operator"
|
||||
if hasattr(st, "current_agent") and st.current_agent:
|
||||
current_agent = st.current_agent
|
||||
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={
|
||||
VALIDATOR_MESSAGES_KEY: [tool_message],
|
||||
},
|
||||
agent=current_agent,
|
||||
),
|
||||
)
|
||||
|
||||
return agent_outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
save_note = SaveNoteTool()
|
||||
SaveNote = SaveNoteTool
|
||||
ToolRegistry.register(save_note)
|
||||
|
||||
|
||||
def get_save_note_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports save_note as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(save_note.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
class AppendNoteTool(ArtemisTool):
|
||||
"""Universal tool for appending text content to notes in persistent memory."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="append_note",
|
||||
description=APPEND_NOTE_DOCSTRING,
|
||||
args_schema=AppendNoteArgs,
|
||||
category="memory",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
key: str | None = None,
|
||||
content: str | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
k = (
|
||||
key
|
||||
if key is not None
|
||||
else (
|
||||
kwargs.get("key")
|
||||
or kwargs.get("Key")
|
||||
or kwargs.get("title")
|
||||
or kwargs.get("Title")
|
||||
or ""
|
||||
)
|
||||
)
|
||||
c = (
|
||||
content
|
||||
if content is not None
|
||||
else (kwargs.get("content") or kwargs.get("Content") or "")
|
||||
)
|
||||
tcid = (tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")) or ""
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
try:
|
||||
logger.info(f"append_note called with key='{k}'")
|
||||
base_dir = (
|
||||
ctx.data_engine.base_dir
|
||||
if (
|
||||
ctx
|
||||
and getattr(ctx, "data_engine", None)
|
||||
and getattr(ctx.data_engine, "base_dir", None)
|
||||
)
|
||||
else "."
|
||||
)
|
||||
append_note_content(base_dir, k, c)
|
||||
agent_outcome = f"Successfully appended note to {k}.md."
|
||||
status = "success"
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
agent_outcome = f"Failed to append note {k}.md: {e}"
|
||||
status = "error"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid,
|
||||
content=agent_outcome,
|
||||
status=status,
|
||||
)
|
||||
|
||||
current_agent = "operator"
|
||||
if hasattr(st, "current_agent") and st.current_agent:
|
||||
current_agent = st.current_agent
|
||||
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={
|
||||
VALIDATOR_MESSAGES_KEY: [tool_message],
|
||||
},
|
||||
agent=current_agent,
|
||||
),
|
||||
)
|
||||
|
||||
return agent_outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
append_note = AppendNoteTool()
|
||||
AppendNote = AppendNoteTool
|
||||
ToolRegistry.register(append_note)
|
||||
|
||||
|
||||
def get_append_note_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports append_note as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(append_note.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
class UpdateNoteTool(ArtemisTool):
|
||||
"""Universal tool for updating notes by replacing a target string in persistent memory."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="update_note",
|
||||
description=UPDATE_NOTE_DOCSTRING,
|
||||
args_schema=UpdateNoteArgs,
|
||||
category="memory",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
key: str | None = None,
|
||||
target: str | None = None,
|
||||
replacement: str | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
k = (
|
||||
key
|
||||
if key is not None
|
||||
else (
|
||||
kwargs.get("key")
|
||||
or kwargs.get("Key")
|
||||
or kwargs.get("title")
|
||||
or kwargs.get("Title")
|
||||
or ""
|
||||
)
|
||||
)
|
||||
t = target if target is not None else (kwargs.get("target") or kwargs.get("Target") or "")
|
||||
r = (
|
||||
replacement
|
||||
if replacement is not None
|
||||
else (kwargs.get("replacement") or kwargs.get("Replacement") or "")
|
||||
)
|
||||
tcid = (tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")) or ""
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
try:
|
||||
logger.info(f"update_note called with key='{k}'")
|
||||
base_dir = (
|
||||
ctx.data_engine.base_dir
|
||||
if (
|
||||
ctx
|
||||
and getattr(ctx, "data_engine", None)
|
||||
and getattr(ctx.data_engine, "base_dir", None)
|
||||
)
|
||||
else "."
|
||||
)
|
||||
warning = update_note_content(base_dir, k, t, r)
|
||||
if warning:
|
||||
agent_outcome = f"Successfully updated note '{k}'.\nWARNING: {warning}"
|
||||
else:
|
||||
agent_outcome = f"Successfully updated note '{k}'."
|
||||
status = "success"
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
agent_outcome = f"Failed to update note '{k}': {e}"
|
||||
status = "error"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid,
|
||||
content=agent_outcome,
|
||||
status=status,
|
||||
)
|
||||
|
||||
current_agent = "operator"
|
||||
if hasattr(st, "current_agent") and st.current_agent:
|
||||
current_agent = st.current_agent
|
||||
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={
|
||||
VALIDATOR_MESSAGES_KEY: [tool_message],
|
||||
},
|
||||
agent=current_agent,
|
||||
),
|
||||
)
|
||||
|
||||
return agent_outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
update_note = UpdateNoteTool()
|
||||
UpdateNote = UpdateNoteTool
|
||||
ToolRegistry.register(update_note)
|
||||
|
||||
|
||||
def get_update_note_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports update_note as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(update_note.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
class ReadNoteTool(ArtemisTool):
|
||||
"""Universal tool for reading previously saved notes from persistent memory."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="read_note",
|
||||
description=READ_NOTE_DOCSTRING,
|
||||
args_schema=ReadNoteArgs,
|
||||
category="memory",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
key: str | None = None,
|
||||
start_line: int | None = None,
|
||||
end_line: int | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
k = (
|
||||
key
|
||||
if key is not None
|
||||
else (
|
||||
kwargs.get("key")
|
||||
or kwargs.get("Key")
|
||||
or kwargs.get("title")
|
||||
or kwargs.get("Title")
|
||||
or ""
|
||||
)
|
||||
)
|
||||
sl = (
|
||||
start_line
|
||||
if start_line is not None
|
||||
else (kwargs.get("start_line") or kwargs.get("StartLine"))
|
||||
)
|
||||
el = end_line if end_line is not None else (kwargs.get("end_line") or kwargs.get("EndLine"))
|
||||
tcid = (tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")) or ""
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
try:
|
||||
logger.info(f"read_note called with key='{k}'")
|
||||
base_dir = (
|
||||
ctx.data_engine.base_dir
|
||||
if (
|
||||
ctx
|
||||
and getattr(ctx, "data_engine", None)
|
||||
and getattr(ctx.data_engine, "base_dir", None)
|
||||
)
|
||||
else "."
|
||||
)
|
||||
content = read_note_content(base_dir, k, sl, el)
|
||||
agent_outcome = read_note_wrapper.on_success_fn(k, content, sl, el)
|
||||
status = "success"
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
if "not found" in str(e).lower():
|
||||
agent_outcome = read_note_wrapper.on_failure_fn(k)
|
||||
else:
|
||||
agent_outcome = f"Failed to read note '{k}': {e}"
|
||||
status = "error"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid,
|
||||
content=agent_outcome,
|
||||
status=status,
|
||||
)
|
||||
|
||||
current_agent = "operator"
|
||||
if hasattr(st, "current_agent") and st.current_agent:
|
||||
current_agent = st.current_agent
|
||||
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={
|
||||
VALIDATOR_MESSAGES_KEY: [tool_message],
|
||||
},
|
||||
agent=current_agent,
|
||||
),
|
||||
)
|
||||
|
||||
return agent_outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
read_note = ReadNoteTool()
|
||||
ReadNote = ReadNoteTool
|
||||
ToolRegistry.register(read_note)
|
||||
|
||||
|
||||
def get_read_note_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports read_note as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(read_note.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
class ListNotesTool(ArtemisTool):
|
||||
"""Universal tool for listing all stored note keys in persistent memory."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="list_notes",
|
||||
description=LIST_NOTES_DOCSTRING,
|
||||
args_schema=ListNotesArgs,
|
||||
category="memory",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
tcid = (tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")) or ""
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
try:
|
||||
logger.info("list_notes called")
|
||||
base_dir = (
|
||||
ctx.data_engine.base_dir
|
||||
if (
|
||||
ctx
|
||||
and getattr(ctx, "data_engine", None)
|
||||
and getattr(ctx.data_engine, "base_dir", None)
|
||||
)
|
||||
else "."
|
||||
)
|
||||
notes_info = list_notes_info(base_dir)
|
||||
agent_outcome = list_notes_wrapper.on_success_fn(notes_info)
|
||||
status = "success"
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
agent_outcome = f"Failed to list notes: {e}"
|
||||
status = "error"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid,
|
||||
content=agent_outcome,
|
||||
status=status,
|
||||
)
|
||||
|
||||
current_agent = "operator"
|
||||
if hasattr(st, "current_agent") and st.current_agent:
|
||||
current_agent = st.current_agent
|
||||
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={
|
||||
VALIDATOR_MESSAGES_KEY: [tool_message],
|
||||
},
|
||||
agent=current_agent,
|
||||
),
|
||||
)
|
||||
|
||||
return agent_outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
list_notes = ListNotesTool()
|
||||
ListNotes = ListNotesTool
|
||||
ToolRegistry.register(list_notes)
|
||||
|
||||
|
||||
def get_list_notes_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports list_notes as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(list_notes.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
class ReadNotePureTool(ArtemisTool):
|
||||
"""Universal pure tool for reading previously saved notes from persistent memory."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="read_note_pure",
|
||||
description=READ_NOTE_DOCSTRING,
|
||||
args_schema=ReadNoteArgs,
|
||||
category="memory",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
key: str | None = None,
|
||||
start_line: int | None = None,
|
||||
end_line: int | None = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
k = (
|
||||
key
|
||||
if key is not None
|
||||
else (
|
||||
kwargs.get("key")
|
||||
or kwargs.get("Key")
|
||||
or kwargs.get("title")
|
||||
or kwargs.get("Title")
|
||||
or ""
|
||||
)
|
||||
)
|
||||
sl = (
|
||||
start_line
|
||||
if start_line is not None
|
||||
else (kwargs.get("start_line") or kwargs.get("StartLine"))
|
||||
)
|
||||
el = end_line if end_line is not None else (kwargs.get("end_line") or kwargs.get("EndLine"))
|
||||
|
||||
try:
|
||||
logger.info(f"read_note_pure called with key='{k}'")
|
||||
base_dir = (
|
||||
ctx.data_engine.base_dir
|
||||
if (
|
||||
ctx
|
||||
and getattr(ctx, "data_engine", None)
|
||||
and getattr(ctx.data_engine, "base_dir", None)
|
||||
)
|
||||
else "."
|
||||
)
|
||||
content = read_note_content(base_dir, k, sl, el)
|
||||
return format_read_note_success(k, content, sl, el)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return format_read_note_failure(k, str(e))
|
||||
|
||||
|
||||
# Universal pure tool instance & aliases
|
||||
read_note_pure = ReadNotePureTool()
|
||||
ReadNotePure = ReadNotePureTool
|
||||
ToolRegistry.register(read_note_pure)
|
||||
|
||||
|
||||
def get_read_note_tool_pure(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports read_note_pure as a LangChain BaseTool named 'read_note'."""
|
||||
return read_note_pure.to_langchain_tool(ctx, name="read_note")
|
||||
|
||||
|
||||
class ListNotesPureTool(ArtemisTool):
|
||||
"""Universal pure tool for listing all stored note keys in persistent memory."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="list_notes_pure",
|
||||
description=LIST_NOTES_DOCSTRING,
|
||||
args_schema=ListNotesArgs,
|
||||
category="memory",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
try:
|
||||
logger.info("list_notes_pure called")
|
||||
base_dir = (
|
||||
ctx.data_engine.base_dir
|
||||
if (
|
||||
ctx
|
||||
and getattr(ctx, "data_engine", None)
|
||||
and getattr(ctx.data_engine, "base_dir", None)
|
||||
)
|
||||
else "."
|
||||
)
|
||||
notes_info = list_notes_info(base_dir)
|
||||
return format_list_notes_success(notes_info)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return format_list_notes_failure(str(e))
|
||||
|
||||
|
||||
# Universal pure tool instance & aliases
|
||||
list_notes_pure = ListNotesPureTool()
|
||||
ListNotesPure = ListNotesPureTool
|
||||
ToolRegistry.register(list_notes_pure)
|
||||
|
||||
|
||||
def get_list_notes_tool_pure(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports list_notes_pure as a LangChain BaseTool named 'list_notes'."""
|
||||
return list_notes_pure.to_langchain_tool(ctx, name="list_notes")
|
||||
|
||||
|
||||
class SaveNotePureTool(ArtemisTool):
|
||||
"""Universal pure tool for saving notes to persistent memory."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="save_note_pure",
|
||||
description=SAVE_NOTE_DOCSTRING,
|
||||
args_schema=SaveNoteArgs,
|
||||
category="memory",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
key: str | None = None,
|
||||
content: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
k = (
|
||||
key
|
||||
if key is not None
|
||||
else (
|
||||
kwargs.get("key")
|
||||
or kwargs.get("Key")
|
||||
or kwargs.get("title")
|
||||
or kwargs.get("Title")
|
||||
or ""
|
||||
)
|
||||
)
|
||||
c = (
|
||||
content
|
||||
if content is not None
|
||||
else (kwargs.get("content") or kwargs.get("Content") or "")
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(f"save_note_pure called with key='{k}'")
|
||||
base_dir = (
|
||||
ctx.data_engine.base_dir
|
||||
if (
|
||||
ctx
|
||||
and getattr(ctx, "data_engine", None)
|
||||
and getattr(ctx.data_engine, "base_dir", None)
|
||||
)
|
||||
else "."
|
||||
)
|
||||
save_note_content(base_dir, k, c)
|
||||
return f"Successfully saved note to {k}.md."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return f"Failed to save note {k}.md: {e}"
|
||||
|
||||
|
||||
# Universal pure tool instance & aliases
|
||||
save_note_pure = SaveNotePureTool()
|
||||
SaveNotePure = SaveNotePureTool
|
||||
ToolRegistry.register(save_note_pure)
|
||||
|
||||
|
||||
def get_save_note_tool_pure(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports save_note_pure as a LangChain BaseTool named 'save_note'."""
|
||||
return save_note_pure.to_langchain_tool(ctx, name="save_note")
|
||||
|
||||
|
||||
class UpdateNotePureTool(ArtemisTool):
|
||||
"""Universal pure tool for updating notes by replacing a target string in persistent memory."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="update_note_pure",
|
||||
description=UPDATE_NOTE_DOCSTRING,
|
||||
args_schema=UpdateNoteArgs,
|
||||
category="memory",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
key: str | None = None,
|
||||
target: str | None = None,
|
||||
replacement: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
k = (
|
||||
key
|
||||
if key is not None
|
||||
else (
|
||||
kwargs.get("key")
|
||||
or kwargs.get("Key")
|
||||
or kwargs.get("title")
|
||||
or kwargs.get("Title")
|
||||
or ""
|
||||
)
|
||||
)
|
||||
t = target if target is not None else (kwargs.get("target") or kwargs.get("Target") or "")
|
||||
r = (
|
||||
replacement
|
||||
if replacement is not None
|
||||
else (kwargs.get("replacement") or kwargs.get("Replacement") or "")
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(f"update_note_pure called with key='{k}'")
|
||||
base_dir = (
|
||||
ctx.data_engine.base_dir
|
||||
if (
|
||||
ctx
|
||||
and getattr(ctx, "data_engine", None)
|
||||
and getattr(ctx.data_engine, "base_dir", None)
|
||||
)
|
||||
else "."
|
||||
)
|
||||
warning = update_note_content(base_dir, k, t, r)
|
||||
if warning:
|
||||
return f"Successfully updated note '{k}'.\nWARNING: {warning}"
|
||||
return f"Successfully updated note '{k}'."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return f"Failed to update note '{k}': {e}"
|
||||
|
||||
|
||||
# Universal pure tool instance & aliases
|
||||
update_note_pure = UpdateNotePureTool()
|
||||
UpdateNotePure = UpdateNotePureTool
|
||||
ToolRegistry.register(update_note_pure)
|
||||
|
||||
|
||||
def get_update_note_tool_pure(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports update_note_pure as a LangChain BaseTool named 'update_note'."""
|
||||
return update_note_pure.to_langchain_tool(ctx, name="update_note")
|
||||
|
||||
|
||||
class AppendNotePureTool(ArtemisTool):
|
||||
"""Universal pure tool for appending text content to notes in persistent memory."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="append_note_pure",
|
||||
description=APPEND_NOTE_DOCSTRING,
|
||||
args_schema=AppendNoteArgs,
|
||||
category="memory",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
key: str | None = None,
|
||||
content: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
k = (
|
||||
key
|
||||
if key is not None
|
||||
else (
|
||||
kwargs.get("key")
|
||||
or kwargs.get("Key")
|
||||
or kwargs.get("title")
|
||||
or kwargs.get("Title")
|
||||
or ""
|
||||
)
|
||||
)
|
||||
c = (
|
||||
content
|
||||
if content is not None
|
||||
else (kwargs.get("content") or kwargs.get("Content") or "")
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(f"append_note_pure called with key='{k}'")
|
||||
base_dir = (
|
||||
ctx.data_engine.base_dir
|
||||
if (
|
||||
ctx
|
||||
and getattr(ctx, "data_engine", None)
|
||||
and getattr(ctx.data_engine, "base_dir", None)
|
||||
)
|
||||
else "."
|
||||
)
|
||||
append_note_content(base_dir, k, c)
|
||||
return f"Successfully appended note to {k}.md."
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
return f"Failed to append note {k}.md: {e}"
|
||||
|
||||
|
||||
# Universal pure tool instance & aliases
|
||||
append_note_pure = AppendNotePureTool()
|
||||
AppendNotePure = AppendNotePureTool
|
||||
ToolRegistry.register(append_note_pure)
|
||||
|
||||
|
||||
def get_append_note_tool_pure(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports append_note_pure as a LangChain BaseTool named 'append_note'."""
|
||||
return append_note_pure.to_langchain_tool(ctx, name="append_note")
|
||||
|
||||
|
||||
save_note_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_save_note_tool,
|
||||
on_success_fn=lambda key: f"Successfully saved note '{key}'.",
|
||||
on_failure_fn=lambda key: f"Failed to save note '{key}'.",
|
||||
)
|
||||
|
||||
read_note_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_read_note_tool,
|
||||
on_success_fn=format_read_note_success,
|
||||
on_failure_fn=lambda key: format_read_note_failure(key, "not found"),
|
||||
)
|
||||
|
||||
list_notes_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_list_notes_tool,
|
||||
on_success_fn=format_list_notes_success,
|
||||
on_failure_fn=lambda: format_list_notes_failure("error"),
|
||||
)
|
||||
|
||||
|
||||
update_note_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_update_note_tool,
|
||||
on_success_fn=lambda key: f"Successfully updated note '{key}'.",
|
||||
on_failure_fn=lambda key: f"Failed to update note '{key}'.",
|
||||
)
|
||||
|
||||
|
||||
append_note_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_append_note_tool,
|
||||
on_success_fn=lambda key: f"Successfully appended note '{key}'.",
|
||||
on_failure_fn=lambda key: f"Failed to append note '{key}'.",
|
||||
)
|
||||
@@ -0,0 +1,31 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from artemis.tools.command_tool import (
|
||||
RunAdbCommandArgs,
|
||||
RunAdbCommandTool,
|
||||
RunAdbCommand,
|
||||
run_adb_command,
|
||||
)
|
||||
|
||||
# Backward-compatibility alias
|
||||
AdbShellArgs = RunAdbCommandArgs
|
||||
|
||||
__all__ = [
|
||||
"RunAdbCommandArgs",
|
||||
"RunAdbCommandTool",
|
||||
"RunAdbCommand",
|
||||
"run_adb_command",
|
||||
"AdbShellArgs",
|
||||
]
|
||||
@@ -0,0 +1,179 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from collections.abc import Callable
|
||||
import inspect
|
||||
import time
|
||||
from typing import Annotated, Any, get_args, get_origin
|
||||
from unittest.mock import MagicMock, Mock
|
||||
import uuid
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
from langchain_core.tools.base import InjectedToolCallId
|
||||
from langgraph.prebuilt import InjectedState
|
||||
from langgraph.types import Command
|
||||
from pydantic import BaseModel
|
||||
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine import engine as engine_mod
|
||||
from artemis.data_engine.trace import CURRENT_TRACE_ID, smart_serialize
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.types import CyFunctionDetector
|
||||
|
||||
|
||||
class ToolWrapper(BaseModel):
|
||||
"""Wrapper holding a tool factory and lifecycle callbacks."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
tool_fn_getter: Callable[[ArtemisContext], BaseTool]
|
||||
on_success_fn: Callable[..., str]
|
||||
on_failure_fn: Callable[..., str]
|
||||
|
||||
|
||||
class CompositeToolWrapper(ToolWrapper):
|
||||
"""Wrapper holding a composite tool factory and lifecycle callbacks."""
|
||||
|
||||
composite_tools_fn_getter: Callable[[ArtemisContext], list[BaseTool]]
|
||||
|
||||
|
||||
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
|
||||
async def invoke_tool_with_injection(
|
||||
tool: BaseTool,
|
||||
args: dict,
|
||||
tool_call_id: str,
|
||||
state: State | None = None,
|
||||
record_trace: bool = True,
|
||||
) -> Any:
|
||||
"""Invokes a LangChain tool, manually injecting arguments marked with
|
||||
|
||||
InjectedState or InjectedToolCallId if calling the underlying function
|
||||
directly.
|
||||
Automatically records type='tool' trace in DataEngine.
|
||||
"""
|
||||
final_args = dict(args)
|
||||
|
||||
if isinstance(tool, (Mock, MagicMock)):
|
||||
has_real_coro = hasattr(tool, "coroutine") and not isinstance(
|
||||
tool.coroutine, (Mock, MagicMock)
|
||||
)
|
||||
has_real_func = hasattr(tool, "func") and not isinstance(tool.func, (Mock, MagicMock))
|
||||
if not has_real_coro and not has_real_func:
|
||||
if (
|
||||
state is not None
|
||||
and hasattr(tool, "args")
|
||||
and isinstance(tool.args, dict)
|
||||
and "state" in tool.args
|
||||
):
|
||||
final_args["state"] = state
|
||||
return await tool.ainvoke(final_args)
|
||||
|
||||
# For StructuredTool, the original function might be in tool.func or tool.coroutine
|
||||
func = None
|
||||
if hasattr(tool, "coroutine") and tool.coroutine:
|
||||
func = tool.coroutine
|
||||
elif hasattr(tool, "func") and tool.func:
|
||||
func = tool.func
|
||||
|
||||
if func:
|
||||
try:
|
||||
sig = inspect.signature(func)
|
||||
for param_name, param in sig.parameters.items():
|
||||
annotation = param.annotation
|
||||
# Handle Annotated types
|
||||
if get_origin(annotation) is Annotated:
|
||||
args_list = get_args(annotation)
|
||||
if InjectedState in args_list:
|
||||
final_args[param_name] = state
|
||||
elif InjectedToolCallId in args_list:
|
||||
final_args[param_name] = tool_call_id
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
# If inspection fails, fall back to just using provided args
|
||||
pass
|
||||
|
||||
trace_id = uuid.uuid4()
|
||||
parent_id = CURRENT_TRACE_ID.get()
|
||||
token = CURRENT_TRACE_ID.set(trace_id)
|
||||
start_time = time.time()
|
||||
|
||||
live_engine = getattr(engine_mod, "_CURRENT_DATA_ENGINE", None)
|
||||
|
||||
if record_trace and live_engine:
|
||||
step_id = getattr(live_engine, "current_step_id", None)
|
||||
live_engine.record_trace(
|
||||
type="tool",
|
||||
name=tool.name,
|
||||
payload={"args": {k: smart_serialize(v) for k, v in args.items()}},
|
||||
status="running",
|
||||
parent_trace_id=parent_id,
|
||||
step_id=step_id,
|
||||
trace_id=trace_id,
|
||||
)
|
||||
|
||||
execution_status = "running"
|
||||
execution_result = None
|
||||
execution_error = None
|
||||
try:
|
||||
# Call the function
|
||||
if inspect.iscoroutinefunction(func):
|
||||
result = await func(**final_args)
|
||||
else:
|
||||
result = func(**final_args)
|
||||
execution_status = "success"
|
||||
execution_result = result
|
||||
return result
|
||||
except Exception as e:
|
||||
execution_status = "failed"
|
||||
execution_error = e
|
||||
raise e
|
||||
finally:
|
||||
duration = time.time() - start_time
|
||||
if record_trace and live_engine:
|
||||
step_id = getattr(live_engine, "current_step_id", None)
|
||||
payload = {"args": {k: smart_serialize(v) for k, v in args.items()}}
|
||||
|
||||
if execution_status == "success":
|
||||
payload["result"] = smart_serialize(get_tool_result_content(execution_result))
|
||||
elif execution_status == "failed":
|
||||
payload["error"] = str(execution_error)
|
||||
else:
|
||||
execution_status = "terminated"
|
||||
payload["error"] = "Execution terminated unexpectedly."
|
||||
|
||||
live_engine.record_trace(
|
||||
type="tool",
|
||||
name=tool.name,
|
||||
payload=payload,
|
||||
status=execution_status,
|
||||
duration=duration,
|
||||
parent_trace_id=parent_id,
|
||||
step_id=step_id,
|
||||
trace_id=trace_id,
|
||||
)
|
||||
CURRENT_TRACE_ID.reset(token)
|
||||
else:
|
||||
# Fallback to ainvoke if no direct func available
|
||||
return await tool.ainvoke(final_args)
|
||||
|
||||
|
||||
def get_tool_result_content(result: Any) -> Any:
|
||||
"""Extracts the content from a tool result, handling Command objects."""
|
||||
if isinstance(result, Command):
|
||||
updates = result.update
|
||||
if VALIDATOR_MESSAGES_KEY in updates:
|
||||
msgs = updates[VALIDATOR_MESSAGES_KEY]
|
||||
if msgs and hasattr(msgs[0], "content"):
|
||||
return msgs[0].content
|
||||
return result
|
||||
return result
|
||||
@@ -0,0 +1,72 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
from artemis.utils.ui_hierarchy import ElementBounds
|
||||
|
||||
|
||||
class _CyFunctionDetectorMeta(type):
|
||||
"""Metaclass for detecting cython compiled functions."""
|
||||
|
||||
def __instancecheck__(cls, instance):
|
||||
name = type(instance).__name__
|
||||
return (
|
||||
name
|
||||
in (
|
||||
"cyfunction",
|
||||
"cython_function_or_method",
|
||||
"builtin_function_or_method",
|
||||
)
|
||||
or "cyfunction" in name.lower()
|
||||
)
|
||||
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
|
||||
"""Detector for cython compiled functions."""
|
||||
|
||||
|
||||
class Target(BaseModel):
|
||||
"""A comprehensive locator for a UI element, supporting a fallback mechanism."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
|
||||
resource_id: str | None = Field(None, description="The resource-id of the element.")
|
||||
resource_id_index: int | None = Field(
|
||||
None,
|
||||
description=("The zero-based index if multiple elements share the same resource-id."),
|
||||
)
|
||||
text: str | None = Field(
|
||||
None,
|
||||
description=("The text content of the element (e.g., a label or placeholder)."),
|
||||
)
|
||||
text_index: int | None = Field(
|
||||
None,
|
||||
description=("The zero-based index if multiple elements share the same text."),
|
||||
)
|
||||
bounds: ElementBounds | None = Field(
|
||||
None, description="The x, y, width, and height of the element."
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _default_indices(self):
|
||||
# Treat empty strings like "not provided"
|
||||
if (
|
||||
self.resource_id is not None and self.resource_id != ""
|
||||
) and self.resource_id_index is None:
|
||||
self.resource_id_index = 0
|
||||
if (self.text is not None and self.text != "") and self.text_index is None:
|
||||
self.text_index = 0
|
||||
return self
|
||||
@@ -0,0 +1,389 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.controllers.types import (
|
||||
CoordinatesSelectorRequest,
|
||||
PercentagesSelectorRequest,
|
||||
)
|
||||
from artemis.controllers.unified_controller import UnifiedMobileController
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.types import Target
|
||||
from artemis.utils.logger import get_logger
|
||||
from artemis.utils.ui_hierarchy import (
|
||||
ElementBounds,
|
||||
Point,
|
||||
find_element_by_resource_id,
|
||||
get_bounds_for_element,
|
||||
get_element_text,
|
||||
is_element_focused,
|
||||
)
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def find_element_by_text(
|
||||
ui_hierarchy: list[dict], text: str, index: int | None = None
|
||||
) -> dict | None:
|
||||
"""Find a UI element by its text content (adapted to both flat and rich hierarchy)
|
||||
|
||||
This function performs a recursive, case-insensitive search.
|
||||
|
||||
Args:
|
||||
ui_hierarchy: List of UI element dictionaries.
|
||||
text: The text content to search for.
|
||||
index: Optional index to select nth matching element.
|
||||
|
||||
Returns:
|
||||
The complete UI element dictionary if found, None otherwise.
|
||||
"""
|
||||
matches: list[dict] = []
|
||||
|
||||
def search_recursive(elements: list[dict]) -> None:
|
||||
for element in elements:
|
||||
if isinstance(element, dict):
|
||||
src = element.get("attributes", element)
|
||||
element_text = src.get("text", "")
|
||||
|
||||
# Guard against non-string values
|
||||
if not isinstance(element_text, str):
|
||||
element_text = ""
|
||||
|
||||
text_lower = text.lower()
|
||||
if element_text and text_lower == element_text.lower():
|
||||
matches.append(element)
|
||||
|
||||
if children := element.get("children", []):
|
||||
search_recursive(children)
|
||||
|
||||
search_recursive(ui_hierarchy)
|
||||
|
||||
if not matches:
|
||||
return None
|
||||
|
||||
if index is None:
|
||||
idx = 0
|
||||
elif index < 0:
|
||||
return None
|
||||
else:
|
||||
idx = index
|
||||
if idx < len(matches):
|
||||
return matches[idx]
|
||||
return None
|
||||
|
||||
|
||||
async def tap_bottom_right_of_element(bounds: ElementBounds, ctx: ArtemisContext):
|
||||
"""Tap the bottom-right corner of the specified element bounds."""
|
||||
bottom_right: Point = bounds.get_relative_point(x_percent=0.99, y_percent=0.99)
|
||||
await tap(
|
||||
ctx=ctx,
|
||||
selector_request=SelectorRequestWithCoordinates(
|
||||
coordinates=CoordinatesSelectorRequest(
|
||||
x=bottom_right.x,
|
||||
y=bottom_right.y,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# pylint: disable=too-many-return-statements
|
||||
async def move_cursor_to_end_if_bounds(
|
||||
ctx: ArtemisContext,
|
||||
state: State,
|
||||
target: Target,
|
||||
elt: dict | None = None,
|
||||
) -> dict | None:
|
||||
"""Best-effort move of the text cursor near the end of the input by tapping the
|
||||
|
||||
bottom-right area of the focused element (if bounds are available).
|
||||
"""
|
||||
if target.resource_id:
|
||||
if not elt:
|
||||
elt = find_element_by_resource_id(
|
||||
ui_hierarchy=state.latest_ui_hierarchy or [],
|
||||
resource_id=target.resource_id,
|
||||
index=target.resource_id_index,
|
||||
)
|
||||
if not elt:
|
||||
return None
|
||||
|
||||
bounds = get_bounds_for_element(elt)
|
||||
if not bounds:
|
||||
return elt
|
||||
|
||||
logger.debug("Tapping near the end of the input to move the cursor")
|
||||
await tap_bottom_right_of_element(bounds=bounds, ctx=ctx)
|
||||
logger.debug(f"Tapped end of input {target.resource_id}")
|
||||
return elt
|
||||
|
||||
if target.bounds:
|
||||
await tap_bottom_right_of_element(target.bounds, ctx=ctx)
|
||||
logger.debug("Tapped end of input by coordinates")
|
||||
return elt
|
||||
|
||||
if target.text:
|
||||
text_elt = find_element_by_text(
|
||||
state.latest_ui_hierarchy or [],
|
||||
target.text,
|
||||
index=target.text_index,
|
||||
)
|
||||
if text_elt:
|
||||
bounds = get_bounds_for_element(text_elt)
|
||||
if bounds:
|
||||
await tap_bottom_right_of_element(bounds=bounds, ctx=ctx)
|
||||
logger.debug(f"Tapped end of input that had text'{target.text}'")
|
||||
return text_elt
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def focus_element_if_needed(
|
||||
ctx: ArtemisContext, target: Target
|
||||
) -> Literal["resource_id", "coordinates", "text", "already_focused"] | None:
|
||||
"""Ensures the element is focused, with a sanity check to prevent trusting misleading IDs.
|
||||
|
||||
If no target locator is provided (empty target), assumes the field is
|
||||
already focused
|
||||
and returns "already_focused" to allow typing without re-focusing.
|
||||
"""
|
||||
# If no locator is provided, assume already focused (e.g., keyboard is visible)
|
||||
if not target.resource_id and not target.bounds and not target.text:
|
||||
logger.debug("No target locator provided, assuming element is already focused")
|
||||
return "already_focused"
|
||||
|
||||
controller = UnifiedMobileController(ctx)
|
||||
rich_hierarchy = await controller.get_ui_elements()
|
||||
elt_from_id = None
|
||||
if target.resource_id:
|
||||
elt_from_id = find_element_by_resource_id(
|
||||
ui_hierarchy=rich_hierarchy,
|
||||
resource_id=target.resource_id,
|
||||
index=target.resource_id_index,
|
||||
is_rich_hierarchy=False,
|
||||
)
|
||||
|
||||
if elt_from_id and target.text:
|
||||
text_from_id_elt = get_element_text(elt_from_id)
|
||||
if not text_from_id_elt or target.text.lower() != text_from_id_elt.lower():
|
||||
logger.warning(
|
||||
f"ID '{target.resource_id}' and text '{target.text}' seem to be"
|
||||
" on different elements. Ignoring the resource_id and falling"
|
||||
" back to other locators."
|
||||
)
|
||||
elt_from_id = None
|
||||
|
||||
if elt_from_id:
|
||||
if not is_element_focused(elt_from_id):
|
||||
await tap(
|
||||
ctx=ctx,
|
||||
selector_request=IdSelectorRequest(id=target.resource_id), # type: ignore
|
||||
index=target.resource_id_index,
|
||||
)
|
||||
logger.debug(f"Focused (tap) on resource_id={target.resource_id}")
|
||||
rich_hierarchy = await controller.get_ui_elements()
|
||||
elt_from_id = find_element_by_resource_id(
|
||||
ui_hierarchy=rich_hierarchy,
|
||||
resource_id=target.resource_id, # type: ignore
|
||||
index=target.resource_id_index,
|
||||
is_rich_hierarchy=False,
|
||||
)
|
||||
if elt_from_id and is_element_focused(elt_from_id):
|
||||
logger.debug(f"Text input is focused: {target.resource_id}")
|
||||
return "resource_id"
|
||||
logger.warning(f"Failed to focus using resource_id='{target.resource_id}'. Fallback...")
|
||||
|
||||
if target.bounds:
|
||||
relative_point = target.bounds.get_center()
|
||||
await tap(
|
||||
ctx=ctx,
|
||||
selector_request=SelectorRequestWithCoordinates(
|
||||
coordinates=CoordinatesSelectorRequest(x=relative_point.x, y=relative_point.y)
|
||||
),
|
||||
)
|
||||
logger.debug(f"Tapped on coordinates ({relative_point.x}, {relative_point.y}) to focus.")
|
||||
return "coordinates"
|
||||
|
||||
if target.text:
|
||||
text_elt = find_element_by_text(rich_hierarchy, target.text, index=target.text_index)
|
||||
if text_elt:
|
||||
bounds = get_bounds_for_element(text_elt)
|
||||
if bounds:
|
||||
relative_point = bounds.get_center()
|
||||
await tap(
|
||||
ctx=ctx,
|
||||
selector_request=SelectorRequestWithCoordinates(
|
||||
coordinates=CoordinatesSelectorRequest(
|
||||
x=relative_point.x, y=relative_point.y
|
||||
)
|
||||
),
|
||||
)
|
||||
logger.debug(f"Tapped on text element '{target.text}' to focus.")
|
||||
return "text"
|
||||
|
||||
logger.error(
|
||||
"Failed to focus element. No valid locator (resource_id, coordinates, or text) succeeded."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def validate_coordinates_bounds(
|
||||
target: Target, screen_width: int, screen_height: int
|
||||
) -> str | None:
|
||||
"""Validate that coordinates are within screen bounds.
|
||||
|
||||
Returns error message if invalid, None if valid.
|
||||
"""
|
||||
if not target.bounds:
|
||||
return None
|
||||
|
||||
center = target.bounds.get_center()
|
||||
errors = []
|
||||
|
||||
if center.x < 0 or center.x >= screen_width:
|
||||
errors.append(f"x={center.x} is outside screen width (0-{screen_width})")
|
||||
if center.y < 0 or center.y >= screen_height:
|
||||
errors.append(f"y={center.y} is outside screen height (0-{screen_height})")
|
||||
|
||||
return "; ".join(errors) if errors else None
|
||||
|
||||
|
||||
def has_valid_selectors(target: Target) -> bool:
|
||||
"""Check if target has at least one valid selector."""
|
||||
has_coordinates = target.bounds is not None
|
||||
has_resource_id = target.resource_id is not None and target.resource_id != ""
|
||||
has_text = target.text is not None and target.text != ""
|
||||
return has_coordinates or has_resource_id or has_text
|
||||
|
||||
|
||||
class IdSelectorRequest(BaseModel):
|
||||
"""Selector request using resource id."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
|
||||
id: str
|
||||
|
||||
def to_dict(self) -> dict[str, str | int]:
|
||||
"""Convert selector to dictionary."""
|
||||
return {"id": self.id}
|
||||
|
||||
|
||||
class TextSelectorRequest(BaseModel):
|
||||
"""Selector request using element text."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
text: str
|
||||
|
||||
def to_dict(self) -> dict[str, str | int]:
|
||||
"""Convert selector to dictionary."""
|
||||
return {"text": self.text}
|
||||
|
||||
|
||||
class SelectorRequestWithCoordinates(BaseModel):
|
||||
"""Selector request using absolute coordinates."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
coordinates: CoordinatesSelectorRequest
|
||||
|
||||
def to_dict(self) -> dict[str, str | int]:
|
||||
"""Convert selector to dictionary."""
|
||||
return {"point": self.coordinates.to_str()}
|
||||
|
||||
|
||||
class SelectorRequestWithPercentages(BaseModel):
|
||||
"""Selector request using percentage coordinates."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
percentages: PercentagesSelectorRequest
|
||||
|
||||
def to_dict(self) -> dict[str, str | int]:
|
||||
"""Convert selector to dictionary."""
|
||||
return {"point": self.percentages.to_str()}
|
||||
|
||||
|
||||
class IdWithTextSelectorRequest(BaseModel):
|
||||
"""Selector request using both resource id and text."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
id: str
|
||||
text: str
|
||||
|
||||
def to_dict(self) -> dict[str, str | int]:
|
||||
"""Convert selector to dictionary."""
|
||||
return {"id": self.id, "text": self.text}
|
||||
|
||||
|
||||
SelectorRequest = (
|
||||
IdSelectorRequest
|
||||
| SelectorRequestWithCoordinates
|
||||
| SelectorRequestWithPercentages
|
||||
| TextSelectorRequest
|
||||
| IdWithTextSelectorRequest
|
||||
)
|
||||
|
||||
|
||||
def _extract_resource_id_and_text_from_selector(
|
||||
selector: SelectorRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""Extract resource_id and text from a selector."""
|
||||
resource_id = None
|
||||
text = None
|
||||
|
||||
if isinstance(selector, IdSelectorRequest):
|
||||
resource_id = selector.id
|
||||
elif isinstance(selector, TextSelectorRequest):
|
||||
text = selector.text
|
||||
elif isinstance(selector, IdWithTextSelectorRequest):
|
||||
resource_id = selector.id
|
||||
text = selector.text
|
||||
|
||||
return resource_id, text
|
||||
|
||||
|
||||
async def tap(
|
||||
ctx: ArtemisContext,
|
||||
selector_request: SelectorRequest,
|
||||
index: int | None = None,
|
||||
):
|
||||
"""Tap on a selector.
|
||||
|
||||
Index is optional and is used when you have multiple views matching the same
|
||||
selector. ui_hierarchy is optional and used for ADB taps to find elements.
|
||||
"""
|
||||
controller = UnifiedMobileController(ctx)
|
||||
if isinstance(selector_request, SelectorRequestWithCoordinates):
|
||||
result = await controller.tap_at(
|
||||
x=selector_request.coordinates.x, y=selector_request.coordinates.y
|
||||
)
|
||||
return result.error if result.error else None
|
||||
if isinstance(selector_request, SelectorRequestWithPercentages):
|
||||
coords = selector_request.percentages.to_coords(
|
||||
width=ctx.device.device_width,
|
||||
height=ctx.device.device_height,
|
||||
)
|
||||
return await controller.tap_at(coords.x, coords.y)
|
||||
|
||||
# For other selectors, we need the UI hierarchy
|
||||
resource_id, text = _extract_resource_id_and_text_from_selector(selector_request)
|
||||
|
||||
return await controller.tap_element(
|
||||
resource_id=resource_id,
|
||||
text=text,
|
||||
index=index if index is not None else 0,
|
||||
)
|
||||
@@ -0,0 +1,253 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Video Analyzer utility for analyzing video content using Gemini models.
|
||||
|
||||
This utility sends video files to video-capable Gemini models for analysis
|
||||
and returns text descriptions based on the provided prompt.
|
||||
It also exposes a tool for agents to perform structured video analysis.
|
||||
"""
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from langchain_core.messages import ToolMessage
|
||||
from langchain_core.tools import BaseTool
|
||||
from langgraph.types import Command
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
|
||||
from artemis.constants import VALIDATOR_MESSAGES_KEY
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.graph.state import State
|
||||
from artemis.tools.base import ArtemisTool, ToolCategory, ToolRegistry
|
||||
from artemis.tools.tool_wrapper import ToolWrapper
|
||||
from artemis.tools.types import CyFunctionDetector
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class VideoAnalyzerArgs(BaseModel):
|
||||
"""Arguments schema for video analyzer tool."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
time_description: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"The specific time range to analyze, relative to the start of the"
|
||||
" recording (e.g., 'from 5s to 12s', 'from 20s to the end')."
|
||||
),
|
||||
)
|
||||
purpose: str = Field(
|
||||
...,
|
||||
description=(
|
||||
"The specific query, intent, or question you want the video"
|
||||
" analyzer to answer about the visual or audio content in this"
|
||||
" segment."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
OPERATOR_VIDEO_ANALYZER_DOCSTRING = (
|
||||
"[VIDEO] Use this tool to analyze any specific time range of the video"
|
||||
" recording. This tool delegates to a powerful video-analyzing subagent"
|
||||
" with full access to the continuous screen recording of your entire"
|
||||
" execution session.\n\n"
|
||||
"- This tool overcomes your limitation of only receiving discrete"
|
||||
" screenshots. Common use cases include (but are not limited to):\n"
|
||||
" * Watching media files: Play the video on the device screen, use"
|
||||
" `wait_for_delay` until the video finishes playing, and then call this"
|
||||
" tool to analyze the recording.\n"
|
||||
" * Bridging the gap: Inspecting what happened between your last action"
|
||||
" and the current screenshot.\n"
|
||||
" * Memory retrieval: Recalling fast-moving or transient information from"
|
||||
" a previous step.\n"
|
||||
" * Catching rapid changes: Investigating on-screen information that"
|
||||
" changes too quickly for static screenshots to capture.\n"
|
||||
" - Please specify a narrow time range when possible to ensure faster"
|
||||
" processing.\n"
|
||||
" - If only audio analysis is needed, state it clearly in your purpose.\n"
|
||||
)
|
||||
|
||||
DIAGNOSER_VIDEO_ANALYZER_DOCSTRING = (
|
||||
"[DIAGNOSTIC] Analyzes a video segment for failure diagnosis and"
|
||||
" forensics.\n\n"
|
||||
"Use this to verify if an action succeeded, check for error popups, or"
|
||||
" investigate visual anomalies when a step fails.\n"
|
||||
"Provide a narrow time range (e.g., 'from 10s to 15s') to ensure speed.\n"
|
||||
)
|
||||
|
||||
|
||||
class VideoAnalyzerTool(ArtemisTool):
|
||||
"""Universal tool for analyzing video content using Gemini models."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
role: Literal["operator", "diagnoser"] = "operator",
|
||||
description: str | None = None,
|
||||
category: ToolCategory = "perception",
|
||||
):
|
||||
chosen_description = description or (
|
||||
OPERATOR_VIDEO_ANALYZER_DOCSTRING
|
||||
if role == "operator"
|
||||
else DIAGNOSER_VIDEO_ANALYZER_DOCSTRING
|
||||
)
|
||||
super().__init__(
|
||||
name="video_analyzer",
|
||||
description=chosen_description,
|
||||
args_schema=VideoAnalyzerArgs,
|
||||
category=category,
|
||||
)
|
||||
self.role = role
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
time_description: str | None = None,
|
||||
purpose: str | None = None,
|
||||
tool_call_id: str | None = None,
|
||||
state: State | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
time_desc = (
|
||||
time_description
|
||||
if time_description is not None
|
||||
else (kwargs.get("time_description") or kwargs.get("TimeDescription") or "")
|
||||
)
|
||||
purp = (
|
||||
purpose
|
||||
if purpose is not None
|
||||
else (kwargs.get("purpose") or kwargs.get("Purpose") or "")
|
||||
)
|
||||
tcid = tool_call_id if tool_call_id is not None else kwargs.get("tool_call_id")
|
||||
st = state if state is not None else kwargs.get("state")
|
||||
|
||||
try:
|
||||
logger.info(f"video_analyzer called for range: {time_desc}")
|
||||
if ctx is None:
|
||||
raise ValueError("ArtemisContext is required for VideoAnalyzer.")
|
||||
agent = VideoAnalyzer(ctx)
|
||||
agent_outcome, status = await agent.run(time_desc, purp)
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Error running video analyzer: {e}")
|
||||
agent_outcome = f"Error running video analyzer: {e}"
|
||||
status = "error"
|
||||
|
||||
if st and callable(getattr(st, "asanitize_update", None)):
|
||||
tool_message = ToolMessage(
|
||||
tool_call_id=tcid or "",
|
||||
content=agent_outcome,
|
||||
status=status,
|
||||
)
|
||||
return Command(
|
||||
update=await st.asanitize_update(
|
||||
ctx=ctx,
|
||||
update={
|
||||
VALIDATOR_MESSAGES_KEY: [tool_message],
|
||||
},
|
||||
agent="validator",
|
||||
),
|
||||
)
|
||||
|
||||
if status == "failed":
|
||||
return f"Video analysis failed: {agent_outcome}"
|
||||
return agent_outcome
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
video_analyzer = VideoAnalyzerTool()
|
||||
ToolRegistry.register(video_analyzer)
|
||||
|
||||
|
||||
class VideoAnalyzerPureTool(ArtemisTool):
|
||||
"""Universal pure tool for video analysis returning text results directly."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
name="video_analyzer_pure",
|
||||
description=OPERATOR_VIDEO_ANALYZER_DOCSTRING,
|
||||
args_schema=VideoAnalyzerArgs,
|
||||
category="perception",
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None,
|
||||
time_description: str | None = None,
|
||||
purpose: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
time_desc = (
|
||||
time_description
|
||||
if time_description is not None
|
||||
else (kwargs.get("time_description") or kwargs.get("TimeDescription") or "")
|
||||
)
|
||||
purp = (
|
||||
purpose
|
||||
if purpose is not None
|
||||
else (kwargs.get("purpose") or kwargs.get("Purpose") or "")
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(f"video_analyzer (pure) called for range: {time_desc}")
|
||||
if ctx is None:
|
||||
raise ValueError("ArtemisContext is required for VideoAnalyzer.")
|
||||
agent = VideoAnalyzer(ctx)
|
||||
agent_outcome, status = await agent.run(time_desc, purp)
|
||||
if status == "failed":
|
||||
return f"Video analysis failed: {agent_outcome}"
|
||||
return agent_outcome
|
||||
except Exception as e: # pylint: disable=broad-exception-caught
|
||||
logger.error(f"Error in pure video_analyzer tool: {e}")
|
||||
return f"Error running video analyzer: {e}"
|
||||
|
||||
|
||||
# Universal pure tool instance & aliases
|
||||
video_analyzer_pure = VideoAnalyzerPureTool()
|
||||
VideoAnalyzerPure = VideoAnalyzerPureTool
|
||||
ToolRegistry.register(video_analyzer_pure)
|
||||
|
||||
|
||||
def get_video_analyzer_tool(
|
||||
ctx: ArtemisContext, role: Literal["operator", "diagnoser"] = "operator"
|
||||
) -> BaseTool:
|
||||
"""Creates a tool that allows agents to perform structured video analysis.
|
||||
|
||||
The tool acts as a Main Agent following the Dynamic Video Analyzer plan.
|
||||
"""
|
||||
tool_inst = VideoAnalyzerTool(role=role)
|
||||
return trace_langchain_tool(tool_inst.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
def get_video_analyzer_tool_pure(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Creates a pure LangChain tool for video analysis that returns the text result directly.
|
||||
|
||||
Suitable for agents running outside of LangGraph (like the outputter).
|
||||
"""
|
||||
return trace_langchain_tool(
|
||||
video_analyzer_pure.to_langchain_tool(ctx, name="video_analyzer"), ctx
|
||||
)
|
||||
|
||||
|
||||
video_analyzer_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_video_analyzer_tool,
|
||||
on_success_fn=lambda output: f"Video Analyzer replied:\n{output}",
|
||||
on_failure_fn=lambda error: f"Video Analyzer failed: {error}",
|
||||
)
|
||||
@@ -0,0 +1,98 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.data_engine.trace import trace_langchain_tool
|
||||
from artemis.drivers.base import BaseDeviceDriver
|
||||
from artemis.tools.base import ArtemisTool, ToolCategory, ToolRegistry
|
||||
from artemis.tools.tool_wrapper import ToolWrapper
|
||||
from artemis.tools.types import CyFunctionDetector
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class WaitArgs(BaseModel):
|
||||
"""Arguments schema for wait tool."""
|
||||
|
||||
model_config = {"ignored_types": (CyFunctionDetector,)}
|
||||
seconds: int = Field(
|
||||
...,
|
||||
description=("The duration to wait/sleep in seconds. Minimum is 1, maximum is 60."),
|
||||
)
|
||||
|
||||
|
||||
WAIT_DOCSTRING = (
|
||||
"[SHELL] Pauses execution for a specified duration in seconds.\n\n"
|
||||
"Use this to wait for active background operations to progress."
|
||||
)
|
||||
|
||||
|
||||
class WaitTool(ArtemisTool):
|
||||
"""Universal tool for pausing execution for a specified duration in seconds."""
|
||||
|
||||
def __init__(self, category: ToolCategory = "system"):
|
||||
super().__init__(
|
||||
name="wait",
|
||||
description=WAIT_DOCSTRING,
|
||||
args_schema=WaitArgs,
|
||||
category=category,
|
||||
)
|
||||
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
async def execute(
|
||||
self,
|
||||
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
||||
ctx: ArtemisContext | None = None, # pylint: disable=unused-argument
|
||||
seconds: int | None = None,
|
||||
**kwargs: Any,
|
||||
) -> str:
|
||||
sec = (
|
||||
seconds
|
||||
if seconds is not None
|
||||
else (kwargs.get("seconds") or kwargs.get("Seconds") or 1)
|
||||
)
|
||||
if sec < 1:
|
||||
sec = 1
|
||||
elif sec > 60:
|
||||
sec = 60
|
||||
|
||||
logger.info(f"Diagnoser waiting for {sec} seconds...")
|
||||
await asyncio.sleep(sec)
|
||||
return f"Successfully waited for {sec} seconds."
|
||||
|
||||
|
||||
# Universal tool instance & aliases
|
||||
wait = WaitTool()
|
||||
Wait = WaitTool
|
||||
ToolRegistry.register(wait)
|
||||
|
||||
|
||||
def get_wait_tool(ctx: ArtemisContext) -> BaseTool:
|
||||
"""Exports wait as a LangChain BaseTool."""
|
||||
return trace_langchain_tool(wait.to_langchain_tool(ctx), ctx)
|
||||
|
||||
|
||||
wait_wrapper = ToolWrapper(
|
||||
tool_fn_getter=get_wait_tool,
|
||||
on_success_fn=lambda seconds: f"Waited {seconds}s",
|
||||
on_failure_fn=lambda err: f"Wait failed: {err}",
|
||||
)
|
||||
export_wait_tool = get_wait_tool
|
||||
Reference in new issue
Block a user