test: add comprehensive unit, integration, and e2e test suites

Introduce complete test coverage for the ARTEMIS platform across multiple layers:
Unit tests: Add tests for agent nodes (Planner, Operator, Validator, Diagnoser, FlashRunner, Explorer), tools, utils (OCR/XML fusion, UI hierarchy), and data engine.
Integration tests: Add live tool verification, reactive runner, IME, and standalone workflow tests.
E2E tests: Add end-to-end scenario tests for media playback and failure recovery flows.
Fixtures & mocks: Add mock inputs (audio, video, screenshot) and test utilities.
This commit is contained in:
chiwanko11 committed 2026-08-13 12:18:32 -07:00
1 parent c1803fcb86
commit 5d2283deaa
115 files changed
+22813

No files matched your search

+31
View File
@@ -0,0 +1,31 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Global pytest fixtures for ARTEMIS test suite."""
import pytest
from artemis.core.context import ExecutionContext
from artemis.drivers.mock.mock_driver import MockDeviceDriver
@pytest.fixture
def mock_driver():
"""Provides an isolated MockDeviceDriver instance."""
return MockDeviceDriver(device_id="fixture-mock-device", width=1080, height=2400)
@pytest.fixture
def test_context():
"""Provides a fresh ExecutionContext."""
return ExecutionContext(task_goal="Test sample task objective", device_id="fixture-mock-device")
+326
View File
@@ -0,0 +1,326 @@
#!/usr/bin/env python3
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import re
import subprocess
import sys
import time
import xml.etree.ElementTree as ET
def run_cmd(cmd):
"""Runs a local shell command and returns stdout, stderr."""
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
return result.stdout.strip(), result.stderr.strip()
def adb_cmd(cmd):
"""Runs an adb command."""
return run_cmd(f"adb {cmd}")
def dump_ui():
"""Dumps the UI hierarchy and returns it as an XML string."""
# Try to dump to a specified path
out, err = adb_cmd("shell uiautomator dump /data/local/tmp/uidump.xml")
path = "/data/local/tmp/uidump.xml"
if "dumped to" not in out and "dumped to" not in err:
# Try default dump
out, err = adb_cmd("shell uiautomator dump")
match = re.search(r"dumped to: (.*)", out + "\n" + err)
if match:
path = match.group(1).strip()
else:
# Fallback common path
path = "/sdcard/window_dump.xml"
local_path = "temp_uidump.xml"
# Ensure we remove local temp file if it exists
if os.path.exists(local_path):
os.remove(local_path)
out, err = adb_cmd(f"pull {path} {local_path}")
if os.path.exists(local_path):
with open(local_path, encoding="utf-8") as f:
content = f.read()
os.remove(local_path)
return content
else:
print(f"Failed to pull UI dump from {path}. Out: {out}, Err: {err}")
return None
def parse_bounds(bounds_str):
"""Parses bounds string '[left,top][right,bottom]' into [left, top, right, bottom]."""
m = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str)
if m:
return [int(x) for x in m.groups()]
return None
def get_center(bounds):
"""Returns the center (x, y) of the bounds."""
return (bounds[0] + bounds[2]) // 2, (bounds[1] + bounds[3]) // 2
def find_nodes(xml_str, filter_fn):
"""Finds all nodes in the XML that match the filter function."""
try:
root = ET.fromstring(xml_str)
except Exception as e:
print(f"Failed to parse XML: {e}")
return []
matched = []
for node in root.iter("node"):
attrib = node.attrib
if filter_fn(attrib):
matched.append(attrib)
return matched
def tap_element(element, description="element"):
bounds_str = element.get("bounds")
if bounds_str:
bounds = parse_bounds(bounds_str)
if bounds:
x, y = get_center(bounds)
print(f"Tapping {description} at [{x}, {y}]")
adb_cmd(f"shell input tap {x} {y}")
return True
return False
def bypass_chrome_welcome(timeout=15):
"""Bypasses Chrome welcome/sync screens if they appear."""
print("Checking for Chrome welcome/sync screens...")
start_time = time.time()
while time.time() - start_time < timeout:
xml = dump_ui()
if not xml:
time.sleep(1)
continue
# Check for "Accept & continue"
accept_nodes = find_nodes(
xml,
lambda a: (
a.get("text") == "Accept & continue" or "terms_accept" in a.get("resource-id", "")
),
)
if accept_nodes:
print("Found 'Accept & continue' screen.")
tap_element(accept_nodes[0], "'Accept & continue' button")
time.sleep(2)
continue
# Check for "No thanks" or "Don't turn on" (Sync promo)
no_thanks_nodes = find_nodes(
xml,
lambda a: (
a.get("text") in ["No thanks", "Don't turn on"]
or "negative_button" in a.get("resource-id", "")
),
)
if no_thanks_nodes:
print("Found Sync/Promo screen.")
tap_element(no_thanks_nodes[0], "'No thanks' button")
time.sleep(2)
continue
# Check if we are already on a webpage or home page (e.g., search bar is visible, or youtube content)
# If we don't see welcome screens anymore, we might be good.
# Let's check if we see "Search or type URL" or similar Chrome home elements, or YouTube.
youtube_nodes = find_nodes(
xml,
lambda a: (
"youtube" in a.get("text", "").lower()
or "youtube" in a.get("content-desc", "").lower()
),
)
chrome_home = find_nodes(
xml,
lambda a: "search or type web address" in a.get("text", "").lower(),
)
if youtube_nodes or chrome_home:
print("Chrome seems to be in main browser view.")
break
print("Waiting for welcome screens to resolve or main view to appear...")
time.sleep(1)
def play_first_video(timeout=20):
"""Finds and clicks the first video on the YouTube page."""
print("Waiting for YouTube to load and finding a video...")
start_time = time.time()
while time.time() - start_time < timeout:
xml = dump_ui()
if not xml:
time.sleep(1)
continue
# Look for video elements.
# Heuristic: clickable elements with content-desc containing "views" and "ago"
# or elements that look like video titles.
video_nodes = []
# Try finding by content-desc (common for video cards)
video_nodes = find_nodes(
xml,
lambda a: a.get("clickable") == "true" and "views" in a.get("content-desc", "").lower(),
)
# If not found, try finding any node with "views" and "ago" in text/desc and click it
if not video_nodes:
video_nodes = find_nodes(
xml,
lambda a: (
(
"views" in a.get("content-desc", "").lower()
and "ago" in a.get("content-desc", "").lower()
)
or ("views" in a.get("text", "").lower() and "ago" in a.get("text", "").lower())
),
)
# If still not found, maybe we are on a cookie consent page?
# Check for "Before you continue to YouTube" or "Reject all" / "Accept all"
consent_nodes = find_nodes(
xml,
lambda a: a.get("text") in ["Reject all", "Accept all", "I agree"],
)
if consent_nodes:
print("Found cookie consent banner. Clicking 'Accept all' or 'I agree'...")
# Prefer Accept/Agree
agree_node = [n for n in consent_nodes if n.get("text") in ["Accept all", "I agree"]]
node_to_click = agree_node[0] if agree_node else consent_nodes[0]
tap_element(node_to_click, "Consent button")
time.sleep(3)
continue
if video_nodes:
print(f"Found {len(video_nodes)} potential video elements.")
# Click the first one
first_video = video_nodes[0]
desc = first_video.get("content-desc", first_video.get("text", "video"))
print(f"Selecting video: {desc[:50]}...")
if tap_element(first_video, "first video"):
return True
print("Still waiting for videos to load...")
time.sleep(2)
print("Failed to find a video to play.")
return False
def verify_playback(timeout=15):
"""Verifies that the video is playing."""
print("Verifying playback...")
start_time = time.time()
while time.time() - start_time < timeout:
xml = dump_ui()
if not xml:
time.sleep(1)
continue
# Check for video player element
player_nodes = find_nodes(
xml,
lambda a: (
"video player" in a.get("text", "").lower()
or "video player" in a.get("content-desc", "").lower()
),
)
if player_nodes:
print("Video player detected on screen.")
# Check if it's paused or if we need to click play
# Sometimes there is a "Play video" button if it didn't auto-play
play_button = find_nodes(
xml,
lambda a: a.get("text") == "Play" or a.get("content-desc") == "Play video",
)
if play_button:
print("Video is paused. Clicking play...")
tap_element(play_button[0], "Play button")
time.sleep(2)
else:
print("Video seems to be playing (no play button blocking it, player active).")
# Check for "TAP TO UNMUTE"
unmute_button = find_nodes(xml, lambda a: "unmute" in a.get("text", "").lower())
if unmute_button:
print("Optional: Clicking unmute...")
tap_element(unmute_button[0], "Unmute button")
return True
print("Waiting for video player to appear...")
time.sleep(2)
return False
def main():
# Ensure a device is connected
out, err = run_cmd("adb devices")
devices = [
line.split()[0]
for line in out.splitlines()[1:]
if line.strip() and not line.startswith("*")
]
if not devices:
print("No ADB devices found. Please connect a device and try again.")
sys.exit(1)
print(f"Using device: {devices[0]}")
# Optional: Clear Chrome to test clean run (highly recommended for testing the script itself)
# To enable, uncomment the line below:
# print("Clearing Chrome data for a clean test...")
# adb_cmd("shell pm clear com.android.chrome")
# time.sleep(2)
# Start Chrome and navigate to YouTube
url = "https://m.youtube.com"
print(f"Starting Chrome and navigating to {url}...")
# -d specifies the data URI
# -n specifies the component
adb_cmd(f"shell am start -n com.android.chrome/com.google.android.apps.chrome.Main -d {url}")
# Wait for app to launch
time.sleep(3)
# Step 1: Bypass welcome screens
bypass_chrome_welcome()
# Step 2: Play the first video
if play_first_video():
# Step 3: Verify it plays
time.sleep(3) # Wait for transition
if verify_playback():
print("SUCCESS: Chrome opened, navigated to YouTube, and video playback started!")
else:
print("FAILURE: Video player not detected or failed to play.")
else:
print("FAILURE: Could not find or click a video.")
if __name__ == "__main__":
main()
@@ -0,0 +1,124 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import logging
import os
import sys
from unittest.mock import patch
from dotenv import load_dotenv
# Ensure we can import from project
sys.path.append(os.path.dirname(__file__))
from artemis.sdk import Agent
from artemis.sdk.builders import Builders
from artemis.sdk.types import AgentProfile
from artemis.agents.validator.validator import ValidatorNode, ValidationErrorCategory
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("test_fa_e2e")
# We want to fail the first validation to trigger the Failure Analyzer
original_validate = ValidatorNode._validate_action_precondition
original_validate_pixel = ValidatorNode._validate_action_precondition_pixel
has_failed_once = False
async def mock_validate_action_precondition(self, session, action_item, state=None):
global has_failed_once
if not has_failed_once:
has_failed_once = True
logger.warning(">>> MOCK VALIDATOR: Simulating pre-execution XML validation failure!")
return (
False,
ValidationErrorCategory.TARGET_DISAPPEARED,
"Simulated pre-execution XML validation failure",
)
return await original_validate(self, session, action_item, state=state)
async def mock_validate_action_precondition_pixel(
self,
session,
action_item,
pre_screenshot_b64,
original_coords=None,
state=None,
):
global has_failed_once
if not has_failed_once:
has_failed_once = True
logger.warning(">>> MOCK VALIDATOR: Simulating pre-execution Pixel validation failure!")
return (
False,
ValidationErrorCategory.TARGET_DISAPPEARED,
"Simulated pre-execution Pixel validation failure",
)
return await original_validate_pixel(
self,
session,
action_item,
pre_screenshot_b64,
original_coords,
state=state,
)
async def main():
load_dotenv()
# Check if emulator/device is connected
import subprocess
res = subprocess.run(["adb", "devices"], capture_output=True, text=True)
print("ADB Devices:")
print(res.stdout)
# 1. Initialize agent
logger.info("Initializing Agent...")
profile = AgentProfile(name="default", from_file="llm-config.override.jsonc")
config = Builders.AgentConfig.with_default_profile(profile).build()
agent = Agent(config=config)
await agent.init()
# 2. Run the task with patches
goal = "Play a video in full screen on YouTube in Chrome"
logger.info(f"Running task with goal: {goal}")
with (
patch.object(
ValidatorNode,
"_validate_action_precondition",
mock_validate_action_precondition,
),
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
mock_validate_action_precondition_pixel,
),
):
try:
result = await agent.run_task(goal=goal, name="test_fa_youtube_fullscreen")
logger.info(f"Task finished! Result: {result}")
except Exception:
logger.exception("Error during execution:")
finally:
await agent.clean()
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,68 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Helper utilities for Explorer integration tests."""
from pathlib import Path
from PIL import Image
from unittest.mock import MagicMock
from artemis.context import ArtemisContext
from artemis.graph.state import State
def create_mock_context() -> MagicMock:
"""Creates a mock ArtemisContext suitable for Explorer tests."""
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.llm_config = MagicMock()
ctx.agent_config = MagicMock()
ctx.agent_config.blacklisted_tools = {"explorer": []}
ctx.data_engine = None
ctx.adb_client = None
return ctx
def create_mock_state() -> MagicMock:
"""Creates a mock State suitable for Explorer tests."""
state = MagicMock(spec=State)
state.operator_raw_data = {"width": 1080, "height": 2400}
state.latest_ui_hierarchy = []
state.latest_screenshot = None
return state
def get_or_create_test_screenshot(preferred_path: Path | None = None) -> Path:
"""Finds an existing test screenshot or generates a valid 1080x2400 mock JPEG."""
if preferred_path and preferred_path.exists() and preferred_path.stat().st_size > 0:
return preferred_path
# Search known repo screenshot paths
candidate_paths = [
Path(__file__).parent / "input_screenshot_test_explorer_all_tools_sequential_mocked.jpg",
Path(__file__).parent / "input_screenshot_test_explorer_ask_vision_coder_tool.jpg",
Path(__file__).parents[3] / "tools" / "inputs" / "screenshot.jpg",
]
for p in candidate_paths:
if p.exists() and p.stat().st_size > 0:
return p
# If none found, generate a mock RGB image
out_path = (
Path(__file__).parent / "input_screenshot_test_explorer_all_tools_sequential_mocked.jpg"
)
img = Image.new("RGB", (1080, 2400), color=(73, 109, 137))
img.save(out_path, format="JPEG")
return out_path
@@ -0,0 +1,280 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import json
import logging
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.explorer.explorer import Explorer
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import (
create_mock_context,
create_mock_state,
)
from tests.integration.agents.explorer.test_explorer_all_tools.seed_mock_image import (
seed_database,
)
# Set up logging to stdout
logging.basicConfig(level=logging.INFO, format="%(message)s")
logger = logging.getLogger("live_tools_test")
def extract_and_print_tool_results(contents, set_idx, logger):
logger.info("\n======================================================================")
logger.info(f" RESULTS FOR INPUT TEST SET {set_idx}")
logger.info("======================================================================")
for content in contents:
if content.role != "tool":
continue
for part in content.parts:
if part.function_response:
name = part.function_response.name
response_data = part.function_response.response
# Extract result text
tool_text = None
if hasattr(response_data, "get"):
tool_text = response_data.get("result")
elif hasattr(response_data, "fields"):
tool_text = response_data.fields.get("result")
else:
tool_text = getattr(response_data, "result", None)
logger.info(f"\n🛠️ Tool: '{name}'")
logger.info("--------------------------------------------------")
logger.info(f"Output Result:\n{tool_text}")
if part.inline_data and part.inline_data.mime_type == "image/jpeg":
logger.info(
f"📷 Visual Data: Attached {len(part.inline_data.data)}"
" bytes of annotated JPEG screenshot"
)
async def run_single_set(set_idx, query, search_query, labels, coords_nx, coords_ny, candidates):
import os
os.environ["GOOGLE_API_KEY"] = "dummy_api_key"
mock_ctx = create_mock_context()
mock_ctx.llm_config.utils = MagicMock()
mock_ctx.llm_config.utils.object_detector = MagicMock()
mock_ctx.llm_config.utils.object_detector.model = "gemini-2.5-flash"
mock_ctx.llm_config.utils.object_detector.timeout = 10.0
mock_state = create_mock_state()
screenshot_path = (
Path(__file__).resolve().parent
/ "input_screenshot_test_explorer_all_tools_sequential_mocked.jpg"
)
# Multi-turn sequential mock client
# Turn 1: get_ocr_list
mock_fc_ocr = MagicMock()
mock_fc_ocr.name = "get_ocr_list"
mock_fc_ocr.args = {}
mock_resp_ocr = MagicMock()
mock_resp_ocr.function_calls = [mock_fc_ocr]
# Turn 2: search_xml_ocr with query
mock_fc_search = MagicMock()
mock_fc_search.name = "search_xml_ocr"
mock_fc_search.args = {"query": search_query}
mock_resp_search = MagicMock()
mock_resp_search.function_calls = [mock_fc_search]
# Turn 3: detect_objects
mock_fc_detect = MagicMock()
mock_fc_detect.name = "detect_objects"
mock_fc_detect.args = {"labels": labels, "target_image_id": "img_0"}
mock_resp_detect = MagicMock()
mock_resp_detect.function_calls = [mock_fc_detect]
# Turn 4: search_xml_ocr with coordinates
mock_fc_coords = MagicMock()
mock_fc_coords.name = "search_xml_ocr"
mock_fc_coords.args = {"nx": coords_nx, "ny": coords_ny}
mock_resp_coords = MagicMock()
mock_resp_coords.function_calls = [mock_fc_coords]
# Turn 5: submit_answer
mock_fc_submit = MagicMock()
mock_fc_submit.name = "submit_answer"
mock_fc_submit.args = {
"candidates": candidates,
"fallback_message": f"Successfully verified test set {set_idx}",
}
mock_resp_submit = MagicMock()
mock_resp_submit.function_calls = [mock_fc_submit]
mock_client = MagicMock()
# Mock client.aio.models.generate_content side effects for ReAct turns
mock_client.aio.models.generate_content = AsyncMock(
side_effect=[
mock_resp_ocr,
mock_resp_search,
mock_resp_detect,
mock_resp_coords,
mock_resp_submit,
]
)
# Mock the internal model call for _detect_single_label inside detect_objects
mock_detect_response = MagicMock()
y_norm, x_norm = 0, 0
if "gear icon" in labels:
y_norm, x_norm = 300, 200
elif "dashboard icon" in labels:
y_norm, x_norm = 500, 500
elif "magnifying glass icon" in labels:
y_norm, x_norm = 140, 875
elif "avatar icon" in labels:
y_norm, x_norm = 825, 375
elif "arrow back icon" in labels:
y_norm, x_norm = 75, 100
mock_detect_response.text = json.dumps([{"point": [y_norm, x_norm]}])
mock_detector_client = MagicMock()
mock_detector_client.aio.models.generate_content = AsyncMock(return_value=mock_detect_response)
with (
patch(
"google.genai.Client",
side_effect=[mock_client, mock_detector_client],
),
):
explorer = Explorer(mock_ctx)
await explorer.run(
query=query,
context_feedback="",
screenshot_path=str(screenshot_path),
state=mock_state,
)
call_args_list = mock_client.aio.models.generate_content.call_args_list
contents = call_args_list[4].kwargs.get("contents") or call_args_list[4].args[0]
extract_and_print_tool_results(contents, set_idx, logger)
async def main():
import shutil
from artemis.config import settings
# Define outputs directory
outputs_dir = Path(__file__).resolve().parent / "outputs"
if outputs_dir.exists():
shutil.rmtree(outputs_dir)
outputs_dir.mkdir(parents=True, exist_ok=True)
# Redirect traces output to outputs/ and only keep the latest run
settings.TRACES_PATH = outputs_dir
# 1. Seed the SQLite DB record first
seed_database()
# 2. Define the 5 groups of input tests
test_sets = [
{
"query": "Verify Settings button",
"search_query": "Settings",
"labels": ["gear icon"],
"coords_nx": (185), # Matches pixel center [200, 225] on [1080, 2400] scale
"coords_ny": 93,
"candidates": [
{
"label": "S1",
"coords": [185, 93],
"description": "Settings Button",
}
],
},
{
"query": "Verify Dashboard button",
"search_query": "Dashboard",
"labels": ["dashboard icon"],
"coords_nx": (462), # Matches pixel center [500, 500] on [1080, 2400] scale
"coords_ny": 208,
"candidates": [
{
"label": "S1",
"coords": [462, 208],
"description": "Dashboard Button",
}
],
},
{
"query": "Verify Search bar",
"search_query": "Search",
"labels": ["magnifying glass icon"],
"coords_nx": (810), # Matches pixel center [875, 140] on [1080, 2400] scale
"coords_ny": 58,
"candidates": [
{
"label": "S1",
"coords": [810, 58],
"description": "Search bar",
}
],
},
{
"query": "Verify Profile icon",
"search_query": "Profile",
"labels": ["avatar icon"],
"coords_nx": (347), # Matches pixel center [375, 825] on [1080, 2400] scale
"coords_ny": 343,
"candidates": [
{
"label": "S1",
"coords": [347, 343],
"description": "Profile icon",
}
],
},
{
"query": "Verify Back button",
"search_query": "Back",
"labels": ["arrow back icon"],
"coords_nx": (92), # Matches pixel center [100, 75] on [1080, 2400] scale
"coords_ny": 31,
"candidates": [
{
"label": "S1",
"coords": [92, 31],
"description": "Back button",
}
],
},
]
# 3. Run all 5 sets sequentially
for i, test in enumerate(test_sets, 1):
await run_single_set(
set_idx=i,
query=test["query"],
search_query=test["search_query"],
labels=test["labels"],
coords_nx=test["coords_nx"],
coords_ny=test["coords_ny"],
candidates=test["candidates"],
)
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,162 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib
import json
from pathlib import Path
import sqlite3
def seed_database():
db_path = Path(__file__).resolve().parents[5] / "traces" / "data_engine.db"
screenshot_path = (
Path(__file__).resolve().parent
/ "input_screenshot_test_explorer_all_tools_sequential_mocked.jpg"
)
if not screenshot_path.exists():
# Try fallback to tests/screenshots/pixel_phone_mockup.png
source_path = (
Path(__file__).resolve().parents[5] / "tests" / "screenshots" / "pixel_phone_mockup.png"
)
import shutil
shutil.copy(source_path, screenshot_path)
print(f"Copied screenshot from {source_path} to {screenshot_path}")
# Compute SHA-256 hash
sha256_hash = hashlib.sha256()
with open(screenshot_path, "rb") as f:
for byte_block in iter(lambda: f.read(4096), b""):
sha256_hash.update(byte_block)
computed_hash = sha256_hash.hexdigest()
print(f"Computed screenshot hash: {computed_hash}")
# Bounding elements
# 1. Settings [100, 200][300, 250] -> center pixel [200, 225] -> normalized [185, 93]
# 2. Dashboard [400, 400][600, 600] -> center pixel [500, 500] -> normalized [462, 208]
# 3. Search [800, 100][950, 180] -> center pixel [875, 140] -> normalized [810, 58]
# 4. Profile [300, 800][450, 850] -> center pixel [375, 825] -> normalized [347, 343]
# 5. Back [50, 50][150, 100] -> center pixel [100, 75] -> normalized [92, 31]
ocr_result = [
{"text": "Screen Dashboard Title", "position": []},
{
"text": "Settings",
"position": [
{"x": 100, "y": 200},
{"x": 300, "y": 200},
{"x": 300, "y": 250},
{"x": 100, "y": 250},
],
},
{
"text": "Dashboard",
"position": [
{"x": 400, "y": 400},
{"x": 600, "y": 400},
{"x": 600, "y": 600},
{"x": 400, "y": 600},
],
},
{
"text": "Search",
"position": [
{"x": 800, "y": 100},
{"x": 950, "y": 100},
{"x": 950, "y": 180},
{"x": 800, "y": 180},
],
},
{
"text": "Profile",
"position": [
{"x": 300, "y": 800},
{"x": 450, "y": 800},
{"x": 450, "y": 850},
{"x": 300, "y": 850},
],
},
{
"text": "Back",
"position": [
{"x": 50, "y": 50},
{"x": 150, "y": 50},
{"x": 150, "y": 100},
{"x": 50, "y": 100},
],
},
]
ui_tree = [
{"class": "android.widget.FrameLayout", "bounds": "[0,0][1080,2400]"},
{
"class": "android.widget.Button",
"text": "Settings",
"resource-id": "com.example.app:id/settings",
"clickable": "true",
"bounds": "[100,200][300,250]",
},
{
"class": "android.widget.Button",
"text": "Dashboard",
"resource-id": "com.example.app:id/dashboard",
"clickable": "true",
"bounds": "[400,400][600,600]",
},
{
"class": "android.widget.Button",
"text": "Search",
"resource-id": "com.example.app:id/search",
"clickable": "true",
"bounds": "[800,100][950,180]",
},
{
"class": "android.widget.Button",
"text": "Profile",
"resource-id": "com.example.app:id/profile",
"clickable": "true",
"bounds": "[300,800][450,850]",
},
{
"class": "android.widget.Button",
"text": "Back",
"resource-id": "com.example.app:id/back",
"clickable": "true",
"bounds": "[50,50][150,100]",
},
]
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
cursor.execute(
"INSERT OR REPLACE INTO images (image_name, timestamp, ocr_result,"
" ui_tree, extra_metadata) VALUES (?, ?, ?, ?, ?)",
(
computed_hash,
1717398000.0,
json.dumps(ocr_result),
json.dumps(ui_tree),
json.dumps({}),
),
)
conn.commit()
conn.close()
print(f"Successfully inserted mock image record {computed_hash} into database at {db_path}")
if __name__ == "__main__":
seed_database()
@@ -0,0 +1,62 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, patch
from artemis.agents.object_detector.object_detector import _run_object_detection
import pytest
@pytest.mark.asyncio
async def test_coordinate_normalization_swap():
"""Verify that _run_object_detection correctly swaps coordinates from [y, x] to [x, y]."""
# 1. Define the mock output returned by the VLM task (VLM native format: [y_norm, x_norm])
mock_vlm_output = [
{
"label": "settings icon",
"point": [800, 200], # [y, x] -> y = 800, x = 200
}
]
# 2. Patch dependencies to run in a pure offline sandbox
with (
patch(
"artemis.agents.object_detector.object_detector._detect_single_label",
new_callable=AsyncMock,
return_value=mock_vlm_output,
),
patch("artemis.agents.object_detector.object_detector.get_llm"),
):
# Mock context setup
mock_ctx = AsyncMock()
mock_ctx.llm_config.utils.object_detector = None
# 3. Execute the function under test
json_output = await _run_object_detection(
ctx=mock_ctx,
image_bytes=b"fake_image_bytes",
queries=["settings icon"],
templates=["find the {label}"],
)
# 4. Verify that the coordinates were successfully swapped
result = json_output
detected_items = result["detected"]
assert len(detected_items) == 1
assert detected_items[0]["label"] == "settings icon"
# CRITICAL ASSERTION:
# The coordinate MUST be swapped from [800, 200] (y, x) to [200, 800] (x, y)
assert detected_items[0]["point"] == [200, 800]
@@ -0,0 +1,334 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import json
import logging
from pathlib import Path
import shutil
import sys
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.explorer.explorer import Explorer
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import (
create_mock_context,
create_mock_state,
get_or_create_test_screenshot,
)
import pytest
def _verify_all_tools_responses(contents: list, logger: logging.Logger):
"""Parses the multi-turn conversational history payload and maps tool outputs."""
logger.info("=== VERIFYING TOOL RESPONSES IN CONVERSATION HISTORY ===")
tool_responses = {}
for content in contents:
for part in content.parts:
if part.function_response:
name = part.function_response.name
response_data = part.function_response.response
# Extract result text
tool_text = None
if hasattr(response_data, "get"):
tool_text = response_data.get("result")
elif hasattr(response_data, "fields"):
tool_text = response_data.fields.get("result")
else:
tool_text = getattr(response_data, "result", None)
if name not in tool_responses:
tool_responses[name] = []
tool_responses[name].append({"text": tool_text, "has_image": False})
logger.info(f"Found tool response for '{name}':\n{tool_text}")
if part.inline_data and part.inline_data.mime_type == "image/jpeg":
# The image byte block corresponds to the preceding function response
if tool_responses:
last_tool = list(tool_responses.keys())[-1]
tool_responses[last_tool][-1]["has_image"] = True
tool_responses[last_tool][-1]["img_bytes"] = part.inline_data.data
logger.info(
f" -> Attached {len(part.inline_data.data)} bytes of"
f" JPEG inline visual content to '{last_tool}'"
)
return tool_responses
@pytest.mark.asyncio
async def test_explorer_all_tools_sequential_mocked():
"""Verify that the Explorer ReAct loop correctly sequences, dispatches, processes, and
annotates screenshot outputs for all available visual tools:
1. get_ocr_list
2. search_xml_ocr (query)
3. detect_objects
4. search_xml_ocr (coordinates)
5. submit_answer
"""
test_name = sys._getframe().f_code.co_name
# Set up standard log file saved on the same level as test file
log_file = Path(__file__).parent / f"log_{test_name}.log"
logger = logging.getLogger(test_name)
logger.setLevel(logging.INFO)
handler = logging.FileHandler(log_file, mode="w", encoding="utf-8")
logger.addHandler(handler)
# Attach the main agent logger to capture internal step logs
agent_logger = logging.getLogger("artemis.agents.explorer.explorer")
agent_logger.setLevel(logging.INFO)
agent_logger.addHandler(handler)
logger.info(f"Starting comprehensive test: {test_name}")
# Define outputs directory and redirect settings to ensure only the latest run is saved
outputs_dir = Path(__file__).resolve().parent / "outputs"
if outputs_dir.exists():
shutil.rmtree(outputs_dir)
outputs_dir.mkdir(parents=True, exist_ok=True)
from artemis.config import settings
settings.TRACES_PATH = outputs_dir
mock_ctx = create_mock_context()
mock_state = create_mock_state()
logger.info(
f"Mock context: device={mock_ctx.device.device_width}x{mock_ctx.device.device_height}"
)
# Load and verify screenshot
input_screenshot = get_or_create_test_screenshot()
logger.info(f"Visual screenshot loaded: {input_screenshot}")
# =========================================================================
# 1. Mock LLM Multi-Turn Responses (Sequentially calling each tool)
# =========================================================================
# Turn 1: call get_ocr_list
mock_fc_ocr = MagicMock()
mock_fc_ocr.name = "get_ocr_list"
mock_fc_ocr.args = {}
mock_resp_ocr = MagicMock()
mock_resp_ocr.function_calls = [mock_fc_ocr]
# Turn 2: call ask_perception_tool
mock_fc_search = MagicMock()
mock_fc_search.name = "ask_perception_tool"
mock_fc_search.args = {
"search_query": "Settings",
"nx": 500,
"ny": 500,
"detect_queries": ["gear icon"],
}
mock_resp_search = MagicMock()
mock_resp_search.function_calls = [mock_fc_search]
# Turn 3: call detect_objects
mock_fc_detect = MagicMock()
mock_fc_detect.name = "detect_objects"
mock_fc_detect.args = {"queries": ["gear icon"], "target_image_id": "img_0"}
mock_resp_detect = MagicMock()
mock_resp_detect.function_calls = [mock_fc_detect]
# Turn 4: call inspect_region
mock_fc_coords = MagicMock()
mock_fc_coords.name = "inspect_region"
mock_fc_coords.args = {
"x_min": 100,
"y_min": 100,
"x_max": 300,
"y_max": 300,
"zoom_factor": 2.0,
}
mock_resp_coords = MagicMock()
mock_resp_coords.function_calls = [mock_fc_coords]
# Turn 5: call submit_answer to complete loop
mock_fc_submit = MagicMock()
mock_fc_submit.name = "submit_answer"
mock_fc_submit.args = {
"candidates": [
{
"label": "T1",
"coords": [462, 208],
"description": "Dashboard Button",
},
{
"label": "S1",
"coords": [462, 208],
"description": "Settings Button",
},
{"label": "D1", "coords": [200, 300], "description": "gear icon"},
],
"fallback_message": ("Successfully verified all visual controls sequentially."),
}
mock_resp_submit = MagicMock()
mock_resp_submit.function_calls = [mock_fc_submit]
mock_client = MagicMock()
mock_client.aio.models.generate_content = AsyncMock(
side_effect=[
mock_resp_ocr,
mock_resp_search,
mock_resp_detect,
mock_resp_coords,
mock_resp_submit,
]
)
mock_ctx._genai_client = mock_client
mock_ctx.llm_config.explorer = None
# =========================================================================
# 2. Mock Downstream Tool Logic Dependencies
# =========================================================================
# Mock SQLite StorageManager for get_ocr_list
mock_storage = MagicMock()
mock_record = MagicMock()
mock_record.ocr_result = [
{"text": "screen description text", "position": []},
{
"text": "Dashboard Button",
"position": [
{"x": 400, "y": 400},
{"x": 600, "y": 400},
{"x": 600, "y": 600},
{"x": 400, "y": 600},
],
},
]
mock_storage.get_image.return_value = mock_record
# Mock low-level layout searching (search_ui_func)
mock_search_ui = MagicMock(
return_value={
"matches": [
{
"matched_text": "Settings Button",
"bounds": [400, 400, 600, 600],
}
]
}
)
# Mock downstream object detection pipeline (_run_object_detection)
mock_detect = AsyncMock(
return_value={"detected": [{"label": "gear icon", "point": [200, 300]}]}
)
# Mock spatial coordinate lookup (search_by_coordinates_func)
mock_lookup = MagicMock(return_value="Matching Elements: Container/DashboardFrame")
logger.info("Starting Explorer.run execution loop with sequential tool mocks...")
with (
patch(
"artemis.agents.explorer.explorer.genai.Client",
return_value=mock_client,
),
patch(
"artemis.agents.explorer.explorer.StorageManager",
return_value=mock_storage,
),
patch("artemis.agents.explorer.explorer.search_ui_func", mock_search_ui),
patch("artemis.agents.explorer.explorer._run_object_detection", mock_detect),
patch(
"artemis.agents.explorer.explorer.search_by_coordinates_func",
mock_lookup,
),
):
explorer = Explorer(mock_ctx)
result = await explorer.run(
query="Test all tools",
context_feedback="",
screenshot_path=str(input_screenshot),
state=mock_state,
version="ultra",
)
# Verify ReAct loop went through all 5 turns
assert mock_client.aio.models.generate_content.call_count == 5
logger.info("Successfully completed all 5 ReAct turns.")
# Retrieve final submission outcome
outcome_data = json.loads(result)
assert "candidates" in outcome_data
assert (
outcome_data["fallback_message"]
== "Successfully verified all visual controls sequentially."
)
logger.info("Submission response verified successfully.")
# Extract all tool outputs from history to assert correctness
call_args_list = mock_client.aio.models.generate_content.call_args_list
# The fifth call receives the accumulated tool history
contents = call_args_list[4].kwargs.get("contents") or call_args_list[4].args[0]
tool_responses = _verify_all_tools_responses(contents, logger)
# ASSERTIONS FOR EACH TOOL INDIVIDUALLY
# Verify each tool execution record in explorer.trace_history
assert len(explorer.trace_history) >= 4
# 1. Verify get_ocr_list
ocr_trace = explorer.trace_history[0]["tool_calls"][0]
assert ocr_trace["name"] == "get_ocr_list"
assert "[T1]" in ocr_trace["response"]["text"]
assert "[462,208]" in ocr_trace["response"]["text"]
assert ocr_trace["response"]["image_path"] is not None
# 2. Verify ask_perception_tool
search_trace = explorer.trace_history[1]["tool_calls"][0]
assert search_trace["name"] == "ask_perception_tool"
assert "XML/OCR Text Search Results" in search_trace["response"]["text"]
assert len(search_trace["response"]["image_paths"]) > 0
# 3. Verify detect_objects
detect_trace = explorer.trace_history[2]["tool_calls"][0]
assert detect_trace["name"] == "detect_objects"
assert "[D" in detect_trace["response"]["text"]
assert detect_trace["response"]["image_path"] is not None
# 4. Verify inspect_region
coords_trace = explorer.trace_history[3]["tool_calls"][0]
assert coords_trace["name"] == "inspect_region"
assert coords_trace["response"]["image_path"] is not None
# Compute expected screenshot hash
import hashlib
sha256_hash = hashlib.sha256()
with open(input_screenshot, "rb") as f:
for byte_block in iter(lambda: f.read(4096), b""):
sha256_hash.update(byte_block)
expected_hash = sha256_hash.hexdigest()
# Should map normalized [500, 500] to pixel [540, 1200]
mock_lookup.assert_called_once_with(expected_hash, 540, 1200)
logger.info(
"All assertions passed! The entire suite of explorer tools is fully functional."
)
# Clean up logging handlers
agent_logger.removeHandler(handler)
logger.removeHandler(handler)
handler.close()
if __name__ == "__main__":
asyncio.run(test_explorer_all_tools_sequential_mocked())
@@ -0,0 +1,372 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import json
import logging
from pathlib import Path
import sys
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.explorer.explorer import Explorer
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import (
create_mock_context,
create_mock_state,
get_or_create_test_screenshot,
)
from PIL import Image
import pytest
def _verify_tool_responses(contents: list, logger: logging.Logger):
"""Parses the multi-turn conversational history payload and maps tool outputs."""
logger.info("=== VERIFYING TOOL RESPONSES IN CONVERSATION HISTORY ===")
tool_responses = {}
for content in contents:
for part in content.parts:
if part.function_response:
name = part.function_response.name
response_data = part.function_response.response
# Extract result text
tool_text = None
if hasattr(response_data, "get"):
tool_text = response_data.get("result")
elif hasattr(response_data, "fields"):
tool_text = response_data.fields.get("result")
else:
tool_text = getattr(response_data, "result", None)
if name not in tool_responses:
tool_responses[name] = []
tool_responses[name].append({"text": tool_text, "has_image": False})
logger.info(f"Found tool response for '{name}':\n{tool_text}")
if part.inline_data and part.inline_data.mime_type == "image/jpeg":
if tool_responses:
last_tool = list(tool_responses.keys())[-1]
tool_responses[last_tool][-1]["has_image"] = True
tool_responses[last_tool][-1]["img_bytes"] = part.inline_data.data
logger.info(
f" -> Attached {len(part.inline_data.data)} bytes of"
f" JPEG inline visual content to '{last_tool}'"
)
return tool_responses
@pytest.mark.asyncio
async def test_explorer_ask_image_processor_tool():
"""Verify that the Explorer ReAct loop correctly dispatches, runs, processes, and
handles the output image/transform details for the 'ask_image_processor'
tool.
The test ends immediately after the Vision Coder responds (in the second
turn).
"""
test_name = sys._getframe().f_code.co_name
# Set up standard log file saved on the same level as test file
log_file = Path(__file__).parent / f"log_{test_name}.log"
logger = logging.getLogger(test_name)
logger.setLevel(logging.INFO)
handler = logging.FileHandler(log_file, mode="w", encoding="utf-8")
logger.addHandler(handler)
# Attach the main agent logger to capture internal step logs
agent_logger = logging.getLogger("artemis.agents.explorer.explorer")
agent_logger.setLevel(logging.INFO)
agent_logger.addHandler(handler)
logger.info(f"Starting test: {test_name}")
# Define outputs directory and redirect settings to ensure only the latest run is saved
outputs_dir = Path(__file__).resolve().parent / "outputs"
if not outputs_dir.exists():
outputs_dir.mkdir(parents=True, exist_ok=True)
from artemis.config import settings
settings.TRACES_PATH = outputs_dir
mock_ctx = create_mock_context()
mock_state = create_mock_state()
logger.info(
f"Mock context: device={mock_ctx.device.device_width}x{mock_ctx.device.device_height}"
)
# Load and verify screenshot
input_screenshot = get_or_create_test_screenshot()
logger.info(f"Visual screenshot loaded: {input_screenshot}")
# =========================================================================
# 1. Mock LLM Multi-Turn Responses (ask_image_processor -> submit_answer)
# =========================================================================
# Turn 1: call ask_image_processor
mock_fc_coder = MagicMock()
mock_fc_coder.name = "ask_image_processor"
mock_fc_coder.args = {
"instruction": (
"Crop the top-right area, convert to HSV, resize, then convert to grayscale."
),
"target_image_id": "img_0",
}
mock_resp_coder = MagicMock()
mock_resp_coder.function_calls = [mock_fc_coder]
# Turn 2: call submit_answer to complete loop immediately after vision coder responds
mock_fc_submit = MagicMock()
mock_fc_submit.name = "submit_answer"
mock_fc_submit.args = {
"candidates": [],
"fallback_message": ("Successfully processed the image via Vision Coder."),
}
mock_resp_submit = MagicMock()
mock_resp_submit.function_calls = [mock_fc_submit]
mock_client = MagicMock()
mock_client.aio.models.generate_content = AsyncMock(
side_effect=[mock_resp_coder, mock_resp_submit]
)
mock_ctx._genai_client = mock_client
mock_ctx.llm_config.explorer = None
# =========================================================================
# 2. Mock ImageProcessor agent
# =========================================================================
# Create the mock output image path
mock_new_image_path = outputs_dir / f"mock_image_processor_output_{test_name}.jpg"
# Create a small dummy image of size 100x100 using PIL so that cv2.imread inside explorer reads these dimensions
img = Image.new("RGB", (100, 100), color="red")
img.save(mock_new_image_path)
# Also write a transform JSON file for the new image
transform_path = Path(str(mock_new_image_path) + ".transform.json")
mock_transform_data = {
"offset_x": 108.0,
"offset_y": 240.0,
"scale_x": 0.5,
"scale_y": 0.5,
}
with open(transform_path, "w") as f:
json.dump(mock_transform_data, f)
# We mock ImageProcessor class itself
mock_image_processor_instance = MagicMock()
mock_image_processor_instance.run = AsyncMock(
return_value={
"outputs": [
{
"image_id": "img_1",
"path": str(mock_new_image_path),
"transform": {
"offset_x": 108.0,
"offset_y": 240.0,
"scale_x": 0.5,
"scale_y": 0.5,
},
"annotations": {"V1": [50, 100], "V2": [180, 240]},
}
],
"summary": "Cropped and highlighted the red button",
}
)
logger.info("Starting Explorer.run execution loop with ask_image_processor...")
with (
patch(
"artemis.agents.explorer.explorer.genai.Client",
return_value=mock_client,
),
patch(
"artemis.agents.image_processor.image_processor.ImageProcessor",
return_value=mock_image_processor_instance,
),
):
explorer = Explorer(mock_ctx)
result = await explorer.run(
query=("Click on the red button in the top-right corner of the screen."),
context_feedback="",
screenshot_path=str(input_screenshot),
state=mock_state,
version="ultra",
)
# Verify initial image pool has img_0
assert "img_0" in explorer.image_pool
assert explorer.image_pool["img_0"]["path"] == str(input_screenshot)
# Verify ReAct loop went through exactly 2 turns
assert mock_client.aio.models.generate_content.call_count == 2
logger.info("Successfully completed both ReAct turns.")
# Verify ImageProcessor run was called with correct parameters
mock_image_processor_instance.run.assert_called_once_with(
"Crop the top-right area, convert to HSV, resize, then convert to grayscale.",
str(input_screenshot),
)
logger.info("Verified ImageProcessor was invoked with correct parameters.")
# Verify new image is registered in explorer's image_pool
assert "img_1" in explorer.image_pool
img_1_info = explorer.image_pool["img_1"]
assert img_1_info["path"] == str(mock_new_image_path)
# Verify transform calculations:
# Since parent (img_0) transform is {"offset_x": 0.0, "offset_y": 0.0, "scale_x": 1.0, "scale_y": 1.0}
# and child transform is {"offset_x": 108.0, "offset_y": 240.0, "scale_x": 0.5, "scale_y": 0.5}
assert img_1_info["transform"]["scale_x"] == 0.5
assert img_1_info["transform"]["scale_y"] == 0.5
assert img_1_info["transform"]["offset_x"] == 108.0
assert img_1_info["transform"]["offset_y"] == 240.0
assert "Cropped and highlighted the red button" in img_1_info["description"]
logger.info("Verified image_pool registrations and coordinate transforms.")
# Retrieve final submission outcome
outcome_data = json.loads(result)
assert (
outcome_data["fallback_message"] == "Successfully processed the image via Vision Coder."
)
logger.info("Submission response verified successfully.")
# Extract all tool outputs from history to assert correctness
call_args_list = mock_client.aio.models.generate_content.call_args_list
# The second call receives the accumulated tool history
contents = call_args_list[1].kwargs.get("contents") or call_args_list[1].args[0]
tool_responses = _verify_tool_responses(contents, logger)
# Assertions for ask_image_processor tool response (Turn 1 output)
assert "ask_image_processor" in tool_responses
coder_resp = tool_responses["ask_image_processor"][0]
assert "Image Processor completed successfully" in coder_resp["text"]
assert "img_1" in coder_resp["text"]
assert "Annotations (wrt original screenshot):" in coder_resp["text"]
assert "img_1:" in coder_resp["text"]
assert "- [V1]: [208, 440]" in coder_resp["text"]
assert "- [V2]: [468, 720]" in coder_resp["text"]
assert coder_resp["has_image"]
assert len(coder_resp["img_bytes"]) > 0
logger.info(
"Verified tool response contains the response text, annotations"
" list, and the attached image bytes."
)
# Clean up dummy files (Commented out temporarily so you can see the images!)
# if mock_new_image_path.exists():
# mock_new_image_path.unlink()
# if transform_path.exists():
# transform_path.unlink()
# if input_screenshot.exists():
# input_screenshot.unlink()
# Clean up logging handlers
agent_logger.removeHandler(handler)
logger.removeHandler(handler)
handler.close()
if __name__ == "__main__":
asyncio.run(test_explorer_ask_image_processor_tool())
def test_cv_canvas_annotations(tmp_path):
import numpy as np
import cv2
from artemis.utils.cv_canvas import ImageCanvas
# Create a dummy image
img_dir = tmp_path / "images"
img_dir.mkdir()
img_path = img_dir / "img_0.jpg"
img_data = np.zeros((100, 100, 3), dtype=np.uint8)
cv2.imwrite(str(img_path), img_data)
# Initialize transforms registry
registry_path = img_dir / "intermediate_transforms.json"
registry_data = {
"img_0": {
"image_id": "img_0",
"path": str(img_path.resolve()),
"transform": {
"offset_x": 0.0,
"offset_y": 0.0,
"scale_x": 1.0,
"scale_y": 1.0,
},
"annotations": {},
}
}
with open(registry_path, "w") as f:
json.dump(registry_data, f)
canvas = ImageCanvas("img_0", str(img_dir))
assert canvas.annotations == {}
# Test _get_resolution and initial width/height
assert canvas._get_resolution() == (100, 100)
assert canvas.width == 100
assert canvas.height == 100
# Test normalized_to_pixel_coords
assert canvas.normalized_to_pixel_coords(500, 500) == (50, 50)
assert canvas.normalized_to_pixel_coords(0, 0) == (0, 0)
assert canvas.normalized_to_pixel_coords(1000, 1000) == (99, 99)
# Draw dot
canvas.draw_dot(10, 20, 1)
assert canvas.annotations == {"V1": [10, 20]}
# Crop
canvas.crop(5, 5, 20, 20)
assert canvas.width == 20
assert canvas.height == 20
assert canvas._get_resolution() == (20, 20)
assert canvas.normalized_to_pixel_coords(500, 500) == (10, 10)
assert canvas.annotations == {"V1": [5, 15]}
# Out of bounds test
canvas.draw_dot(2, 2, 2)
assert canvas.annotations == {"V1": [5, 15], "V2": [2, 2]}
# Crop again to discard V1
canvas.crop(1, 1, 5, 5)
assert canvas.width == 5
assert canvas.height == 5
assert canvas.normalized_to_pixel_coords(500, 500) == (2, 2)
# V1 was at [5, 15], after crop(1, 1), it becomes [4, 14], which is out of bounds of 5x5.
# V2 was at [2, 2], after crop(1, 1), it becomes [1, 1], which is inside 5x5.
assert canvas.annotations == {"V2": [1, 1]}
# Resize by factor
canvas.resize_by_factor(2.0)
assert canvas.width == 10
assert canvas.height == 10
assert canvas._get_resolution() == (10, 10)
assert canvas.normalized_to_pixel_coords(500, 500) == (5, 5)
assert canvas.annotations == {"V2": [2, 2]}
# Save and verify registry
canvas.save(final=True)
with open(registry_path) as f:
saved_registry = json.load(f)
assert "img_1" in saved_registry
assert saved_registry["img_1"]["annotations"] == {"V2": [2, 2]}
assert saved_registry["img_1"]["is_output"] is True
@@ -0,0 +1,232 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import json
import logging
from pathlib import Path
import shutil
from unittest.mock import patch
from artemis.config import settings
from artemis.agents.image_processor.image_processor import ImageProcessor
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import (
create_mock_context,
get_or_create_test_screenshot,
)
from artemis.utils.python_executor import PythonExecutor
import pytest
# Setup logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("test_explorer_ask_image_processor_jupyter")
@pytest.mark.asyncio
async def test_explorer_ask_image_processor_jupyter():
test_name = "test_explorer_ask_image_processor_jupyter"
# 1. Setup outputs directory
outputs_dir = Path(__file__).resolve().parent / "outputs" / test_name
outputs_dir.mkdir(parents=True, exist_ok=True)
# Redirect traces path to the new outputs directory
settings.TRACES_PATH = outputs_dir
# 2. Prepare the input image
src_image_path = get_or_create_test_screenshot()
input_screenshot = outputs_dir / "input_screenshot.jpg"
shutil.copy2(src_image_path, input_screenshot)
logger.info(f"Copied input screenshot to: {input_screenshot}")
# 3. Formulate the instruction with the injected G-Logo coordinate prior
instruction = (
'The user wants to find "the red patch above the green patch". Let\'s'
" analyze the screen layout. \nIn the dock / app icons, we have:\n-"
" Play Store (9): colorful triangle (has green, blue, yellow, red)\n-"
" Gmail (10): M icon (red, yellow, green, blue)\n- Photos (11):"
" pinwheel (red, yellow, green, blue)\n- Chrome (16): circular logo"
" with green on the bottom, red on top, yellow on the side, blue"
" center.\nWait, let's look at the Photos icon (11) or Chrome icon (16)"
" or Gmail (10) or Google search bar logo (21) or Play Store"
" (9).\nLet's zoom into Chrome (16), Gmail (10), Photos (11), Play"
" Store (9), Google logo (21) to analyze the relative positions of red"
" patches and green patches.\nThe G-Logo is exactly at pixel"
" coordinates [2173, 107, 2241, 174].\nLet's crop these icons to find"
' which icon has a "red patch above the green patch" or is it a'
" specific patch?\nWait!\nLet's look at Chrome (16):\nRed is at the"
" top, green is at the bottom/left. Red is directly above green? Yes,"
" the top slice of Chrome is red, and the bottom slice is green.\nLet's"
" look at Photos (11):\nIt has four petals. The top petal is red? No,"
" top petal is red/pink, bottom is green? Let's check.\nLet's look at"
" Gmail (10):\nThe red is the M shape, green is at the bottom"
" right?\nLet's write Python code to crop these areas and output the"
" crop to see clearly.\nSpecifically, let's crop:\n- Play Store (bounds"
" around 9)\n- Gmail (bounds around 10)\n- Photos (bounds around 11)\n-"
" Chrome (bounds around 16)\n- Google widget logo (bounds around"
" 21)\nWe will save a visualization of these cropped regions side by"
" side or separately so we can examine the colors."
)
# 4. Intercept Python executions to record them for the Jupyter Notebook
executed_turns = []
original_execute = PythonExecutor.execute
def wrapped_execute(self, code: str) -> str:
logger.info(f"Intercepting Python code execution. Code:\n{code}\n")
result = original_execute(self, code)
logger.info(f"Execution result:\n{result}\n")
executed_turns.append({"code": code, "output": result})
return result
from langchain_core.messages import AIMessage
from unittest.mock import AsyncMock, MagicMock
sample_code = """
from artemis.utils.cv_canvas import ImageCanvas
import glob
from pathlib import Path
from artemis.config import settings
dirs = sorted(glob.glob(str(Path(settings.TRACES_PATH) / "images" / "image_processor" / "intermediates_*")))
intermediate_dir = dirs[-1] if dirs else "."
canvas = ImageCanvas("img_0", intermediate_dir)
canvas.crop(10, 10, 50, 50)
canvas.save(final=True)
"""
mock_llm = MagicMock()
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(
side_effect=[
AIMessage(
content="",
tool_calls=[
{
"name": "execute_python",
"args": {"code": sample_code},
"id": "call_1",
}
],
),
AIMessage(
content="",
tool_calls=[
{
"name": "submit_result",
"args": {"summary": "Cropped target G-Logo region."},
"id": "call_2",
}
],
),
]
)
mock_llm.bind_tools.return_value = mock_bound
mock_ctx = create_mock_context()
image_processor = ImageProcessor(mock_ctx)
# Run with patched executor and mock LLM
logger.info("Running Image Processor with G-Logo prior...")
with (
patch.object(PythonExecutor, "execute", wrapped_execute),
patch(
"artemis.agents.image_processor.image_processor.get_llm",
return_value=mock_llm,
),
):
result = await image_processor.run(instruction, str(input_screenshot))
logger.info(f"Image Processor finished. Result: {result}")
# 5. Compile and save the Jupyter Notebook (.ipynb)
notebook_cells = []
# Add a markdown introduction cell
notebook_cells.append(
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Image Processor Execution Trace\n",
f"**Task**: {instruction[:150]}...\n",
"**Input Image**: `input_screenshot.jpg`",
],
}
)
for turn_idx, turn in enumerate(executed_turns):
# Add code cell
notebook_cells.append(
{
"cell_type": "code",
"execution_count": turn_idx + 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [line + "\n" for line in turn["output"].split("\n")],
}
],
"source": [line + "\n" for line in turn["code"].split("\n")],
}
)
# If a result was submitted, add a final markdown cell
outputs = result.get("outputs", [])
if outputs:
final_image_path = outputs[-1]["path"]
notebook_cells.append(
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 🎉 Execution Completed Successfully\n",
f"**Summary**: {result.get('summary')}\n",
f"**Output Image**: `{Path(final_image_path).name}`",
],
}
)
# Copy final image to output directory root for easy access
shutil.copy2(final_image_path, outputs_dir / "final_output.jpg")
notebook_json = {
"cells": notebook_cells,
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3",
}
},
"nbformat": 4,
"nbformat_minor": 2,
}
notebook_path = outputs_dir / "vision_coder_execution.ipynb"
with open(notebook_path, "w", encoding="utf-8") as f:
json.dump(notebook_json, f, indent=2)
logger.info(f"Saved compiled Jupyter notebook to: {notebook_path}")
# 6. Assertions
assert "error" not in result, f"Image Processor failed: {result.get('error')}"
assert "outputs" in result, "Image Processor did not return outputs"
assert len(result["outputs"]) > 0, "Image Processor did not return any output images"
logger.info("Test completed successfully! All assertions passed.")
if __name__ == "__main__":
asyncio.run(test_explorer_ask_image_processor_jupyter())
@@ -0,0 +1,130 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import time
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.object_detector.object_detector import _run_object_detection
from tests.integration.agents.explorer.test_explorer_all_tools.helpers import create_mock_context
import pytest
@pytest.mark.asyncio
async def test_object_detector_concurrency_and_throttling(tmp_path):
"""Verify that object detector requests run in parallel and are correctly throttled by the semaphore."""
# Create a dummy image file for the test
dummy_image_path = tmp_path / "dummy.jpg"
dummy_image_path.write_bytes(b"dummy image data")
# Setup mock context
mock_ctx = create_mock_context()
# Configure the object detector LLM config explicitly to avoid MagicMocks for timeout and model
od_cfg = MagicMock()
od_cfg.model = "gemini-robotics-er-early-access"
od_cfg.fix_model = "gemini-2.5-flash"
od_cfg.timeout = 10.0
mock_ctx.llm_config.utils.object_detector = od_cfg
# Track active requests
active_requests = 0
max_concurrent_requests = 0
request_lock = asyncio.Lock()
from langchain_core.messages import AIMessage
mock_llm = MagicMock()
mock_llm.ainvoke = AsyncMock(side_effect=lambda *args, **kwargs: mock_generate_content())
async def mock_generate_content(*args, **kwargs):
nonlocal active_requests, max_concurrent_requests
async with request_lock:
active_requests += 1
if active_requests > max_concurrent_requests:
max_concurrent_requests = active_requests
try:
await asyncio.sleep(0.1)
finally:
async with request_lock:
active_requests -= 1
return AIMessage(content='[{"point": [500, 500]}]')
mock_llm.ainvoke = AsyncMock(side_effect=mock_generate_content)
with patch(
"artemis.agents.object_detector.object_detector.get_llm",
return_value=mock_llm,
):
# --- TEST 1: Concurrency Assertion (5 labels) ---
# 5 labels should run in parallel, completing in ~0.1s (much faster than a sequential 0.5s)
# Reset trackers
active_requests = 0
max_concurrent_requests = 0
start_time = time.time()
labels_5 = [f"label_{i}" for i in range(5)]
templates = ["Find {labels_str}"]
result_5 = await _run_object_detection(
ctx=mock_ctx,
image_path=str(dummy_image_path),
queries=labels_5,
templates=templates,
)
elapsed_5 = time.time() - start_time
# Concurrency assertions
assert max_concurrent_requests == 5, (
f"Expected 5 concurrent requests, got {max_concurrent_requests}"
)
assert elapsed_5 < 0.3, f"Expected execution to take < 0.3s, but took {elapsed_5:.3f}s"
# Verify result structure and swapped coordinates [y, x] -> [x, y]
parsed_result_5 = result_5
assert len(parsed_result_5["detected"]) == 5
for item in parsed_result_5["detected"]:
assert item["point"] == [500, 500]
# --- TEST 2: Throttling Assertion (10 labels) ---
# 10 labels should be throttled by the semaphore limit of 6.
# It should run in two batches, completing in ~0.2s (much faster than a sequential 1.0s).
# Reset trackers
active_requests = 0
max_concurrent_requests = 0
start_time = time.time()
labels_10 = [f"label_{i}" for i in range(10)]
result_10 = await _run_object_detection(
ctx=mock_ctx,
image_path=str(dummy_image_path),
queries=labels_10,
templates=templates,
)
elapsed_10 = time.time() - start_time
# Throttling assertions
assert max_concurrent_requests == 6, (
f"Expected max concurrent requests to be capped at 6, got {max_concurrent_requests}"
)
assert elapsed_10 < 0.5, (
f"Expected execution to take < 0.5s (two batches of 0.1s), but took {elapsed_10:.3f}s"
)
parsed_result_10 = result_10
assert len(parsed_result_10["detected"]) == 10
for item in parsed_result_10["detected"]:
assert item["point"] == [500, 500]
@@ -0,0 +1,126 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import base64
import logging
import os
import sys
from dotenv import load_dotenv
# Ensure we can import from project
sys.path.append(os.path.dirname(__file__))
from artemis.sdk import Agent
from artemis.sdk.builders import Builders
from artemis.sdk.types import AgentProfile
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.agents.validator.failure_analyzer import FailureAnalyzer, ValidationErrorCategory
from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.utils.notes import get_note_file_path
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("test_fa_standalone")
async def main():
load_dotenv()
# 1. Initialize Agent to set up device and connection pools
logger.info("Initializing Agent and connecting to device...")
profile = AgentProfile(name="default", from_file="llm-config.override.jsonc")
config = Builders.AgentConfig.with_default_profile(profile).build()
agent = Agent(config=config)
await agent.init()
# Create the ArtemisContext manually using agent's initialized clients
context = ArtemisContext(
device=agent._device_context,
adb_client=agent._adb_client,
ui_adb_client=agent._ui_adb_client,
llm_config=profile.llm_config,
agent_config=config,
)
# Set up trace session and data engine
from artemis.data_engine.engine import DataEngine
data_engine = DataEngine(ctx=context)
context.data_engine = data_engine
data_engine.start_session(goal="Play a video in full screen on YouTube in Chrome")
# Write the task plan note
task_plan_content = (
"- [ ] Open Chrome and navigate to the YouTube website (youtube.com)\n"
"- [ ] Select and play any video\n"
"- [ ] Make the playing video full screen"
)
task_plan_path = get_note_file_path(data_engine.base_dir, "task_plan")
task_plan_path.parent.mkdir(parents=True, exist_ok=True)
task_plan_path.write_text(task_plan_content, encoding="utf-8")
# 2. Get initial screenshot
controller = UnifiedMobileController(context)
screen_data = await controller.controller.get_screen_data()
screenshot_b64 = screen_data.base64
# Set up State
state = State(
messages=[],
initial_goal="Play a video in full screen on YouTube in Chrome",
latest_ui_hierarchy=screen_data.elements,
latest_screenshot=str(
data_engine.get_image_path(
data_engine.get_or_create_image(base64.b64decode(screenshot_b64))
)
),
focused_app_info="",
device_date="",
structured_decisions="",
complete_subgoals_by_ids=[],
validator_messages=[],
)
# 3. Instantiate and run Failure Analyzer directly
logger.info("Starting Failure Analyzer directly...")
analyzer = FailureAnalyzer(context)
analyzer.max_iterations = 45
# We pass a dummy failed action representing the fullscreen button tap
dummy_failed_action = {
"action": "tap",
"coordinates": [968, 883],
"target_text": "fullscreen button",
}
try:
result = await analyzer.analyze(
state=state,
failed_action=dummy_failed_action,
error_msg="Simulated launch failure to trigger standalone FA run",
pre_screenshot=screenshot_b64,
post_screenshot=screenshot_b64,
error_category=ValidationErrorCategory.TARGET_DISAPPEARED,
)
logger.info(f"Failure Analyzer direct execution finished! Result: {result}")
except Exception:
logger.exception("Error during Failure Analyzer execution:")
finally:
await agent.clean()
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,58 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import logging
import os
import sys
from dotenv import load_dotenv
# Ensure we can import from project
sys.path.append(os.path.dirname(__file__))
from artemis.sdk import Agent
from artemis.sdk.builders import Builders
from artemis.sdk.types import AgentProfile
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("test_flash_standalone")
async def main():
load_dotenv()
# 1. Initialize Agent to set up device and connection pools
logger.info("Initializing Agent and connecting to device...")
profile = AgentProfile(name="default", from_file="llm-config.override.jsonc")
config = Builders.AgentConfig.with_default_profile(profile).build()
agent = Agent(config=config)
await agent.init()
# 2. Run task using 'flash' profile
logger.info("Starting Flash runner task execution...")
try:
result = await agent.run_task(
goal="Open USA Today in Chrome browser and view the USA news",
profile="flash",
)
logger.info(f"Flash runner task execution finished! Result: {result}")
except Exception:
logger.exception("Error during Flash task execution:")
finally:
await agent.clean()
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,57 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import logging
import os
import sys
from dotenv import load_dotenv
# Ensure we can import from project
sys.path.append(os.path.dirname(__file__))
from artemis.sdk import Agent
from artemis.sdk.builders import Builders
from artemis.sdk.types import AgentProfile
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("test_flash_standalone")
async def main():
load_dotenv()
# 1. Initialize Agent to set up device and connection pools
logger.info("Initializing Agent and connecting to device...")
profile = AgentProfile(name="default", from_file="llm-config.override.jsonc")
config = Builders.AgentConfig.with_default_profile(profile).build()
agent = Agent(config=config)
await agent.init()
# 2. Run task using 'flash' profile
logger.info("Starting Flash runner task execution...")
try:
result = await agent.run_task(
goal="Play a video in full screen on YouTube in Chrome", profile="flash"
)
logger.info(f"Flash runner task execution finished! Result: {result}")
except Exception:
logger.exception("Error during Flash task execution:")
finally:
await agent.clean()
if __name__ == "__main__":
asyncio.run(main())
+21
View File
@@ -0,0 +1,21 @@
import time
import uiautomator2 as u2
def test():
device = u2.connect("emulator-5554")
print("Testing send_keys with immediate IME switch off")
device.set_fastinput_ime(True)
device.send_keys("Test 1")
device.set_fastinput_ime(False)
time.sleep(2)
print("Testing send_keys with delay before IME switch off")
device.set_fastinput_ime(True)
device.send_keys("Test 2")
time.sleep(1.0)
device.set_fastinput_ime(False)
if __name__ == "__main__":
test()
+102
View File
@@ -0,0 +1,102 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
from pathlib import Path
import sys
# Add root to sys.path
root_dir = Path(__file__).parent.parent
if str(root_dir) not in sys.path:
sys.path.insert(0, str(root_dir))
from adbutils import AdbClient
from artemis.clients.ui_automator_client import UIAutomatorClient
from artemis.context import DeviceContext, DevicePlatform, ArtemisContext
from artemis.controllers.unified_controller import UnifiedMobileController
import pytest
@pytest.mark.asyncio
async def test_actions():
print("=========================================")
print(" Physical Action Direct Validation Testbench")
print("=========================================")
adb = AdbClient(host="localhost", port=5037)
devices = adb.device_list()
if not devices:
print("❌ Error: No active Android emulator or physical device detected.")
return
device_id = devices[0].serial
print(f"✅ Target device locked: {device_id}")
ui_client = UIAutomatorClient(device_id=device_id)
print("🔄 Connecting to UIAutomator2 and syncing screen properties...")
try:
ui_data = ui_client.get_screen_data()
w, h = ui_data.width, ui_data.height
print(f"✅ Screen sync ready, physical resolution: {w}x{h}")
except Exception as e:
print(f"❌ Screen properties sync failed: {e}")
return
ctx = ArtemisContext(
trace_id="physical-test-bench",
device=DeviceContext(
host_platform="LINUX",
mobile_platform=DevicePlatform.ANDROID,
device_id=device_id,
device_width=w,
device_height=h,
),
adb_client=adb,
ui_adb_client=ui_client,
)
ctrl = UnifiedMobileController(ctx)
print("\n[Task 1] Extract Live UI Hierarchy...")
try:
elems = await ctrl.get_ui_elements()
print(f"✅ Parsed UI element count: {len(elems)}")
except Exception as e:
print(f"❌ Failed to extract UI hierarchy: {e}")
print("\n[Task 2] Capture Live Screenshot...")
try:
shot = await ctrl.take_screenshot()
print(f"✅ Screenshot captured, Base64 payload length: {len(shot)}")
except Exception as e:
print(f"❌ Failed to capture screenshot: {e}")
print("\n[Task 3] Simulate Drag/Swipe Down...")
try:
# Swipe up from bottom to top
err = await ctrl.swipe_coords(w // 2, int(h * 0.8), w // 2, int(h * 0.2), duration=400)
if err:
print(f"❌ Swipe rejected by controller: {err}")
else:
print("✅ Swipe command injected successfully!")
except Exception as e:
print(f"❌ Swipe execution exception: {e}")
print("\n🧹 Safely releasing connection and driver handles...")
await ctrl.cleanup()
print("🎉 Direct physical validation tests completed successfully!")
if __name__ == "__main__":
asyncio.run(test_actions())
@@ -0,0 +1,45 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Integration tests for ReactiveRunner and Pipeline."""
import pytest
from artemis.core.context import ExecutionContext
from artemis.core.state import ExecutionStatus
from artemis.drivers.mock.mock_driver import MockDeviceDriver
from artemis.engine.pipeline import Pipeline
from artemis.engine.reactive_runner import ReactiveRunner
@pytest.mark.asyncio
async def test_reactive_runner_execution():
"""Verify ReactiveRunner executes workflow cycles on mock driver."""
ctx = ExecutionContext(task_goal="Turn on Bluetooth in Settings", device_id="test-emulator")
driver = MockDeviceDriver(device_id="test-emulator")
runner = ReactiveRunner(ctx=ctx, driver=driver)
state = await runner.run(max_turns=5)
assert state.status == ExecutionStatus.SUCCESS
assert state.current_turn >= 1
assert len(state.steps) >= 1
assert state.steps[0].action_name == "wait_for_delay"
@pytest.mark.asyncio
async def test_pipeline_dispatch():
"""Verify Pipeline executes with default mock driver."""
state = await Pipeline.execute(goal="Open YouTube and play video", profile="flash")
assert state.status == ExecutionStatus.SUCCESS
assert state.task_goal == "Open YouTube and play video"
+21
View File
@@ -0,0 +1,21 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for checker tools.
The Checker agent does not define its own specific exposed tools for testing.
It primarily reuses common tools (e.g. from scratchpad) and its behavior is
tested at the agent level. Therefore, there are no specific checker tools to
test here.
"""
+95
View File
@@ -0,0 +1,95 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest.mock import MagicMock
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
import pytest
INPUTS_DIR = Path(__file__).parent / "inputs"
@pytest.fixture(scope="session")
def inputs_dir():
return INPUTS_DIR
@pytest.fixture
def mock_adb_client():
client = MagicMock()
# Dummy outputs for commands and physical interactions that tools might use
client.shell.return_value = "dummy adb output"
# Basic UI hierarchy for mobile read_hierarchy tool
client.get_hierarchy.return_value = (
"<?xml version='1.0' encoding='UTF-8'?><hierarchy><node index='0'"
" text='dummy'/></hierarchy>"
)
return client
@pytest.fixture
def artemis_context(inputs_dir, mock_adb_client):
"""Provides a ArtemisContext with loaded database and actual side-effects,
except the ADB connection is customized to read from fixtures instead of a
real device.
"""
ctx = MagicMock(spec=ArtemisContext)
ctx.execution_setup = MagicMock()
ctx.execution_setup.traces_path = str(inputs_dir)
# Initialize real DataEngine if the fixture database exists
db_path = inputs_dir / "data_engine.db"
if db_path.exists():
engine = DataEngine(ctx)
engine.start_session(goal="Test goal")
import uuid
engine.current_step_id = uuid.uuid4()
ctx.data_engine = engine
else:
ctx.data_engine = None
ctx.ui_adb_client = mock_adb_client
# Fake device representation
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
# Trace output directory setup to point inside inputs or a dedicated test folder
trace_out = inputs_dir / "trace_output"
trace_out.mkdir(exist_ok=True)
ctx.trace_dir = trace_out
ctx.agent_config = MagicMock()
ctx.agent_config.blacklisted_tools = {}
return ctx
@pytest.fixture
def mock_state(inputs_dir):
"""Provides a dummy graph State with realistic artifact paths for tools that need state injection."""
from artemis.graph.state import State
state = MagicMock(spec=State)
state.latest_screenshot = str(inputs_dir / "screenshot.jpg")
state.video_path = str(inputs_dir / "recording.mp4")
state.indexed_points = []
state.indexed_elements = []
return state
@@ -0,0 +1,128 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from artemis.tools.diagnoser_submit_answer_tool import get_submit_answer_tool
def test_diagnoser_submit_answer_tool(artemis_context):
tool = get_submit_answer_tool(artemis_context)
# Execute the tool with some typical payload
result = tool.invoke(
{
"analysis": "The problem is caused by a network timeout.",
"actionable_steps": ["Check network connection", "Restart the server"],
}
)
# Assert that the tool executes without raising errors and produces expected output structure
assert result == "Answer submitted successfully."
assert "error" not in result.lower()
def test_submit_answer_tool_subclass_and_registry():
"""Verify SubmitAnswerTool is an ArtemisTool subclass and registered in ToolRegistry."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.diagnoser_submit_answer_tool import (
DiagnoserSubmitAnswerTool,
SubmitAnswer,
SubmitAnswerArgs,
SubmitAnswerTool,
submit_answer,
)
assert issubclass(SubmitAnswerTool, ArtemisTool)
assert issubclass(SubmitAnswer, ArtemisTool)
assert issubclass(DiagnoserSubmitAnswerTool, ArtemisTool)
assert isinstance(submit_answer, ArtemisTool)
assert isinstance(submit_answer, SubmitAnswerTool)
assert submit_answer.name == "submit_answer"
assert submit_answer.category == "custom"
assert submit_answer.args_schema == SubmitAnswerArgs
# Registry lookup
reg_tool = ToolRegistry.get("submit_answer")
assert reg_tool is not None
assert isinstance(reg_tool, SubmitAnswerTool)
# GenAI FunctionDeclaration export
declaration = submit_answer.to_genai_declaration()
assert declaration.name == "submit_answer"
assert "analysis" in declaration.parameters.properties
assert "actionable_steps" in declaration.parameters.properties
def test_ask_diagnoser_tool_subclass_and_registry():
"""Verify AskDiagnoserTool is an ArtemisTool subclass and registered in ToolRegistry."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.diagnostic_tool import (
AskDiagnoser,
AskDiagnoserArgs,
AskDiagnoserTool,
DiagnosticTool,
ask_diagnoser,
)
assert issubclass(AskDiagnoserTool, ArtemisTool)
assert issubclass(AskDiagnoser, ArtemisTool)
assert issubclass(DiagnosticTool, ArtemisTool)
assert isinstance(ask_diagnoser, ArtemisTool)
assert isinstance(ask_diagnoser, AskDiagnoserTool)
assert ask_diagnoser.name == "ask_diagnoser"
assert ask_diagnoser.category == "custom"
assert ask_diagnoser.args_schema == AskDiagnoserArgs
# Registry lookup
reg_tool = ToolRegistry.get("ask_diagnoser")
assert reg_tool is not None
assert isinstance(reg_tool, AskDiagnoserTool)
# GenAI FunctionDeclaration export
declaration = ask_diagnoser.to_genai_declaration()
assert declaration.name == "ask_diagnoser"
assert "query" in declaration.parameters.properties
def test_analyze_logs_tool_subclass_and_registry():
"""Verify AnalyzeLogsTool is an ArtemisTool subclass and registered in ToolRegistry."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.log_tool import (
AnalyzeLogs,
AnalyzeLogsArgs,
AnalyzeLogsTool,
LogTool,
analyze_logs,
)
assert issubclass(AnalyzeLogsTool, ArtemisTool)
assert issubclass(AnalyzeLogs, ArtemisTool)
assert issubclass(LogTool, ArtemisTool)
assert isinstance(analyze_logs, ArtemisTool)
assert isinstance(analyze_logs, AnalyzeLogsTool)
assert analyze_logs.name == "analyze_logs"
assert analyze_logs.category == "custom"
assert analyze_logs.args_schema == AnalyzeLogsArgs
# Registry lookup
reg_tool = ToolRegistry.get("analyze_logs")
assert reg_tool is not None
assert isinstance(reg_tool, AnalyzeLogsTool)
# GenAI FunctionDeclaration export
declaration = analyze_logs.to_genai_declaration()
assert declaration.name == "analyze_logs"
assert "specific_query" in declaration.parameters.properties
@@ -0,0 +1,83 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from artemis.agents.explorer.explorer import Explorer
import pytest
@pytest.fixture
def explorer_instance(artemis_context, mock_state):
explorer = Explorer(artemis_context)
explorer.image_name = "test_img"
explorer.screenshot_path = mock_state.latest_screenshot
# Initialize the image pool with img_0 required by processing tools
explorer.image_pool = {
"img_0": {
"path": mock_state.latest_screenshot,
"transform": {
"offset_x": 0.0,
"offset_y": 0.0,
"scale_x": 1.0,
"scale_y": 1.0,
},
}
}
return explorer
@pytest.mark.asyncio
async def test_exec_detect_objects(explorer_instance):
result = await explorer_instance.exec_detect_objects(
["button", "text"], target_image_id="img_0"
)
assert isinstance(result, dict)
assert "text" in result
assert "image_path" in result
@pytest.mark.asyncio
async def test_exec_ask_perception_tool(explorer_instance):
result = await explorer_instance.exec_ask_perception_tool("login", 500, 500, ["button"])
assert isinstance(result, dict)
assert "text" in result
assert "image_paths" in result
@pytest.mark.asyncio
async def test_exec_get_ocr_list(explorer_instance):
result = await explorer_instance.exec_get_ocr_list()
assert isinstance(result, dict)
assert "text" in result
assert "image_path" in result
@pytest.mark.asyncio
async def test_exec_inspect_region(explorer_instance):
# Depending on cv2 logic, it might return an error string if coordinates are out of bounds or image is missing
# But it shouldn't raise unhandled exceptions
result = await explorer_instance.exec_inspect_region(100, 100, 400, 400, 2.0)
assert isinstance(result, dict)
assert "text" in result
assert "image_path" in result
@pytest.mark.asyncio
async def test_exec_ask_image_processor(explorer_instance):
result = await explorer_instance.exec_ask_image_processor(
"Invert colors", target_image_id="img_0"
)
assert isinstance(result, dict)
assert "text" in result
assert "image_paths" in result
@@ -0,0 +1,101 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from artemis.agents.history_analyzer.history_analyzer import HistoryAnalyzer
from artemis.config import parse_llm_config
import pytest
@pytest.mark.asyncio
async def test_exec_get_step_details(artemis_context):
"""Test the exec_get_step_details method with manually configured steps."""
analyzer = HistoryAnalyzer(artemis_context)
analyzer.history_steps = [
{
"step_number": 0,
"relative_time": 0.0,
"summary": "Initial step",
"action_taken": "none",
"operator_raw_thinking": "Starting up",
"last_execution_result": "Success",
},
{
"step_number": 1,
"relative_time": 5.0,
"summary": "Second step",
"action_taken": "click",
"operator_raw_thinking": "Clicking button",
"last_execution_result": "Success",
},
]
first_step_num = 0
result = analyzer.exec_get_step_details(first_step_num, first_step_num)
assert "Error:" not in result
assert "No steps found" not in result
parsed = json.loads(result)
assert isinstance(parsed, list)
assert len(parsed) == 1
assert parsed[0]["step_number"] == first_step_num
assert "summary" in parsed[0]
@pytest.mark.asyncio
async def test_get_step_details_tool_with_fixture(artemis_context):
"""Test _get_step_details_tool against real steps loaded from the inputs/data_engine.db fixture."""
if not artemis_context.data_engine:
pytest.skip("DataEngine fixture not initialized")
artemis_context.data_engine.current_session_id = "1a7fc344-ea69-4b1a-b066-d8a667502b8c"
steps = artemis_context.data_engine.get_agent_friendly_steps()
assert len(steps) > 0
analyzer = HistoryAnalyzer(artemis_context)
tool = analyzer._get_step_details_tool(steps)
result = tool.invoke({"start_step": 1, "end_step": 3})
assert isinstance(result, str)
assert "Error:" not in result
assert "No steps found" not in result
parsed = json.loads(result)
assert isinstance(parsed, list)
assert len(parsed) == 3
assert parsed[0]["step_number"] == 1
assert "YouTube" in parsed[0]["summary"]
@pytest.mark.asyncio
async def test_history_analyzer_blackbox_run(artemis_context):
"""End-to-end blackbox test of HistoryAnalyzer.run(query) using real steps from inputs/data_engine.db and real LLM calls."""
if not artemis_context.data_engine:
pytest.skip("DataEngine fixture not initialized")
artemis_context.llm_config = parse_llm_config()
artemis_context.data_engine.current_session_id = "1a7fc344-ea69-4b1a-b066-d8a667502b8c"
analyzer = HistoryAnalyzer(artemis_context)
query = (
"What application did the operator open and what query did they search"
" for? Briefly summarize the session steps."
)
result = await analyzer.run(query)
assert isinstance(result, str)
assert len(result) > 0
assert "Error:" not in result
assert "No history recorded" not in result
+19
View File
@@ -0,0 +1,19 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This file is intentionally empty.
The hopper agent does not expose any tools directly. It is invoked
programmatically by other tools (e.g., launch_app_tool) as an LLM utility.
"""
@@ -0,0 +1,43 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from artemis.tools.image_processor_tool import get_ask_image_processor_tool
import pytest
@pytest.mark.asyncio
async def test_ask_image_processor_tool(artemis_context, mock_state):
"""Tests the ask_image_processor tool as a blackbox.
Allows actual side effects and full lifecycle execution (including real LLM
API calls). Uses shared pytest fixtures for realistic context.
"""
tool = get_ask_image_processor_tool(artemis_context)
# Passing a simple visual reasoning instruction.
# The tool will use target_image_id="img_0" corresponding to the artifact in mock_state.
result = await tool.ainvoke(
{
"instruction": "Identify the primary colors present in this image.",
"target_image_id": "img_0",
"state": mock_state,
}
)
# Assert execution without errors and produces expected output structure
assert result is not None
assert isinstance(result, dict)
assert "outputs" in result
assert "summary" in result
assert isinstance(result["outputs"], list)
Binary file not shown.
@@ -0,0 +1,61 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from artemis.agents.log_analyzer.log_analyzer import LogAnalyzerNode
import pytest
@pytest.mark.asyncio
async def test_spawn_log_reader(artemis_context, mock_state):
"""Test the spawn_log_reader tool exposed by LogAnalyzerNode."""
from artemis.sdk.utils import load_llm_config_override
from pathlib import Path
node = LogAnalyzerNode(ctx=artemis_context)
artemis_context.llm_config = load_llm_config_override(Path("llm-config.json"))
if artemis_context.data_engine:
artemis_context.data_engine.start_session("test_session")
tool = node._get_spawn_log_reader_tool()
# Let the tool run its lifecycle. It will use the real LLM APIs.
# Note: State injection is normally handled by LangGraph/tool invocation,
# but since we are testing the coroutine directly, we can pass it manually.
result = await tool.coroutine(
specific_query=("Find any errors related to network timeout in the logs."),
state=mock_state,
)
assert result is not None
assert isinstance(result, (str, list))
assert len(result) > 0
@pytest.mark.asyncio
async def test_log_analyzer_run(artemis_context, mock_state):
"""Test LogAnalyzerNode.run end-to-end execution with grounding configuration."""
from artemis.config.llm import parse_llm_config
node = LogAnalyzerNode(ctx=artemis_context)
artemis_context.llm_config = parse_llm_config()
if artemis_context.data_engine:
artemis_context.data_engine.start_session("test_log_analyzer_session")
result = await node.run(
prompt="Analyze recent error logs for potential crash or exception tags.",
state=mock_state,
)
assert result is not None
assert isinstance(result, (str, list))
assert len(result) > 0
@@ -0,0 +1,58 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for object_detector universal tools."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.object_detection_tool import (
ObjectDetection,
ObjectDetectionArgs,
ObjectDetectionTool,
ObjectDetectorTool,
OperatorObjectDetection,
OperatorObjectDetectionArgs,
OperatorObjectDetectionTool,
OperatorObjectDetectorTool,
object_detection,
operator_object_detection,
)
def test_object_detector_tool_subclass_and_registry():
"""Verify ObjectDetectionTool is an ArtemisTool subclass and registered in ToolRegistry."""
assert issubclass(ObjectDetectionTool, ArtemisTool)
assert issubclass(ObjectDetection, ArtemisTool)
assert issubclass(ObjectDetectorTool, ArtemisTool)
assert issubclass(OperatorObjectDetectionTool, ArtemisTool)
assert issubclass(OperatorObjectDetectionTool, ObjectDetectionTool)
assert issubclass(OperatorObjectDetection, ArtemisTool)
assert issubclass(OperatorObjectDetectorTool, ArtemisTool)
assert isinstance(object_detection, ArtemisTool)
assert isinstance(object_detection, ObjectDetectionTool)
assert isinstance(operator_object_detection, ArtemisTool)
assert isinstance(operator_object_detection, OperatorObjectDetectionTool)
assert object_detection.name == "object_detection"
assert object_detection.category == "perception"
assert object_detection.args_schema == ObjectDetectionArgs
assert operator_object_detection.name == "object_detection"
assert operator_object_detection.category == "perception"
assert operator_object_detection.args_schema == OperatorObjectDetectionArgs
# Registry lookup
reg_tool = ToolRegistry.get("object_detection")
assert reg_tool is not None
assert isinstance(reg_tool, ObjectDetectionTool)
@@ -0,0 +1,68 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from langchain_core.tools import BaseTool
from artemis.agents.operator.operator import OperatorNode
import pytest
@pytest.mark.asyncio
async def test_operator_tools(artemis_context, mock_state):
"""Test all tools exposed by the OperatorNode."""
node = OperatorNode(ctx=artemis_context)
click_tool = node._get_click_tool()
assert isinstance(click_tool, BaseTool)
res = click_tool.invoke({"target": 1, "times": 1, "delay_ms": 100})
assert res == "Action Recorded"
input_text_tool = node._get_input_text_tool()
assert isinstance(input_text_tool, BaseTool)
res = input_text_tool.invoke({"text": "hello", "target": 1, "clear_exist": True})
assert res == "Action Recorded"
swipe_tool = node._get_swipe_tool()
assert isinstance(swipe_tool, BaseTool)
res = swipe_tool.invoke({"gesture": "up"})
assert res == "Action Recorded"
press_key_tool = node._get_press_key_tool()
assert isinstance(press_key_tool, BaseTool)
res = press_key_tool.invoke({"key": "ENTER"})
assert res == "Action Recorded"
manage_app_tool = node._get_manage_app_tool()
assert isinstance(manage_app_tool, BaseTool)
res = manage_app_tool.invoke({"action": "launch", "app_name": "Settings"})
assert res == "Action Recorded"
wait_for_delay_tool = node._get_wait_for_delay_tool()
assert isinstance(wait_for_delay_tool, BaseTool)
res = wait_for_delay_tool.invoke({"time_in_ms": 1000})
assert res == "Action Recorded"
long_press_tool = node._get_long_press_tool()
assert isinstance(long_press_tool, BaseTool)
res = long_press_tool.invoke({"target": 1, "duration": 1000})
assert res == "Action Recorded"
wait_for_text_tool = node._get_wait_for_text_tool()
assert isinstance(wait_for_text_tool, BaseTool)
res = wait_for_text_tool.invoke({"text": "Success", "state": "visible", "timeout_ms": 5000})
assert res == "Action Recorded"
reply_tool = node._get_reply_to_checker_tool()
assert isinstance(reply_tool, BaseTool)
res = reply_tool.invoke({"reasoning": "This is a test reply."})
assert isinstance(res, str)
@@ -0,0 +1,61 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from artemis.agents.outputter.tools import (
get_search_history_tool,
get_step_details_tool,
get_step_screenshot_tool,
)
from artemis.context import ArtemisContext
import pytest
@pytest.fixture
def history_steps(artemis_context: ArtemisContext):
if artemis_context.data_engine:
return artemis_context.data_engine.get_agent_friendly_steps()
return []
def test_get_step_details_tool(history_steps):
"""Test get_step_details_tool to ensure it runs without errors."""
tool = get_step_details_tool(history_steps)
result = tool.invoke({"start_step": 1, "end_step": 5})
assert isinstance(result, str)
assert len(result) > 0
def test_get_step_screenshot_tool(artemis_context: ArtemisContext, history_steps):
"""Test get_step_screenshot_tool to ensure it handles requests gracefully."""
tool = get_step_screenshot_tool(artemis_context, history_steps)
result = tool.invoke({"step_number": 1})
if isinstance(result, list):
assert len(result) == 2
assert result[0]["type"] == "text"
assert result[1]["type"] == "image_url"
else:
assert isinstance(result, str)
assert len(result) > 0
def test_get_search_history_tool(artemis_context: ArtemisContext):
"""Test get_search_history_tool to ensure it can search through history."""
tool = get_search_history_tool(artemis_context)
result = tool.invoke({"query": "test"})
assert isinstance(result, str)
assert len(result) > 0
+17
View File
@@ -0,0 +1,17 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# The planner agent does not have exposed tools of its own to test here.
# It uses shared tools from artemis.tools.scratchpad, which are tested in their respective modules.
@@ -0,0 +1,16 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# The summarizer agent does not have exposed tools of its own to test here.
@@ -0,0 +1,75 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
from pathlib import Path
from artemis.agents.validator.failure_analyzer import ValidationErrorCategory
from artemis.agents.validator.validator import ValidatorNode
import pytest
@pytest.mark.asyncio
async def test_safety_net_validation(artemis_context, mock_state):
"""Test the safety_net_validation tool."""
node = ValidatorNode(artemis_context)
action_item = {
"action": "tap",
"coordinates": [100, 200],
"target_text": "Submit",
}
# We pass session=None so it relies on fallback or fails gracefully
# We assert it runs without unhandled exceptions.
try:
passed, category, reason = await node._validate_action_precondition(
session=None, action_item=action_item, state=mock_state
)
except Exception as e:
pytest.fail(f"Tool raised an exception: {e}")
assert isinstance(passed, bool)
assert isinstance(category, ValidationErrorCategory)
assert isinstance(reason, str)
@pytest.mark.asyncio
async def test_safety_net_pixel_validation(artemis_context, mock_state):
"""Test the safety_net_pixel_validation tool."""
node = ValidatorNode(artemis_context)
action_item = {"action": "tap", "coordinates": [100, 200]}
# Read the real screenshot from the mock_state to provide as pre_screenshot_b64
screenshot_path = mock_state.latest_screenshot
pre_screenshot_b64 = "dummy_base64"
if screenshot_path and Path(screenshot_path).exists():
with open(screenshot_path, "rb") as f:
pre_screenshot_b64 = base64.b64encode(f.read()).decode("utf-8")
try:
passed, category, reason = await node._validate_action_precondition_pixel(
session=None,
action_item=action_item,
pre_screenshot_b64=pre_screenshot_b64,
original_coords=[100, 200],
state=mock_state,
)
except Exception as e:
pytest.fail(f"Tool raised an exception: {e}")
assert isinstance(passed, bool)
assert isinstance(category, ValidationErrorCategory)
assert isinstance(reason, str)
@@ -0,0 +1,103 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
@pytest.mark.asyncio
async def test_video_analyzer_tools(artemis_context, mock_state, inputs_dir):
artemis_context.device.mobile_platform = "android"
artemis_context.adb_client = artemis_context.ui_adb_client
agent = VideoAnalyzer(ctx=artemis_context)
agent.local_files_to_cleanup = set()
agent.local_dirs_to_cleanup = set()
agent.cloud_files_to_cleanup = set()
agent.blackboard_entries = []
agent.sub_system_prompt = "You are a sub agent"
agent.audio_system_prompt = "You are an audio sub agent"
agent.model_name = "gemini-3.6-flash"
# We must provide a valid FunctionDeclaration for the tool otherwise Pydantic will complain
from google.genai import types
agent.submit_answer_tool = types.FunctionDeclaration(
name="submit_answer",
description=("Submit the final findings for the requested segment analysis."),
parameters=types.Schema(
type=types.Type.OBJECT,
properties={
"summary": types.Schema(type=types.Type.STRING, description="summary"),
"analysis": types.Schema(type=types.Type.STRING, description="analysis"),
"timeline_events": types.Schema(
type=types.Type.ARRAY,
items=types.Schema(
type=types.Type.OBJECT,
properties={
"start_time": types.Schema(type=types.Type.NUMBER),
"end_time": types.Schema(type=types.Type.NUMBER),
"transcription": types.Schema(type=types.Type.STRING),
"confidence_score": types.Schema(type=types.Type.NUMBER),
"verification_timestamp_secs": types.Schema(type=types.Type.NUMBER),
},
),
),
},
),
)
import google.genai as genai
from artemis.config import settings
agent.client = genai.Client(
api_key=settings.GOOGLE_API_KEY.get_secret_value() if settings.GOOGLE_API_KEY else None
)
artemis_context._genai_client = agent.client
mock_session = MagicMock()
mock_session.local_video_path = Path(inputs_dir) / "recording.mp4"
mock_session.start_time = 0.0
mock_session.data_engine_start_time = None
mock_session.process = MagicMock()
mock_session.process.returncode = None
mock_session.android_video_segments = []
mock_session.android_segment_index = 0
from unittest.mock import AsyncMock
sample_video = Path(inputs_dir) / "recording.mp4"
mock_driver = MagicMock()
mock_driver.stop_video_recording = AsyncMock(return_value=sample_video)
artemis_context._active_driver = mock_driver
with patch(
"artemis.controllers.android_controller.get_active_session",
return_value=mock_session,
):
result = await agent.exec_extract_segment_metadata(start_time=0.0, end_time=1.0)
assert isinstance(result, str)
assert len(result) > 0
# Test sub-agent with extremely short audio query to not waste resources and time
audio_result = await agent.exec_analyze_audio_only(
start_time=0.0,
end_time=1.0,
specific_query="Reply with a short summary only.",
)
assert isinstance(audio_result, str)
+98
View File
@@ -0,0 +1,98 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.checker.checker import CheckerResult, run_async_check
from artemis.context import ArtemisContext
import pytest
@pytest.mark.asyncio
async def test_run_async_check(tmp_path):
notes_dir = tmp_path / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
# Put a dummy task plan file
(notes_dir / "task_plan.md").write_text("Plan")
# Put a dummy verification_chat file
import hashlib
subgoal_text = "Open WhatsApp"
s_hash = hashlib.md5(subgoal_text.encode("utf-8")).hexdigest()
(notes_dir / f"verification_chat_{s_hash}.json").write_text("[]")
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = str(tmp_path)
mock_ctx.data_engine.get_agent_friendly_steps.return_value = []
mock_ctx.execution_setup = None
# Mock controller for screen data
mock_controller = MagicMock()
mock_controller.take_screenshot = AsyncMock(return_value="dummy_screenshot_base64")
from artemis.controllers.device_controller import ScreenDataResponse
mock_screen_data = ScreenDataResponse(
base64="dummy_screenshot_base64",
elements=[],
width=1080,
height=2400,
platform="android",
)
mock_controller.get_screen_data = AsyncMock(return_value=mock_screen_data)
# Mock LLM response with with_structured_output
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.content = "Task completed successfully."
mock_response.tool_calls = []
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
mock_llm.bind_tools.return_value = mock_llm
mock_structured_llm = MagicMock()
mock_structured_llm.ainvoke = AsyncMock(
return_value=CheckerResult(success=True, reason="Task completed successfully.")
)
mock_llm.with_structured_output.return_value = mock_structured_llm
with (
patch(
"artemis.agents.checker.checker.UnifiedMobileController",
return_value=mock_controller,
),
patch("artemis.agents.checker.checker.get_llm", return_value=mock_llm),
patch(
"artemis.utils.task_tree.build_plan_and_history",
return_value="Task tree",
),
):
result = await run_async_check(
mock_ctx,
subgoal_text=subgoal_text,
subgoal_hash=s_hash,
raw_perception_data={"screenshot_b64": "dummy", "width": 1080, "height": 2400},
latest_ui_hierarchy=[],
)
assert result["status"] == "success"
assert result["reason"] == "Task completed successfully."
assert mock_llm.astream.called
assert mock_llm.with_structured_output.called
+308
View File
@@ -0,0 +1,308 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, Mock, patch
from langchain_core.messages import AIMessage, ToolMessage
from langgraph.types import Command
from artemis.agents.diagnoser.diagnoser import Diagnoser
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
import pytest
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
ctx.data_engine = Mock()
ctx.data_engine.base_dir = "/tmp"
ctx.data_engine.get_agent_friendly_steps.return_value = []
# Mock execution_setup
mock_setup = Mock()
mock_setup.video_recording_tools_enabled = True
ctx.execution_setup = mock_setup
return ctx
@pytest.fixture
def mock_state():
state = Mock(spec=State)
state.initial_goal = "Test goal"
state.latest_screenshot = None
state.latest_ui_hierarchy = []
return state
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.agents.diagnoser.diagnoser.get_video_analyzer_tool")
@patch("artemis.agents.diagnoser.diagnoser.get_analyze_logs_tool")
@pytest.mark.asyncio
async def test_diagnostic_agent_simple_call(
mock_get_log_tool,
mock_get_video_tool,
mock_get_llm,
mock_context,
mock_state,
):
"""Test that DiagnosticAgent calls LLM and returns result."""
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
mock_response = AIMessage(content="I diagnosed the issue: it is a network error.")
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock tools
mock_video_tool = Mock()
mock_video_tool.name = "video_analyzer"
mock_get_video_tool.return_value = mock_video_tool
mock_log_tool = Mock()
mock_log_tool.name = "analyze_logs"
mock_get_log_tool.return_value = mock_log_tool
agent = Diagnoser(mock_context)
result = await agent.run("Diagnose why the app crashed", mock_state)
assert result == "I diagnosed the issue: it is a network error."
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.agents.diagnoser.diagnoser.get_video_analyzer_tool")
@patch("artemis.agents.diagnoser.diagnoser.get_analyze_logs_tool")
@pytest.mark.asyncio
async def test_diagnostic_agent_tool_calls(
mock_get_log_tool,
mock_get_video_tool,
mock_get_llm,
mock_context,
mock_state,
):
"""Test that DiagnosticAgent handles tool calls."""
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
# Simulate response with tool calls
mock_response = AIMessage(
content="I need to check logs and video.",
tool_calls=[
{
"name": "analyze_logs",
"args": {"specific_query": "Check for crashes"},
"id": "call_1",
},
{
"name": "video_analyzer",
"args": {
"time_description": "around 10s",
"purpose": "Check UI",
},
"id": "call_2",
},
],
)
# Second call returns final answer
mock_final_response = AIMessage(content="Diagnosis complete.")
responses = [mock_response, mock_final_response]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock tools
mock_log_tool = AsyncMock()
mock_log_tool.name = "analyze_logs"
mock_log_tool.args = {"specific_query": None, "state": None}
mock_log_tool.ainvoke.return_value = "Logs show no crash."
mock_get_log_tool.return_value = mock_log_tool
mock_video_tool = AsyncMock()
mock_video_tool.name = "video_analyzer"
mock_video_tool.args = {
"time_description": None,
"purpose": None,
"state": None,
}
mock_command = Command(
update={
VALIDATOR_MESSAGES_KEY: [
ToolMessage(content="Video shows UI loading.", tool_call_id="call_2")
]
}
)
mock_video_tool.ainvoke.return_value = mock_command
mock_get_video_tool.return_value = mock_video_tool
agent = Diagnoser(mock_context)
result = await agent.run("Diagnose issue", mock_state)
assert result == "Diagnosis complete."
mock_log_tool.ainvoke.assert_called_once_with(
{"specific_query": "Check for crashes", "state": mock_state}
)
mock_video_tool.ainvoke.assert_called_once()
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.agents.diagnoser.diagnoser.get_video_analyzer_tool")
@patch("artemis.agents.diagnoser.diagnoser.get_analyze_logs_tool")
@pytest.mark.asyncio
async def test_diagnostic_agent_no_video_tool_when_disabled(
mock_get_log_tool,
mock_get_video_tool,
mock_get_llm,
mock_context,
mock_state,
):
"""Test that Diagnoser does not bind video tool when disabled."""
# Set video recording to False
mock_context.execution_setup.video_recording_tools_enabled = False
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
mock_response = AIMessage(content="Diagnosis complete.")
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock tools
mock_video_tool = Mock()
mock_video_tool.name = "video_analyzer"
mock_get_video_tool.return_value = mock_video_tool
mock_log_tool = Mock()
mock_log_tool.name = "analyze_logs"
mock_get_log_tool.return_value = mock_log_tool
agent = Diagnoser(mock_context)
await agent.run("Diagnose issue", mock_state)
# Verify bind_tools was called
mock_llm.bind_tools.assert_called_once()
bound_tools = mock_llm.bind_tools.call_args[1]["tools"]
# Verify video_analyzer is NOT in bound tools
tool_names = [t.name for t in bound_tools if hasattr(t, "name")]
assert "video_analyzer" not in tool_names
# Verify log tool IS in bound tools (should still be there)
assert "analyze_logs" in tool_names
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.tools.video_tool.VideoAnalyzer")
@pytest.mark.asyncio
async def test_diagnoser_background_job_lifecycle(
mock_video_analyzer_class, mock_get_llm, mock_state
):
# Setup context
ctx = ArtemisContext.model_construct(
device=Mock(),
llm_config=Mock(),
execution_setup=Mock(video_recording_tools_enabled=True),
)
ctx.data_engine = None
# The Mock VideoAnalyzer agent running inside the background job
mock_agent_instance = AsyncMock()
async def mock_agent_run(time_desc, purpose):
return "Video shows step successfully completed.", "success"
mock_agent_instance.run.side_effect = mock_agent_run
mock_video_analyzer_class.return_value = mock_agent_instance
# Prepare LLM responses
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
# Turn 1: LLM requests tool call
mock_response_1 = AIMessage(
content="Checking the video first.",
tool_calls=[
{
"name": "video_analyzer",
"args": {
"time_description": "from 0s to 5s",
"purpose": "check UI",
},
"id": "call_video_1",
}
],
)
# Turn 2: LLM gets the initial job start message, continues, then background job finishes, and result is injected.
mock_response_2 = AIMessage(content="Final complete diagnosis.")
responses = [mock_response_1, mock_response_2]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock asanitize_update on mock_state
async def mock_asanitize_update(ctx, update, agent):
return update
mock_state.asanitize_update = mock_asanitize_update
# Instantiate the Diagnoser
agent = Diagnoser(ctx)
# We want to use the actual video analyzer tool so it triggers the background task
from artemis.tools.video_tool import get_video_analyzer_tool
real_video_tool = get_video_analyzer_tool(ctx, role="diagnoser")
with patch(
"artemis.tools.log_tool.get_analyze_logs_tool",
return_value=Mock(name="analyze_logs"),
):
with patch(
"artemis.tools.video_tool.get_video_analyzer_tool",
return_value=real_video_tool,
):
result = await agent.run("Diagnose video issue", mock_state)
assert result == "Final complete diagnosis."
# Verify job registry
assert len(ctx.background_jobs) == 1
job_id = list(ctx.background_jobs.keys())[0]
job = ctx.background_jobs[job_id]
assert job["tool_call_id"] == "call_video_1"
assert job["status"] == "completed"
assert job["result"] == "Video shows step successfully completed."
assert job["consumed"] is True
@@ -0,0 +1,343 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, Mock, patch
from adbutils import AdbClient
from langchain_core.messages import AIMessage, ToolMessage
from artemis.agents.diagnoser.diagnoser import Diagnoser
from artemis.context import ArtemisContext
from artemis.graph.state import State
import pytest
@pytest.fixture(scope="session", autouse=True)
def ensure_device_connected():
try:
client = AdbClient()
if not client.device_list():
pytest.skip("No active Android device/emulator found. Skipping diagnostic tool tests.")
except Exception as e:
pytest.skip(f"ADB Client connection failed: {e}. Skipping tests.")
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
ctx.data_engine = Mock()
ctx.data_engine.base_dir = "/tmp"
ctx.data_engine.get_agent_friendly_steps.return_value = []
# Mock execution_setup
mock_setup = Mock()
mock_setup.video_recording_tools_enabled = True
ctx.execution_setup = mock_setup
return ctx
@pytest.fixture
def mock_state():
state = Mock(spec=State)
state.initial_goal = "Test goal"
state.latest_screenshot = None
state.latest_ui_hierarchy = []
return state
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
@pytest.mark.asyncio
async def test_maestro_uninstall_timeout(
mock_get_ui_tool,
mock_get_log_tool,
mock_get_video_tool,
mock_get_llm,
mock_context,
mock_state,
):
"""Test Maestro Uninstall Timeout / Blocker Failure."""
# Mock LLM calls
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
mock_response_1 = AIMessage(
content="I need to check the UI hierarchy.",
tool_calls=[
{
"name": "get_ui_hierarchy",
"args": {},
"id": "call_hierarchy",
}
],
)
mock_response_2 = AIMessage(
content=(
"Diagnoser identifies the blocker failure and recommends restarting"
" ADB/uninstalling conflicting tools manually."
)
)
responses = [mock_response_1, mock_response_2]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock tool
mock_ui_tool = AsyncMock()
mock_ui_tool.name = "get_ui_hierarchy"
mock_ui_tool.args = {}
mock_ui_tool.ainvoke.return_value = "Error: pm list packages timed out"
mock_get_ui_tool.return_value = mock_ui_tool
# Run agent
agent = Diagnoser(mock_context)
result = await agent.run("Diagnose why the app crashed", mock_state)
assert "recommends restarting ADB" in result
assert "uninstalling conflicting tools" in result
# Assert get_ui_hierarchy was called
mock_ui_tool.ainvoke.assert_called_once()
# Assert that the ToolMessage was sent back to LLM
assert len(mock_llm.astream.call_args_list) == 2
sent_messages = mock_llm.astream.call_args_list[1][0][0]
tool_message = sent_messages[-2]
assert isinstance(tool_message, ToolMessage)
assert tool_message.tool_call_id == "call_hierarchy"
assert tool_message.content == "Error: pm list packages timed out"
assert tool_message.status == "error"
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
@pytest.mark.asyncio
async def test_uiautomator2_connection_lost(
mock_get_ui_tool,
mock_get_log_tool,
mock_get_video_tool,
mock_get_llm,
mock_context,
mock_state,
):
"""Test UIAutomator2 Daemon Connection Lost."""
# Mock LLM
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
mock_response_1 = AIMessage(
content="Let me check the UI hierarchy.",
tool_calls=[
{
"name": "get_ui_hierarchy",
"args": {},
"id": "call_hierarchy_lost",
}
],
)
mock_response_2 = AIMessage(content="Diagnoser reports connection loss with UIAutomator2.")
responses = [mock_response_1, mock_response_2]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock tool to throw ConnectionError
mock_ui_tool = AsyncMock()
mock_ui_tool.name = "get_ui_hierarchy"
mock_ui_tool.args = {}
mock_ui_tool.ainvoke.side_effect = ConnectionError(
"UIAutomator2 connection lost, reconnecting..."
)
mock_get_ui_tool.return_value = mock_ui_tool
# Run agent
agent = Diagnoser(mock_context)
result = await agent.run("Diagnose crash", mock_state)
assert "reports connection loss" in result
# Assert that tool error is wrapped and returned in the ToolMessage
assert len(mock_llm.astream.call_args_list) == 2
sent_messages = mock_llm.astream.call_args_list[1][0][0]
tool_message = sent_messages[-2]
assert isinstance(tool_message, ToolMessage)
assert tool_message.tool_call_id == "call_hierarchy_lost"
assert "UIAutomator2 connection lost, reconnecting..." in tool_message.content
assert tool_message.status == "error"
# Verify that the trace span recorded the failure
record_trace_calls = mock_context.data_engine.record_trace.call_args_list
tool_trace_calls = [c for c in record_trace_calls if c[1].get("name") == "get_ui_hierarchy"]
failed_trace = next((c for c in tool_trace_calls if c[1].get("status") == "failed"), None)
assert failed_trace is not None
assert "UIAutomator2 connection lost, reconnecting..." in failed_trace[1]["payload"]["error"]
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
@pytest.mark.asyncio
async def test_ui_hierarchy_dump_stall(
mock_get_ui_tool,
mock_get_log_tool,
mock_get_video_tool,
mock_get_llm,
mock_context,
mock_state,
):
"""Test UI Hierarchy Dump Stall (UI Thread Frozen)."""
# Mock LLM
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
mock_response_1 = AIMessage(
content="Let me fetch the UI hierarchy to see what's on screen.",
tool_calls=[
{
"name": "get_ui_hierarchy",
"args": {},
"id": "call_hierarchy_stall",
}
],
)
mock_response_2 = AIMessage(
content=(
"Diagnoser identifies frozen UI and suggests rendering-safe"
" recovery actions (like ADB wake commands or a wait/retry cycle)."
)
)
responses = [mock_response_1, mock_response_2]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock tool
mock_ui_tool = AsyncMock()
mock_ui_tool.name = "get_ui_hierarchy"
mock_ui_tool.args = {}
mock_ui_tool.ainvoke.return_value = (
"Error retrieving UI hierarchy: dump_hierarchy timed out after 3 seconds"
)
mock_get_ui_tool.return_value = mock_ui_tool
# Run agent
agent = Diagnoser(mock_context)
result = await agent.run("Diagnose stall", mock_state)
assert "rendering-safe recovery actions" in result
# Assert ToolMessage was passed back with correct error details
assert len(mock_llm.astream.call_args_list) == 2
sent_messages = mock_llm.astream.call_args_list[1][0][0]
tool_message = sent_messages[-2]
assert isinstance(tool_message, ToolMessage)
assert tool_message.tool_call_id == "call_hierarchy_stall"
assert "dump_hierarchy timed out after 3 seconds" in tool_message.content
assert tool_message.status == "error"
@patch("artemis.agents.diagnoser.diagnoser.get_llm")
@patch("artemis.tools.video_tool.get_video_analyzer_tool")
@patch("artemis.tools.log_tool.get_analyze_logs_tool")
@patch("artemis.agents.diagnoser.diagnoser.get_ui_hierarchy_tool")
@pytest.mark.asyncio
async def test_device_offline(
mock_get_ui_tool,
mock_get_log_tool,
mock_get_video_tool,
mock_get_llm,
mock_context,
mock_state,
):
"""Test Device Offline (ADB Transport Lost)."""
# Mock LLM
mock_llm = Mock()
mock_llm.bind_tools.return_value = mock_llm
mock_response_1 = AIMessage(
content="Let's grab the UI hierarchy.",
tool_calls=[
{
"name": "get_ui_hierarchy",
"args": {},
"id": "call_hierarchy_offline",
}
],
)
mock_response_2 = AIMessage(
content=("The device is offline. Diagnoser flags the offline device state.")
)
responses = [mock_response_1, mock_response_2]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
mock_get_llm.return_value = mock_llm
# Mock tool
mock_ui_tool = AsyncMock()
mock_ui_tool.name = "get_ui_hierarchy"
mock_ui_tool.args = {}
mock_ui_tool.ainvoke.side_effect = RuntimeError("device offline")
mock_get_ui_tool.return_value = mock_ui_tool
# Run agent
agent = Diagnoser(mock_context)
result = await agent.run("Diagnose offline", mock_state)
assert "flags the offline device state" in result
# Assert ToolMessage was passed back with correct error details
assert len(mock_llm.astream.call_args_list) == 2
sent_messages = mock_llm.astream.call_args_list[1][0][0]
tool_message = sent_messages[-2]
assert isinstance(tool_message, ToolMessage)
assert tool_message.tool_call_id == "call_hierarchy_offline"
assert "device offline" in tool_message.content
assert tool_message.status == "error"
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,127 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Universal Explorer and Multi-Model Grounding Engine."""
import json
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import pytest
from langchain_core.messages import AIMessage
from artemis.agents.explorer.explorer import (
UNIVERSAL_EXPLORER_TOOLS,
Explorer,
)
from artemis.config import settings
from artemis.context import ArtemisContext
from artemis.graph.state import State
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
mock_llm_cfg = Mock()
mock_llm_cfg.model = "claude-3-7-sonnet"
mock_llm_cfg.temperature = 0.1
ctx.llm_config.explorer = mock_llm_cfg
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.agent_config = Mock()
ctx.agent_config.blacklisted_tools = {}
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
return ctx
def test_explorer_universal_engine_detection(mock_context):
"""Test that Explorer routes to Universal Engine when non-Gemini model is configured."""
with patch.object(settings, "GOOGLE_API_KEY", None):
agent = Explorer(mock_context)
assert agent.use_native_gemini is False
assert agent.model_name == "claude-3-7-sonnet"
def test_universal_explorer_tool_schemas():
"""Verify schema integrity of Universal Explorer tool declarations."""
assert len(UNIVERSAL_EXPLORER_TOOLS) == 6
names = [t["function"]["name"] for t in UNIVERSAL_EXPLORER_TOOLS]
assert "ask_perception_tool" in names
assert "detect_objects" in names
assert "get_ocr_list" in names
assert "ask_image_processor" in names
assert "inspect_region" in names
assert "submit_answer" in names
@pytest.mark.asyncio
async def test_run_universal_explorer_success(mock_context, tmp_path):
"""Test universal Explorer tool invocation and submit_answer coordinate parsing."""
agent = Explorer(mock_context)
agent.use_native_gemini = False
fake_screenshot = tmp_path / "screenshot.jpg"
fake_screenshot.write_bytes(b"fake_image_bytes")
# Mock tool call response for submit_answer
msg_response = AIMessage(
content="Submitting grounded elements.",
tool_calls=[
{
"name": "submit_answer",
"args": {
"candidates": [
{
"label": "D1",
"coords": [500, 600],
"description": "Settings button",
}
],
"fallback_message": "",
},
"id": "exp_call_1",
}
],
)
mock_llm = MagicMock()
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(return_value=msg_response)
mock_llm.bind_tools.return_value = mock_bound
with patch(
"artemis.agents.explorer.explorer.get_llm",
return_value=mock_llm,
):
mock_state = Mock(spec=State)
res_str = await agent._run_universal(
query="find settings icon",
context_feedback="",
screenshot_path=str(fake_screenshot),
state=mock_state,
minimal_list="",
version="pro",
prompt_template="You are an explorer.",
max_iterations=3,
)
res = json.loads(res_str)
assert len(res["candidates"]) == 1
assert res["candidates"][0]["label"] == "D1"
assert res["candidates"][0]["coords"] == [500, 600]
+213
View File
@@ -0,0 +1,213 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Universal Failure Analyzer."""
import base64
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import pytest
from langchain_core.messages import AIMessage
from artemis.agents.validator.failure_analyzer import (
FailureAnalyzer,
PixelTargetDisappearedStrategy,
TargetDisappearedStrategy,
ValidationErrorCategory,
VALIDATOR_TOOLS_DECLARATION,
)
from artemis.context import ArtemisContext
class DummyState:
def __init__(self, latest_screenshot=None):
self.latest_screenshot = latest_screenshot
self.indexed_points = []
self.indexed_elements = []
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
mock_llm_cfg = Mock()
mock_llm_cfg.model = "gemini-2.5-pro"
mock_llm_cfg.temperature = 0.1
ctx.llm_config.get_agent.return_value = mock_llm_cfg
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
ctx.data_engine.get_agent_friendly_steps.return_value = []
ctx.adb_client = None
ctx.driver = Mock()
return ctx
@pytest.mark.asyncio
async def test_failure_analyzer_cannot_fix(mock_context, tmp_path):
"""Test that FailureAnalyzer natively executes and handles 'cannot_fix' report."""
screenshot_file = tmp_path / "shot.jpg"
screenshot_file.write_bytes(b"dummy_bytes")
state = DummyState(latest_screenshot=str(screenshot_file))
failed_action = {"action": "click", "coordinates": [100, 200]}
error_msg = "Element not found"
analyzer = FailureAnalyzer(mock_context)
mock_llm = MagicMock()
mock_response = AIMessage(
content="I see the error.",
tool_calls=[
{
"name": "report_failure_analysis",
"args": {
"status": "cannot_fix",
"analysis": "The element was definitely not there.",
},
"id": "call_1",
}
],
)
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(return_value=mock_response)
mock_llm.bind_tools.return_value = mock_bound
with (
patch("artemis.agents.validator.failure_analyzer.get_llm", return_value=mock_llm),
patch(
"artemis.agents.validator.failure_analyzer.UnifiedMobileController"
) as mock_controller_cls,
):
mock_controller = Mock()
mock_controller_cls.return_value = mock_controller
mock_device_data = Mock()
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
mock_device_data.elements = []
mock_device_data.width = 1080
mock_device_data.height = 2400
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
mock_controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
result = await analyzer.analyze(
state,
failed_action,
error_msg,
pre_screenshot=base64.b64encode(b"dummy").decode(),
post_screenshot=base64.b64encode(b"dummy").decode(),
)
assert result["status"] == "cannot_fix"
assert "not there" in result["analysis"]
assert result["new_remaining_actions"] == []
def test_report_failure_analysis_tool_declaration():
"""Test that report_failure_analysis tool declaration is correct and matches."""
report_tool = next(
(t for t in VALIDATOR_TOOLS_DECLARATION if t.name == "report_failure_analysis"),
None,
)
assert report_tool is not None
props = report_tool.parameters.get("properties", {})
assert "analysis" in props
assert "single paragraph" in props["analysis"]["description"]
@pytest.mark.asyncio
async def test_failure_analyzer_strategy_routing(mock_context):
"""Test that FailureAnalyzer selects appropriate strategies based on error category."""
analyzer = FailureAnalyzer(mock_context)
strat_disappeared = analyzer._select_strategy(ValidationErrorCategory.TARGET_DISAPPEARED)
assert isinstance(strat_disappeared, TargetDisappearedStrategy)
assert "target_disappeared" in strat_disappeared.get_prompt_template_name()
strat_pixel = analyzer._select_strategy(ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED)
assert isinstance(strat_pixel, PixelTargetDisappearedStrategy)
assert "pixel_target_disappeared" in strat_pixel.get_prompt_template_name()
@pytest.mark.asyncio
async def test_failure_analyzer_click_sequence(mock_context, tmp_path):
"""Test click_sequence execution in FailureAnalyzer."""
screenshot_file = tmp_path / "shot.jpg"
screenshot_file.write_bytes(b"dummy_bytes")
state = DummyState(latest_screenshot=str(screenshot_file))
analyzer = FailureAnalyzer(mock_context)
# Turn 1: Call click_sequence tool with direct coordinates
call_step_1 = AIMessage(
content="Let me click in sequence.",
tool_calls=[
{
"name": "click_sequence",
"args": {"sequence": [[100, 200], [300, 400]]},
"id": "call_seq_1",
}
],
)
# Turn 2: Report failure fixed
call_step_2 = AIMessage(
content="Repaired successfully.",
tool_calls=[
{
"name": "report_failure_analysis",
"args": {
"status": "fixed",
"analysis": "Sequence clicked successfully.",
},
"id": "call_report_2",
}
],
)
mock_llm = MagicMock()
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(side_effect=[call_step_1, call_step_2])
mock_llm.bind_tools.return_value = mock_bound
with (
patch("artemis.agents.validator.failure_analyzer.get_llm", return_value=mock_llm),
patch(
"artemis.agents.validator.failure_analyzer.UnifiedMobileController"
) as mock_controller_cls,
):
mock_controller = Mock()
mock_controller_cls.return_value = mock_controller
mock_device_data = Mock()
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
mock_device_data.elements = []
mock_device_data.width = 1080
mock_device_data.height = 2400
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
mock_controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
mock_controller.tap_at = AsyncMock(return_value=Mock(error=None))
result = await analyzer.analyze(
state,
{"action": "click", "coordinates": [100, 200]},
"Element not found",
pre_screenshot=base64.b64encode(b"dummy").decode(),
post_screenshot=base64.b64encode(b"dummy").decode(),
error_category=ValidationErrorCategory.TARGET_DISAPPEARED,
)
assert result["status"] == "fixed"
assert mock_controller.tap_at.call_count == 2
@@ -0,0 +1,460 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
from unittest.mock import AsyncMock, Mock, patch
from artemis.agents.validator.failure_analyzer import (
_exec_click,
_exec_click_sequence,
_exec_input_text,
_exec_long_press,
_exec_manage_app,
_exec_press_key,
_exec_swipe,
_exec_wait_for_delay,
)
from artemis.context import ArtemisContext
import pytest
class DummyState:
def __init__(self, latest_screenshot=None):
self.latest_screenshot = latest_screenshot
self.indexed_points = []
self.indexed_elements = []
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
ctx.data_engine.get_or_create_image.return_value = "dummy_image_name"
ctx.data_engine.get_image_path.return_value = tmp_path / "dummy_image_name.jpg"
return ctx
@pytest.fixture
def mock_controller():
controller = Mock()
mock_device_data = Mock()
mock_device_data.base64 = base64.b64encode(b"mocked_screenshot").decode()
mock_device_data.elements = []
mock_device_data.width = 1080
mock_device_data.height = 2400
controller.get_screen_data = AsyncMock(return_value=mock_device_data)
controller.get_ui_elements = AsyncMock(return_value=mock_device_data.elements)
controller.tap_at = AsyncMock(return_value=Mock(error=None))
controller.type_text = AsyncMock(return_value=True)
controller.erase_text = AsyncMock(return_value=True)
controller.swipe_coords = AsyncMock(return_value=None)
controller.press_enter = AsyncMock(return_value=True)
controller.go_back = AsyncMock(return_value=True)
controller.go_home = AsyncMock(return_value=True)
controller.press_key = AsyncMock(return_value=True)
controller.terminate_app = AsyncMock(return_value=True)
return controller
# ==========================================
# SLIDE 1: click
# ==========================================
@pytest.mark.asyncio
async def test_click_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_click(
target=[500, 600],
state=state,
controller=mock_controller,
ctx=mock_context,
times=2,
delay_ms=150,
)
assert "Clicked at [500, 600]" in outcome
assert mock_controller.tap_at.call_count == 1
mock_controller.tap_at.assert_any_call(540, 1440, times=2, delay_ms=150)
@pytest.mark.asyncio
async def test_click_input_validation(mock_context, mock_controller):
state = DummyState()
# Coordinates holding non-integer values
outcome, _, _, _ = await _exec_click(
target=["invalid", "invalid"],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during click:" in outcome
@pytest.mark.asyncio
async def test_click_edge_cases(mock_context, mock_controller):
state = DummyState()
# Underflow/overflow coordinates should be clamped to valid boundary
outcome, _, _, _ = await _exec_click(
target=[-10, 1050],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Clicked at [-10, 1050]" in outcome
mock_controller.tap_at.assert_called_with(0, 2399, times=1, delay_ms=100)
@pytest.mark.asyncio
async def test_click_graceful_failure(mock_context, mock_controller):
state = DummyState()
mock_controller.tap_at.return_value = Mock(error="Click failed connection lost")
outcome, _, _, _ = await _exec_click(
target=[500, 600],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error executing click: Click failed connection lost" in outcome
# ==========================================
# SLIDE 2: click_sequence
# ==========================================
@pytest.mark.asyncio
async def test_click_sequence_happy_path(mock_context, mock_controller):
state = DummyState()
state.indexed_points = [[540, 1200]] # Mapped from index 1 (1-indexed)
outcome, img_bytes, shot_path, xml_list = await _exec_click_sequence(
sequence=[1, [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
assert mock_controller.tap_at.call_count == 2
mock_controller.tap_at.assert_any_call(540, 1200)
mock_controller.tap_at.assert_any_call(324, 960)
@pytest.mark.asyncio
async def test_click_sequence_input_validation(mock_context, mock_controller):
state = DummyState()
state.indexed_points = [[540, 1200], [270, 600]]
# Subsequent elements can also be integers (mixed array of indexes and coordinates)
outcome, _, _, _ = await _exec_click_sequence(
sequence=[1, 2, [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
assert mock_controller.tap_at.call_count == 3
mock_controller.tap_at.assert_any_call(540, 1200)
mock_controller.tap_at.assert_any_call(270, 600)
mock_controller.tap_at.assert_any_call(324, 960)
# Reset indexed_points since _exec_click_sequence overwrites them
# via _capture_screenshot_and_parse_ui
state.indexed_points = [[540, 1200], [270, 600]]
# Index out of bounds
outcome, _, _, _ = await _exec_click_sequence(
sequence=[5, [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during click sequence: Index 5 is out of range" in outcome
@pytest.mark.asyncio
async def test_click_sequence_edge_cases(mock_context, mock_controller):
state = DummyState()
# Coordinates in sequence outside range should be clamped
outcome, _, _, _ = await _exec_click_sequence(
sequence=[[0, 0], [1200, -50]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
mock_controller.tap_at.assert_any_call(0, 0)
mock_controller.tap_at.assert_any_call(1079, 0)
@pytest.mark.asyncio
async def test_click_sequence_graceful_failure(mock_context, mock_controller):
state = DummyState()
mock_controller.tap_at.side_effect = [
Mock(error=None),
Mock(error="Connection error on step 2"),
]
outcome, _, _, _ = await _exec_click_sequence(
sequence=[[100, 200], [300, 400]],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error executing click at" in outcome
assert "Connection error on step 2" in outcome
@pytest.mark.asyncio
async def test_click_sequence_stringified_formats(mock_context, mock_controller):
state = DummyState()
state.indexed_points = [[540, 1200]]
# Test stringified coordinate formats commonly generated by LLMs:
# 1. string representation of lists: ["[500, 280]", "[884, 362]"]
outcome, _, _, _ = await _exec_click_sequence(
sequence=["[500, 280]", "[884, 362]"],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
mock_controller.tap_at.assert_any_call(540, 672)
mock_controller.tap_at.assert_any_call(954, 868)
# 2. comma-separated strings: ["500,280", "884,362"]
outcome, _, _, _ = await _exec_click_sequence(
sequence=["500,280", "884,362"],
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
# 3. whole sequence as a string
outcome, _, _, _ = await _exec_click_sequence(
sequence="[[500, 280], [884, 362]]",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Sequence clicked successfully" in outcome
# ==========================================
# SLIDE 3: long_press
# ==========================================
@pytest.mark.asyncio
async def test_long_press_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_long_press(
target=[500, 500],
duration=1500,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Long pressed at [500, 500]" in outcome
mock_controller.tap_at.assert_called_once_with(
540, 1200, long_press=True, long_press_duration=1500
)
@pytest.mark.asyncio
async def test_long_press_input_validation(mock_context, mock_controller):
state = DummyState()
# Invalid target type
outcome, _, _, _ = await _exec_long_press(
target="not-a-list",
duration=1500,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during long press:" in outcome
# ==========================================
# SLIDE 4: input_text
# ==========================================
@pytest.mark.asyncio
async def test_input_text_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_input_text(
text="hello",
target=[500, 500],
clear_exist=True,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Executed typing 'hello'." in outcome
mock_controller.tap_at.assert_called_once_with(540, 1200)
mock_controller.erase_text.assert_called_once()
mock_controller.type_text.assert_called_once_with("hello")
@pytest.mark.asyncio
async def test_input_text_input_validation(mock_context, mock_controller):
state = DummyState()
outcome, _, _, _ = await _exec_input_text(
text="hello",
target="invalid",
clear_exist=True,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during input text:" in outcome
# ==========================================
# SLIDE 5: swipe
# ==========================================
@pytest.mark.asyncio
async def test_swipe_happy_path_direction(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_swipe(
action="up",
duration=400,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Swipe completed successfully" in outcome
mock_controller.swipe_coords.assert_called_once_with(648, 1920, 648, 480, 400)
@pytest.mark.asyncio
async def test_swipe_happy_path_coords(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_swipe(
action=[100, 900, 100, 100],
duration=400,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Swipe completed successfully" in outcome
mock_controller.swipe_coords.assert_called_once_with(108, 2160, 108, 240, 400)
@pytest.mark.asyncio
async def test_swipe_input_validation(mock_context, mock_controller):
state = DummyState()
outcome, _, _, _ = await _exec_swipe(
action="invalid_direction",
duration=400,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error during swipe: Invalid direction:" in outcome
# ==========================================
# SLIDE 6: press_key
# ==========================================
@pytest.mark.asyncio
async def test_press_key_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_press_key(
key="BACK",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Executed key press 'BACK'." in outcome
mock_controller.go_back.assert_called_once()
@pytest.mark.asyncio
async def test_press_key_input_validation(mock_context, mock_controller):
state = DummyState()
outcome, _, _, _ = await _exec_press_key(
key="INVALID_KEY",
state=state,
controller=mock_controller,
ctx=mock_context,
)
# The current execution doesn't raise error on unknown keys, just logs error
assert "Error executing key press 'INVALID_KEY'." in outcome
# ==========================================
# SLIDE 7: manage_app
# ==========================================
@pytest.mark.asyncio
async def test_manage_app_happy_path(mock_context, mock_controller):
state = DummyState()
with (
patch("artemis.agents.validator.tool_declarations.find_package") as mock_find_package,
patch("artemis.agents.validator.tool_declarations.launch_app_with_retries") as mock_launch,
):
mock_find_package.return_value = "com.google.android.youtube"
mock_launch.return_value = (True, "")
outcome, img_bytes, shot_path, xml_list = await _exec_manage_app(
action="launch",
app_name="YouTube",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Launched app 'YouTube' (com.google.android.youtube) successfully." in outcome
mock_find_package.assert_called_once_with(mock_context, "YouTube", use_fallback=False)
mock_launch.assert_called_once_with(mock_context, "com.google.android.youtube")
@pytest.mark.asyncio
async def test_manage_app_input_validation(mock_context, mock_controller):
state = DummyState()
with patch("artemis.agents.validator.tool_declarations.find_package") as mock_find_package:
mock_find_package.return_value = None
outcome, _, _, _ = await _exec_manage_app(
action="launch",
app_name="NonExistentApp",
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Error finding package for app: NonExistentApp" in outcome
# ==========================================
# SLIDE 8: wait_for_delay
# ==========================================
@pytest.mark.asyncio
async def test_wait_for_delay_happy_path(mock_context, mock_controller):
state = DummyState()
outcome, img_bytes, shot_path, xml_list = await _exec_wait_for_delay(
time_in_ms=100,
state=state,
controller=mock_controller,
ctx=mock_context,
)
assert "Waited for 100ms successfully." in outcome
+113
View File
@@ -0,0 +1,113 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Universal FlashRunner."""
from unittest.mock import Mock, patch
import pytest
from langchain_core.messages import HumanMessage, ToolMessage
from artemis.agents.flash.runner import FlashRunner
from artemis.context import ArtemisContext
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
mock_llm_cfg = Mock()
mock_llm_cfg.model = "gemini-2.5-flash"
mock_llm_cfg.temperature = 0.1
ctx.llm_config.get_agent.return_value = mock_llm_cfg
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = None
ctx.adb_client = None
ctx.driver = Mock()
return ctx
def test_flash_runner_tools_initialization(mock_context):
"""Verify FlashRunner gathers correct active tools."""
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="Test Open Settings")
tools = runner._get_tools()
tool_names = [t.name for t in tools]
assert "click" in tool_names
assert "click_sequence" in tool_names
assert "swipe" in tool_names
assert "input_text" in tool_names
assert "ask_explorer" in tool_names
assert "report_task_status" in tool_names
# Validator's report_failure_analysis should NOT be in FlashRunner
assert "report_failure_analysis" not in tool_names
def test_flash_runner_screenshot_pruning(mock_context):
"""Verify that earlier screenshot blocks are pruned, keeping only the latest."""
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="Test Pruning")
messages = [
HumanMessage(
content=[
{"type": "text", "text": "Turn 1 goal"},
{
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,SCREENSHOT_1"},
},
]
),
ToolMessage(
tool_call_id="tc1",
name="click",
content=[
{"type": "text", "text": "Clicked button"},
{
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,SCREENSHOT_2"},
},
],
),
ToolMessage(
tool_call_id="tc2",
name="swipe",
content=[
{"type": "text", "text": "Swiped up"},
{
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,SCREENSHOT_3"},
},
],
),
]
runner._prune_intermediate_screenshots(messages)
# Check Turn 1: image should be removed
assert len(messages[0].content) == 1
assert messages[0].content[0]["type"] == "text"
# Check Turn 2 (Tool 1): image should be removed
assert len(messages[1].content) == 1
assert messages[1].content[0]["type"] == "text"
# Check Turn 3 (Tool 2 - Latest): image MUST be preserved
assert len(messages[2].content) == 2
assert messages[2].content[0]["type"] == "text"
assert messages[2].content[1]["type"] == "image_url"
assert messages[2].content[1]["image_url"]["url"] == "data:image/jpeg;base64,SCREENSHOT_3"
@@ -0,0 +1,289 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for VisualStepSummarizer and Context Compressor in Flash profile."""
import asyncio
from unittest.mock import AsyncMock, Mock
import pytest
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from artemis.agents.flash.context_compressor import compress_flash_messages
from artemis.agents.flash.summarizer import VisualStepSummarizer
from artemis.context import ArtemisContext
from artemis.sdk.builders import Builders
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.data_engine = None
return ctx
@pytest.mark.asyncio
async def test_summarizer_dispatch_and_caching(mock_context):
"""Verify non-blocking dispatch and summary generation."""
summarizer = VisualStepSummarizer(mock_context)
mock_llm_response = AIMessage(
content="[Step 1 State: Tapped 'Login' button. Home view opened.]"
)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(return_value=mock_llm_response)
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [500, 600]},
pre_img_bytes=b"fake_pre",
post_img_bytes=b"fake_post",
exec_outcome="Clicked successfully",
)
# Await background task flush
await summarizer.flush()
assert summarizer.has_summary(1)
summary = summarizer.get_summary(1)
assert "[Step 1 State: Tapped 'Login' button. Home view opened.]" in summary
# Verify memory optimization: image buffers are released
assert summarizer._step_inputs[1]["pre_img_bytes"] is None
assert summarizer._step_inputs[1]["post_img_bytes"] is None
@pytest.mark.asyncio
async def test_summarizer_retrigger_stalled_steps(mock_context):
"""Verify that earlier stalled steps are re-triggered when subsequent steps arrive."""
summarizer = VisualStepSummarizer(mock_context)
attempt_count = 0
async def mock_ainvoke(messages):
nonlocal attempt_count
attempt_count += 1
# Fail the first attempt (Step 1 first call)
if attempt_count == 1:
raise asyncio.CancelledError("Simulated timeout")
# Check messages for step number
msg_str = str(messages)
if "Step 1" in msg_str:
return AIMessage(content="[Step 1 State: Recovered Step 1 summary]")
return AIMessage(content="[Step 2 State: Step 2 summary]")
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(side_effect=mock_ainvoke)
# Dispatch step 1 (fails on first attempt)
summarizer.dispatch(
step_number=1,
action_name="click",
action_args={"target": [100, 100]},
pre_img_bytes=b"pre1",
post_img_bytes=b"post1",
exec_outcome="Outcome 1",
)
await summarizer.flush()
assert not summarizer.has_summary(1)
# Dispatch step 2 (should detect stalled Step 1 and re-trigger)
summarizer.dispatch(
step_number=2,
action_name="swipe",
action_args={"action": "up"},
pre_img_bytes=b"pre2",
post_img_bytes=b"post2",
exec_outcome="Outcome 2",
)
await summarizer.flush()
assert summarizer.has_summary(1)
assert summarizer.has_summary(2)
assert "Recovered Step 1" in summarizer.get_summary(1)
def test_context_compressor_with_ready_summaries(mock_context):
"""Verify that ready summaries replace past image blocks and prune outdated XML."""
summarizer = VisualStepSummarizer(mock_context)
summarizer._summaries[1] = "[Step 1 State: Tapped Search bar. Keyboard appeared.]"
summarizer._summaries[2] = "[Step 2 State: Typed 'flights'. Suggestions displayed.]"
messages = [
HumanMessage(
content=[
{"type": "text", "text": "Task: Search flights"},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
{"type": "text", "text": "--- UI Element List ---\nOld Tree 1"},
]
),
ToolMessage(
tool_call_id="tc1",
name="click",
content=[
{"type": "text", "text": "Action 'click' completed."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
{"type": "text", "text": "--- UI Element List ---\nOld Tree 2"},
],
),
ToolMessage(
tool_call_id="tc2",
name="input_text",
content=[
{"type": "text", "text": "Action 'input_text' completed."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_3"}},
{"type": "text", "text": "--- UI Element List ---\nLive Latest Tree 3"},
],
),
]
compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
# Initial HumanMessage: Historical initial image pruned, task text preserved
assert len(messages[0].content) == 1
assert messages[0].content[0]["text"] == "Task: Search flights"
# Step 1 (ToolMessage 1): Image replaced with Step 1 summary, old XML pruned
assert len(messages[1].content) == 2
assert messages[1].content[0]["text"] == "Action 'click' completed."
assert messages[1].content[1]["text"] == "[Step 1 State: Tapped Search bar. Keyboard appeared.]"
# Step 2 (ToolMessage 2 - Latest live screen): Image and latest live XML MUST be preserved
assert len(messages[2].content) == 3
assert messages[2].content[0]["text"] == "Action 'input_text' completed."
assert messages[2].content[1]["type"] == "image_url"
assert "Live Latest Tree 3" in messages[2].content[2]["text"]
def test_context_compressor_graceful_fallback(mock_context):
"""Verify that when a summary is not ready yet, it cleanly falls back to standard pruned state."""
summarizer = VisualStepSummarizer(mock_context)
# Step 1 has no summary in cache
messages = [
HumanMessage(
content=[
{"type": "text", "text": "Task: Search"},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_1"}},
]
),
ToolMessage(
tool_call_id="tc1",
name="click",
content=[
{"type": "text", "text": "Action 'click' completed."},
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,IMG_2"}},
],
),
]
compress_flash_messages(messages, summarizer=summarizer, prune_history_xml=True)
# Turn 1 should simply have image omitted (standard fallback)
assert len(messages[0].content) == 1
assert messages[0].content[0]["text"] == "Task: Search"
# Turn 2 (Latest) preserves image
assert len(messages[1].content) == 2
assert messages[1].content[1]["type"] == "image_url"
def test_flash_config_and_builder():
"""Verify Flash profile configuration model and SDK builder fluent API."""
cfg = Builders.AgentConfig.with_flash_config(
max_turns=25,
explorer_mode="flash",
step_summarizer=True,
step_summarizer_model="gemini-2.5-flash-lite",
prune_history_xml=True,
).build()
assert cfg.flash.max_turns == 25
assert cfg.flash.explorer_mode == "flash"
assert cfg.flash.step_summarizer.enabled is True
assert cfg.flash.step_summarizer.model == "gemini-2.5-flash-lite"
assert cfg.flash.step_summarizer.prune_history_xml is True
@pytest.mark.asyncio
async def test_data_engine_integration_with_step_summarizer(mock_context):
"""Verify that generated summaries sync into DataEngine and are visible to get_agent_friendly_steps."""
mock_de = Mock()
mock_de.update_step_summary = Mock()
mock_context.data_engine = mock_de
summarizer = VisualStepSummarizer(mock_context)
summarizer._llm = Mock()
summarizer._llm.ainvoke = AsyncMock(
return_value=AIMessage(content="[Step 1 State: Input text confirmed.]")
)
summarizer.dispatch(
step_number=1,
action_name="input_text",
action_args={"text": "hello"},
pre_img_bytes=b"pre",
post_img_bytes=b"post",
exec_outcome="Executed typing",
)
await summarizer.flush()
mock_de.update_step_summary.assert_called_once_with(1, "[Step 1 State: Input text confirmed.]")
def test_draw_action_overlay_rendering():
"""Verify that draw_action_overlay_on_image correctly renders markers on images."""
import io
from PIL import Image
from artemis.utils.visualization import draw_action_overlay_on_image
# Create a blank RGB image in memory
img = Image.new("RGB", (1080, 2400), color="white")
buf = io.BytesIO()
img.save(buf, format="JPEG")
raw_bytes = buf.getvalue()
# 1. Click overlay
annotated_click = draw_action_overlay_on_image(
image_bytes=raw_bytes,
action_name="click",
action_args={"target": [540, 1200]},
)
assert len(annotated_click) > 0
assert annotated_click != raw_bytes
# 2. Swipe overlay
annotated_swipe = draw_action_overlay_on_image(
image_bytes=raw_bytes,
action_name="swipe",
action_args={"start": [500, 1800], "end": [500, 600]},
)
assert len(annotated_swipe) > 0
assert annotated_swipe != raw_bytes
# 3. Click Sequence overlay
annotated_seq = draw_action_overlay_on_image(
image_bytes=raw_bytes,
action_name="click_sequence",
action_args={"sequence": [[200, 300], [400, 500], [600, 700]]},
)
assert len(annotated_seq) > 0
assert annotated_seq != raw_bytes
# 4. Fail-safe on corrupt bytes
corrupt_result = draw_action_overlay_on_image(
image_bytes=b"invalid_bytes",
action_name="click",
action_args={"target": [100, 100]},
)
assert corrupt_result == b"invalid_bytes"
+377
View File
@@ -0,0 +1,377 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from pathlib import Path
from unittest.mock import MagicMock, patch
from langchain_core.messages import ToolMessage
from artemis.agents.history_analyzer.history_analyzer import HistoryAnalyzer
from artemis.context import ArtemisContext
import pytest
@pytest.mark.asyncio
async def test_history_analyzer_no_history():
# Mock context with a data engine that has no steps
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.get_agent_friendly_steps.return_value = []
analyzer = HistoryAnalyzer(mock_ctx)
result = await analyzer.run("What happened?")
assert result == "No history recorded for this session yet."
@pytest.mark.asyncio
async def test_history_analyzer_simple_query_no_tool_call():
# Mock context and data engine
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
steps = [
{
"step_number": 1,
"relative_time": "1.2s",
"summary": "Opened the settings app",
},
{
"step_number": 2,
"relative_time": "4.5s",
"summary": "Toggled the wifi switch",
},
]
mock_ctx.data_engine.get_agent_friendly_steps.return_value = steps
# Mock LLM to return a direct text response (no tool calls)
mock_response = MagicMock()
mock_response.content = "First you opened settings, then you toggled wifi."
mock_response.tool_calls = []
mock_llm = MagicMock()
mock_llm.bind_tools.return_value = mock_llm
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
with (
patch(
"artemis.agents.history_analyzer.history_analyzer.get_llm",
return_value=mock_llm,
),
patch("pathlib.Path.exists", return_value=False), # Force system prompt fallback
):
analyzer = HistoryAnalyzer(mock_ctx)
result = await analyzer.run("Summarize the session")
assert result == "First you opened settings, then you toggled wifi."
assert mock_llm.astream.called
@pytest.mark.asyncio
async def test_history_analyzer_detailed_query_with_tool_call():
# Mock context and data engine with full details
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
steps = [
{
"step_number": 1,
"relative_time": "1.2s",
"summary": "Opened the settings app",
"action_taken": {"action": "tap", "coordinates": [500, 600]},
"operator_raw_thinking": "Need to open settings to configure wifi",
"last_execution_result": {"status": "success"},
},
{
"step_number": 2,
"relative_time": "4.5s",
"summary": "Toggled the wifi switch",
"action_taken": {"action": "tap", "coordinates": [100, 200]},
"operator_raw_thinking": "Toggle the switch to turn wifi on",
"last_execution_result": {"status": "success"},
},
]
mock_ctx.data_engine.get_agent_friendly_steps.return_value = steps
# Turn 1 LLM response: Tool call to get_step_details for step 2
mock_response_turn_1 = MagicMock()
mock_response_turn_1.content = ""
mock_tool_call = {
"name": "get_step_details",
"args": {"start_step": 2, "end_step": 2},
"id": "call_123456",
}
mock_response_turn_1.tool_calls = [mock_tool_call]
# Turn 2 LLM response: Final answer using the details
mock_response_turn_2 = MagicMock()
mock_response_turn_2.content = (
"In step 2, the operator was thinking: 'Toggle the switch to turn wifi on'."
)
mock_response_turn_2.tool_calls = []
mock_llm = MagicMock()
mock_llm.bind_tools.return_value = mock_llm
responses = [mock_response_turn_1, mock_response_turn_2]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
with (
patch(
"artemis.agents.history_analyzer.history_analyzer.get_llm",
return_value=mock_llm,
),
patch("pathlib.Path.exists", return_value=False), # Force fallback prompt
):
analyzer = HistoryAnalyzer(mock_ctx)
result = await analyzer.run("What was the operator thinking in step 2?")
assert (
result == "In step 2, the operator was thinking: 'Toggle the switch to turn wifi on'."
)
assert mock_llm.astream.call_count == 2
# Verify the tool was invoked with correct parameters internally
tool_msg = mock_llm.astream.call_args_list[1][0][0][
-2
] # ToolMessage is second to last after final response is appended
assert isinstance(tool_msg, ToolMessage)
assert tool_msg.tool_call_id == "call_123456"
# Parse the JSON in the ToolMessage content
tool_result = json.loads(tool_msg.content)
assert len(tool_result) == 1
assert tool_result[0]["step_number"] == 2
assert tool_result[0]["operator_raw_thinking"] == "Toggle the switch to turn wifi on"
assert tool_result[0]["action_taken"] == {
"action": "tap",
"coordinates": [100, 200],
}
@pytest.mark.asyncio
async def test_history_analyzer_read_note_tool_call():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
steps = [{"step_number": 1, "relative_time": "1.0s", "summary": "Initiated"}]
mock_ctx.data_engine.get_agent_friendly_steps.return_value = steps
# Turn 1 LLM response: Tool call to list_notes
mock_response_turn_1 = MagicMock()
mock_response_turn_1.content = ""
mock_tool_call_list = {"name": "list_notes", "args": {}, "id": "call_list"}
mock_response_turn_1.tool_calls = [mock_tool_call_list]
# Turn 2 LLM response: Tool call to read_note for tactical_plan
mock_response_turn_2 = MagicMock()
mock_response_turn_2.content = ""
mock_tool_call_read = {
"name": "read_note",
"args": {"key": "tactical_plan"},
"id": "call_read",
}
mock_response_turn_2.tool_calls = [mock_tool_call_read]
# Turn 3 LLM response: Final answer
mock_response_turn_3 = MagicMock()
mock_response_turn_3.content = "The plan was to toggle wifi."
mock_response_turn_3.tool_calls = []
mock_llm = MagicMock()
mock_llm.bind_tools.return_value = mock_llm
responses = [
mock_response_turn_1,
mock_response_turn_2,
mock_response_turn_3,
]
call_count = 0
async def mock_astream(*args, **kwargs):
nonlocal call_count
if call_count < len(responses):
yield responses[call_count]
call_count += 1
mock_llm.astream.side_effect = mock_astream
# Mock the file system read using patch
fake_plan_content = "Goal: Toggle wifi"
def mock_exists(path):
# Let Path("/tmp/fake_traces/notes").exists() and notes/tactical_plan.md exist
return True
def mock_read_text(self, encoding="utf-8"):
return fake_plan_content
def mock_glob(self, pattern):
return [Path("/tmp/fake_traces/notes/tactical_plan.md")]
with (
patch(
"artemis.agents.history_analyzer.history_analyzer.get_llm",
return_value=mock_llm,
),
patch("pathlib.Path.exists", mock_exists),
patch("pathlib.Path.read_text", mock_read_text),
patch("pathlib.Path.glob", mock_glob),
):
analyzer = HistoryAnalyzer(mock_ctx)
result = await analyzer.run("What is the saved tactical plan?")
assert result == "The plan was to toggle wifi."
assert mock_llm.astream.call_count == 3
# Verify list_notes response
list_msg = mock_llm.astream.call_args_list[1][0][0][
3
] # Index 3 contains the ToolMessage for list_notes
assert isinstance(list_msg, ToolMessage)
assert list_msg.tool_call_id == "call_list"
assert list_msg.content == "Here are all the notes:\n- tactical_plan (1 lines)"
# Verify read_note response
read_msg = mock_llm.astream.call_args_list[2][0][0][
5
] # Index 5 contains the ToolMessage for read_note
assert isinstance(read_msg, ToolMessage)
assert read_msg.tool_call_id == "call_read"
assert (
read_msg.content == "Successfully read note 'tactical_plan'. 'tactical_plan' note"
f" content:\n{fake_plan_content}"
)
def test_history_analyzer_robust_tools_behavior():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
steps = [
{
"step_number": 1,
"relative_time": "1.0s",
"summary": "Action 1",
"action_taken": {"action": "tap"},
}
]
analyzer = HistoryAnalyzer(mock_ctx)
# 1. Test get_step_details with string inputs
details_tool = analyzer._get_step_details_tool(steps)
result_details = details_tool.invoke({"start_step": "1", "end_step": "1"})
parsed_result = json.loads(result_details)
assert len(parsed_result) == 1
assert parsed_result[0]["step_number"] == 1
# 2. Test read_note with .md suffix
from artemis.tools.scratchpad import get_read_note_tool_pure
read_tool = get_read_note_tool_pure(analyzer.ctx)
def mock_exists(path):
return "tactical_plan.md" in str(path)
with (
patch("pathlib.Path.exists", mock_exists),
patch("pathlib.Path.read_text", return_value="Tactical plan content"),
):
result_read = read_tool.invoke({"key": "tactical_plan.md"})
assert (
result_read == "Successfully read note 'tactical_plan.md'. 'tactical_plan.md'"
" note content:\nTactical plan content"
)
@pytest.mark.asyncio
async def test_history_analyzer_integration_with_task_tree():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = "/tmp/fake_traces"
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "1.0s",
"summary": "Step 1",
},
]
mock_ctx.data_engine.get_agent_friendly_steps.return_value = steps
mock_response = MagicMock()
mock_response.content = "Answer"
mock_response.tool_calls = []
mock_llm = MagicMock()
mock_llm.bind_tools.return_value = mock_llm
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
fake_plan = """- [/] Active Subgoal
- [ ] Pending Subgoal"""
mock_path = MagicMock()
mock_path.exists.return_value = True
mock_path.read_text.return_value = fake_plan
# Mock build_plan_and_history
with (
patch(
"artemis.agents.history_analyzer.history_analyzer.get_llm",
return_value=mock_llm,
),
patch(
"artemis.agents.history_analyzer.history_analyzer.get_note_file_path",
return_value=mock_path,
),
patch(
"artemis.agents.history_analyzer.history_analyzer.build_plan_and_history"
) as mock_build,
):
mock_build.return_value = "Mocked operation history"
analyzer = HistoryAnalyzer(mock_ctx)
await analyzer.run("query")
# Verify build_plan_and_history call args
mock_build.assert_called_once()
args, kwargs = mock_build.call_args
assert args[0] == fake_plan
assert args[1] == steps
import hashlib
expected_hash = hashlib.md5(b"Active Subgoal").hexdigest()
assert args[2] == expected_hash
assert kwargs.get("last_n_detailed") == 1
assert kwargs.get("min_summaries") == len(steps)
File diff suppressed because it is too large. Load diff
+508
View File
@@ -0,0 +1,508 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, Mock, patch
from pydantic import BaseModel
import pytest
from artemis.agents.outputter.outputter import outputter
from artemis.config import LLM, OutputConfig # noqa: E402
from artemis.context import ArtemisContext # noqa: E402
from artemis.utils.logger import get_logger # noqa: E402
logger = get_logger(__name__)
class _CyFunctionDetectorMeta(type):
def __instancecheck__(self, instance):
name = type(instance).__name__
return (
name
in (
"cyfunction",
"cython_function_or_method",
"builtin_function_or_method",
)
or "cyfunction" in name.lower()
)
class CyFunctionDetector(metaclass=_CyFunctionDetectorMeta):
pass
class MockPydanticSchema(BaseModel):
model_config = {"ignored_types": (CyFunctionDetector,)}
color: str
price: float
currency_symbol: str
website_url: str
mock_dict = {
"color": "green",
"price": 20,
"currency_symbol": "$",
"website_url": "http://superwebsite.fr",
}
class DummyState:
def __init__(self, messages, initial_goal, operator_raw_data=None):
self.messages = messages
self.initial_goal = initial_goal
self.operator_raw_data = operator_raw_data
@pytest.fixture
def mock_context():
"""Create a properly mocked context with all required fields."""
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = {
"planner": LLM(provider="openai", model="gpt-5-nano"),
"operator": LLM(provider="openai", model="gpt-5-nano"),
"validator": LLM(provider="openai", model="gpt-5-nano"),
}
ctx.device = Mock()
ctx.data_engine = None
return ctx
@pytest.fixture
def mock_state():
"""Create a mock state with test data."""
return DummyState(
messages=[],
initial_goal="Find a green product on my website",
)
def setup_mock_llm(mock_get_llm, react_response_content="Paris", structured_response=None):
mock_llm = Mock()
mock_llm_fallback = Mock()
# Mock bind_tools
mock_llm_with_tools = Mock()
mock_llm_with_tools.ainvoke = AsyncMock()
mock_react_response = Mock()
mock_react_response.content = react_response_content
mock_react_response.tool_calls = []
mock_llm_with_tools.ainvoke.return_value = mock_react_response
mock_llm.bind_tools.return_value = mock_llm_with_tools
mock_llm_fallback.bind_tools.return_value = mock_llm_with_tools
# Mock with_structured_output
mock_structured_llm = Mock()
mock_structured_llm.ainvoke = AsyncMock()
if structured_response is not None:
mock_structured_llm.ainvoke.return_value = structured_response
mock_llm.with_structured_output.return_value = mock_structured_llm
mock_llm_fallback.with_structured_output.return_value = mock_structured_llm
def get_llm_side_effect(ctx, name, is_utils=False, use_fallback=False):
if use_fallback:
return mock_llm_fallback
return mock_llm
mock_get_llm.side_effect = get_llm_side_effect
return mock_llm, mock_llm_with_tools, mock_structured_llm
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
async def test_outputter_with_pydantic_model(mock_get_llm, mock_context, mock_state):
"""Test outputter with Pydantic model output."""
expected_structured = MockPydanticSchema(
color="green",
price=20,
currency_symbol="$",
website_url="http://superwebsite.fr",
)
setup_mock_llm(
mock_get_llm,
react_response_content="Raw details about green product",
structured_response=expected_structured,
)
config = OutputConfig(
structured_output=MockPydanticSchema,
output_description=None,
)
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
assert isinstance(result, dict)
assert result.get("color") == "green"
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
async def test_outputter_with_dict(mock_get_llm, mock_context, mock_state):
"""Test outputter with dictionary output."""
expected_dict = {
"color": "green",
"price": 20,
"currency_symbol": "$",
"website_url": "http://superwebsite.fr",
}
setup_mock_llm(
mock_get_llm,
react_response_content="Raw details",
structured_response=expected_dict,
)
config = OutputConfig(
structured_output=mock_dict,
output_description=None,
)
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
assert isinstance(result, dict)
assert result.get("color") == "green"
assert result.get("price") == 20
assert result.get("currency_symbol") == "$"
assert result.get("website_url") == "http://superwebsite.fr"
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
async def test_outputter_with_natural_language_output(mock_get_llm, mock_context, mock_state):
"""Test outputter with natural language description output (returns JSON string)."""
expected_json = (
'{"color": "green", "price": 20, "currency_symbol": "$", "website_url":'
' "http://superwebsite.fr"}'
)
setup_mock_llm(mock_get_llm, react_response_content=expected_json)
config = OutputConfig(
structured_output=None,
output_description=(
"A JSON object with a color, a price, a currency_symbol and a website_url key"
),
)
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
assert isinstance(result, dict)
assert result.get("color") == "green"
assert result.get("price") == 20
assert result.get("currency_symbol") == "$"
assert result.get("website_url") == "http://superwebsite.fr"
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
async def test_outputter_with_history_and_image(mock_get_llm, mock_context):
"""Test outputter with explicit history and image."""
expected_json = '{"answer": "Paris"}'
_, mock_llm_with_tools, _ = setup_mock_llm(mock_get_llm, react_response_content=expected_json)
config = OutputConfig(
structured_output=None,
output_description="The capital of France",
)
class StateWithRawData:
def __init__(self, messages, initial_goal, operator_raw_data):
self.messages = messages
self.initial_goal = initial_goal
self.operator_raw_data = operator_raw_data
state = StateWithRawData(
messages=[],
initial_goal="What is the capital of France?",
operator_raw_data={"screenshot_b64": "mock_b64_data"},
)
await outputter(
ctx=mock_context,
output_config=config,
graph_output=state,
plan_and_history="Plan: Find capital\nStep 1: Opened browser",
)
call_args = mock_llm_with_tools.ainvoke.call_args
assert call_args is not None
messages = call_args[0][0]
assert (
"Your sole objective is to verify whether the user's initial goal was"
" achieved" in messages[0].content
)
human_content = messages[1].content
assert isinstance(human_content, list)
assert human_content[0]["type"] == "text"
assert "Step 1: Opened browser" in human_content[0]["text"]
assert "Plan: Find capital" in human_content[0]["text"]
assert human_content[1]["type"] == "image_url"
assert human_content[1]["image_url"]["url"] == "data:image/jpeg;base64,mock_b64_data"
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
async def test_outputter_builds_concise_history(mock_get_llm):
"""Test that the outputter dynamically builds clean Summary ➡️ Action history."""
expected_json = '{"answer": "Success"}'
_, mock_llm_with_tools, _ = setup_mock_llm(mock_get_llm, react_response_content=expected_json)
config = OutputConfig(
structured_output=None,
output_description="Test output",
)
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = {
"planner": LLM(provider="openai", model="gpt-5-nano"),
"operator": LLM(provider="openai", model="gpt-5-nano"),
"validator": LLM(provider="openai", model="gpt-5-nano"),
}
ctx.device = Mock()
mock_data_engine = Mock()
steps = [
{
"step_id": "step_1",
"step_number": 1,
"relative_time": "2.5s",
"summary": "Tapped search button",
"action_taken": [{"action": "click", "target_text": "Search"}],
"last_execution_result": {"status": "success"},
}
]
mock_data_engine.get_agent_friendly_steps.return_value = steps
mock_data_engine.base_dir = "/tmp/mock_session"
ctx.data_engine = mock_data_engine
class StateWithRawData:
def __init__(self, messages, initial_goal, operator_raw_data):
self.messages = messages
self.initial_goal = initial_goal
self.operator_raw_data = operator_raw_data
state = StateWithRawData(
messages=[],
initial_goal="Search for something",
operator_raw_data=None,
)
# Test Case 1: Fallback path (no plan file)
from pathlib import Path
original_exists = Path.exists
def mock_exists_fallback(self, *args, **kwargs):
if self.name == "task_plan.md":
return False
return original_exists(self, *args, **kwargs)
with patch("pathlib.Path.exists", mock_exists_fallback):
await outputter(
ctx=ctx,
output_config=config,
graph_output=state,
)
call_args = mock_llm_with_tools.ainvoke.call_args
assert call_args is not None
messages = call_args[0][0]
human_content_fallback = messages[1].content[0]["text"]
assert "--- Execution History ---" in human_content_fallback
assert "- *Step 1 (Start: 2.5s): Tapped search button*" in human_content_fallback
assert "*Action*: Tapped 'Search' at None" in human_content_fallback
# Reset mock call history
mock_llm_with_tools.ainvoke.reset_mock()
# Test Case 2: Standard path (plan file exists)
def mock_exists_standard(self, *args, **kwargs):
if self.name == "task_plan.md":
return True
return original_exists(self, *args, **kwargs)
original_read_text = Path.read_text
def mock_read_text(self, *args, **kwargs):
if self.name == "task_plan.md":
return "- [ ] Goal 1\n - [/] Subgoal 1.1"
return original_read_text(self, *args, **kwargs)
with (
patch("pathlib.Path.exists", mock_exists_standard),
patch("pathlib.Path.read_text", mock_read_text),
):
await outputter(
ctx=ctx,
output_config=config,
graph_output=state,
)
call_args = mock_llm_with_tools.ainvoke.call_args
assert call_args is not None
messages = call_args[0][0]
human_content_standard = messages[1].content[0]["text"]
assert "--- Execution History ---" in human_content_standard
assert "- *Step 1 (Start: 2.5s): Tapped search button*" in human_content_standard
assert "*Action*: Tapped 'Search' at None" in human_content_standard
@patch("artemis.tools.video_tool.VideoAnalyzer")
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
async def test_outputter_executes_video_analyzer_tool(
mock_get_llm, mock_video_analyzer, mock_context, mock_state
):
"""Test that the outputter ReAct loop can successfully invoke the video_analyzer tool."""
# Setup mock VideoAnalyzer
mock_agent_instance = Mock()
mock_agent_instance.run = AsyncMock(
return_value=("The user played a video for 5 seconds.", "success")
)
mock_video_analyzer.return_value = mock_agent_instance
# Setup mock LLM with two turns
mock_llm = Mock()
mock_llm_fallback = Mock()
mock_llm_with_tools = Mock()
mock_llm_with_tools.ainvoke = AsyncMock()
# Turn 1: LLM calls video_analyzer tool
tool_call = {
"name": "video_analyzer",
"args": {
"time_description": "from 5s to 10s",
"purpose": "verify video played",
},
"id": "call_123",
}
msg_turn1 = Mock()
msg_turn1.content = ""
msg_turn1.tool_calls = [tool_call]
# Turn 2: LLM returns final answer
msg_turn2 = Mock()
msg_turn2.content = "Video played successfully."
msg_turn2.tool_calls = []
mock_llm_with_tools.ainvoke.side_effect = [msg_turn1, msg_turn2]
mock_llm.bind_tools.return_value = mock_llm_with_tools
mock_llm_fallback.bind_tools.return_value = mock_llm_with_tools
def get_llm_side_effect(ctx, name, is_utils=False, use_fallback=False):
if use_fallback:
return mock_llm_fallback
return mock_llm
mock_get_llm.side_effect = get_llm_side_effect
config = OutputConfig(
structured_output=None,
output_description="Verify video",
)
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
# Assert VideoAnalyzer was called correctly
mock_video_analyzer.assert_called_once_with(mock_context)
mock_agent_instance.run.assert_called_once_with("from 5s to 10s", "verify video played")
# Assert LLM's ainvoke was called twice
assert mock_llm_with_tools.ainvoke.call_count == 2
# Assert tool message was appended (messages list is mutated in place,
# so it eventually has 5 elements)
sent_messages = mock_llm_with_tools.ainvoke.call_args_list[1][0][0]
assert len(sent_messages) == 5
# Verify the sequence of messages
assert sent_messages[2] == msg_turn1
tool_message = sent_messages[3]
from langchain_core.messages import ToolMessage
assert isinstance(tool_message, ToolMessage)
assert tool_message.content == "The user played a video for 5 seconds."
assert tool_message.status == "success"
assert sent_messages[4] == msg_turn2
# Assert final result
assert result == "Video played successfully."
@patch("artemis.agents.outputter.outputter.get_llm")
@pytest.mark.asyncio
async def test_outputter_executes_save_note_tool(mock_get_llm, mock_context, mock_state):
"""Test that the outputter ReAct loop can successfully invoke the save_note tool."""
# Setup mock LLM with two turns
mock_llm = Mock()
mock_llm_fallback = Mock()
mock_llm_with_tools = Mock()
mock_llm_with_tools.ainvoke = AsyncMock()
# Turn 1: LLM calls save_note tool
tool_call = {
"name": "save_note",
"args": {"key": "verification_code", "content": "123456"},
"id": "call_save_123",
}
msg_turn1 = Mock()
msg_turn1.content = ""
msg_turn1.tool_calls = [tool_call]
# Turn 2: LLM returns final answer
msg_turn2 = Mock()
msg_turn2.content = "Verification code is 123456."
msg_turn2.tool_calls = []
mock_llm_with_tools.ainvoke.side_effect = [msg_turn1, msg_turn2]
mock_llm.bind_tools.return_value = mock_llm_with_tools
mock_llm_fallback.bind_tools.return_value = mock_llm_with_tools
def get_llm_side_effect(ctx, name, is_utils=False, use_fallback=False):
if use_fallback:
return mock_llm_fallback
return mock_llm
mock_get_llm.side_effect = get_llm_side_effect
config = OutputConfig(
structured_output=None,
output_description="Save verification code to note",
)
# Setup mock data_engine on context to support saving notes
mock_data_engine = Mock()
mock_data_engine.base_dir = "/tmp/mock_session"
mock_context.data_engine = mock_data_engine
# Patch save_note_content utility to avoid writing to disk
with patch("artemis.tools.scratchpad.save_note_content") as mock_save_content:
result = await outputter(ctx=mock_context, output_config=config, graph_output=mock_state)
mock_save_content.assert_called_once_with(
"/tmp/mock_session", "verification_code", "123456"
)
# Assert final result
assert result == "Verification code is 123456."
@@ -0,0 +1,102 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import MagicMock, mock_open, patch
from artemis.agents.outputter.tools import (
get_search_history_tool,
get_step_details_tool,
get_step_screenshot_tool,
)
from artemis.context import ArtemisContext
import pytest
@pytest.fixture
def mock_context():
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.global_base_dir = "/tmp/fake_traces"
return ctx
def test_get_step_details_tool():
steps = [
{
"step_number": 1,
"relative_time": "1s",
"summary": "Step 1",
"action_taken": "tap",
},
{
"step_number": 2,
"relative_time": "2s",
"summary": "Step 2",
"action_taken": "swipe",
},
]
tool = get_step_details_tool(steps)
# Test valid range
result = tool.invoke({"start_step": 1, "end_step": 1})
parsed = json.loads(result)
assert len(parsed) == 1
assert parsed[0]["step_number"] == 1
assert parsed[0]["action_taken"] == "tap"
# Test invalid range
result_invalid = tool.invoke({"start_step": 3, "end_step": 4})
assert "No steps found" in result_invalid
def test_get_step_screenshot_tool(mock_context):
steps = [
{"step_number": 1, "pre_image_name": "img1", "post_image_name": "img2"},
]
tool = get_step_screenshot_tool(mock_context, steps)
# Mock open and exists
with (
patch("pathlib.Path.exists", return_value=True),
patch("builtins.open", mock_open(read_data=b"fake_image_bytes")),
):
result = tool.invoke({"step_number": 1})
assert isinstance(result, list)
assert result[0]["type"] == "text"
assert result[1]["type"] == "image_url"
assert "data:image/jpeg;base64" in result[1]["image_url"]["url"]
def test_search_history_tool(mock_context):
tool = get_search_history_tool(mock_context)
# Mock steps returned by storage
mock_step_1 = MagicMock()
mock_step_1.step_number = 1
mock_step_1.pre_image_name = "img1"
mock_step_1.post_image_name = None
mock_context.data_engine.storage.get_steps.return_value = [mock_step_1]
# Mock search_ui_by_hash
mock_context.data_engine.storage.search_ui_by_hash.return_value = [
{"type": "ocr", "matched_text": "verification code: 1234", "score": 0.9}
]
result = tool.invoke({"query": "verification"})
parsed = json.loads(result)
assert len(parsed) == 1
assert parsed[0]["step_number"] == 1
assert parsed[0]["matches"][0]["matched_text"] == "verification code: 1234"
+89
View File
@@ -0,0 +1,89 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import Mock, patch
from artemis.agents.planner.planner import PlannerNode
from artemis.context import DevicePlatform, ArtemisContext
import pytest
class DummyState:
def __init__(
self,
initial_goal,
needs_replan=False,
operator_raw_data=None,
operator_replan_reason=None,
):
self.initial_goal = initial_goal
self.needs_replan = needs_replan
self.operator_raw_data = operator_raw_data
self.operator_replan_reason = operator_replan_reason
async def asanitize_update(self, ctx, update, agent):
return update
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.device = Mock()
ctx.device.mobile_platform = DevicePlatform.ANDROID
ctx.adb_client = Mock()
ctx.ui_adb_client = Mock()
mock_screen_data = Mock()
mock_screen_data.base64 = "YmFzZTY0"
mock_screen_data.elements = []
mock_screen_data.width = 1080
mock_screen_data.height = 2400
ctx.ui_adb_client.get_screen_data.return_value = mock_screen_data
ctx.data_engine = None
return ctx
@pytest.mark.asyncio
async def test_planner_initial_plan(mock_context):
state = DummyState(
initial_goal="Test Goal",
operator_raw_data={"screenshot_b64": "dummy_base64_string"},
)
node = PlannerNode(mock_context)
mock_llm = Mock()
mock_response = Mock()
mock_response.content = "- [ ] Subgoal 1"
mock_response.tool_calls = []
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
mock_llm.bind_tools = Mock(return_value=mock_llm)
with patch("artemis.agents.planner.planner.get_llm", return_value=mock_llm):
await node(state)
assert mock_llm.astream.called
args, kwargs = mock_llm.astream.call_args
messages = args[0]
system_msg = messages[0].content
assert "## Role" in system_msg
human_msg_content = messages[1].content
assert isinstance(human_msg_content, list)
assert human_msg_content[0]["type"] == "text"
assert "Goal: Test Goal" in human_msg_content[0]["text"]
+112
View File
@@ -0,0 +1,112 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.operator.operator import OperatorNode
from artemis.context import ArtemisContext
from artemis.graph.graph import execution_check_node
from artemis.graph.state import State
import pytest
@pytest.mark.asyncio
async def test_snapshot_rollback_flow(tmp_path):
# Setup dummy directories
base_dir = tmp_path / "trace"
notes_dir = base_dir / "notes"
notes_dir.mkdir(parents=True)
task_plan_path = notes_dir / "task_plan.md"
task_plan_path.write_text("Initial Plan", encoding="utf-8")
# Mock Context
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = base_dir
mock_ctx.execution_setup = None
# Mock State
mock_state = MagicMock(spec=State)
mock_state.subagent_calls = []
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
mock_state.initial_goal = "Test goal"
mock_state.short_term_memory = ""
mock_state.operator_raw_data = {
"screenshot_b64": "YmFzZTY0",
"xml_hierarchy": "<hierarchy></hierarchy>",
"ocr_results": [],
}
mock_state.current_step_id = "test-step-id"
# Mock Controller and LLM to let Operator run minimally
mock_controller = MagicMock()
mock_controller.take_screenshot = AsyncMock(return_value="YmFzZTY0")
mock_device_data = MagicMock()
mock_device_data.elements = []
mock_controller.get_screen_data = AsyncMock(return_value=mock_device_data)
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.tool_calls = []
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
mock_llm.bind_tools.return_value = mock_llm
# 1. Run Operator to trigger snapshot
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
node = OperatorNode(mock_ctx)
await node(mock_state)
# Verify snapshot was created
snapshot_dir = notes_dir.with_name("notes_snapshot")
assert snapshot_dir.exists()
assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "Initial Plan"
assert mock_ctx.task_plan_snapshot == snapshot_dir
# 2. Simulate Operator optimistic modification
task_plan_path.write_text("Modified Plan", encoding="utf-8")
# 3. Run execution_check_node with failure to trigger rollback
future = asyncio.Future()
future.set_result({"status": "failed"})
mock_ctx.checker_task = future
await execution_check_node(mock_state, mock_ctx)
# Verify rollback restored the file
assert task_plan_path.read_text(encoding="utf-8") == "Initial Plan"
# 4. Run Operator again to create new snapshot (simulate next attempt)
# First modify it again to simulate operator working on restored plan
task_plan_path.write_text("Modified Plan 2", encoding="utf-8")
with patch("artemis.agents.operator.operator.get_llm", return_value=mock_llm):
await node(mock_state)
assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "Modified Plan 2"
# 5. Run execution_check_node with success to trigger commit (cleanup)
future_success = asyncio.Future()
future_success.set_result({"status": "success"})
mock_ctx.checker_task = future_success
await execution_check_node(mock_state, mock_ctx)
# Verify snapshot was deleted
assert not snapshot_dir.exists()
+216
View File
@@ -0,0 +1,216 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import MagicMock, Mock, patch
from artemis.agents.summarizer.summarizer import SummarizerNode
from artemis.context import ArtemisContext
import pytest
class DummyState:
def __init__(
self,
structured_decisions=None,
operator_raw_thinking=None,
operator_native_thinking=None,
last_execution_result=None,
current_step_id=None,
):
self.structured_decisions = structured_decisions
self.operator_raw_thinking = operator_raw_thinking
self.operator_native_thinking = operator_native_thinking
self.last_execution_result = last_execution_result
self.current_step_id = current_step_id
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.device = None
ctx.data_engine = Mock()
ctx.background_tasks = []
return ctx
@pytest.mark.asyncio
async def test_summarizer_success(mock_context):
"""Test that SummarizerNode successfully generates summary from raw thinking and execution result."""
decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}])
thinking = "The operator clicked the search button."
result = {
"status": "success",
"executed_actions": [{"action": "tap", "coordinates": [500, 600]}],
}
state = DummyState(
structured_decisions=decisions,
operator_raw_thinking=thinking,
last_execution_result=result,
current_step_id="12345678-1234-5678-1234-567812345678",
)
# Mock LLM response
mock_response = Mock()
mock_response.content = "Step completed successfully."
mock_llm = MagicMock()
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
with (
patch(
"artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm
) as mock_get_llm,
patch("asyncio.create_task") as mock_create_task,
):
# Run the node call
node = SummarizerNode(mock_context)
await node(state)
# Verify task was registered in context
assert len(mock_context.background_tasks) == 1
assert mock_context.background_tasks[0] == mock_create_task.return_value
# Verify background task was spawned
assert mock_create_task.called
background_coroutine = mock_create_task.call_args[0][0]
# Await the background task synchronously for test verification
await background_coroutine
# Verify LLM was invoked with proper structured messages
assert mock_llm.astream.called
invoked_messages = mock_llm.astream.call_args[0][0]
system_msg = invoked_messages[0].content
human_msg = invoked_messages[1].content
assert "Summarizer Agent" in system_msg
assert "Decisions made" in human_msg
assert "The operator clicked the search button." in human_msg
# Verify Data Engine updated the step summary
from uuid import UUID
mock_context.data_engine.update_step_summary.assert_called_once_with(
UUID("12345678-1234-5678-1234-567812345678"),
"Step completed successfully.",
)
@pytest.mark.asyncio
async def test_summarizer_with_native_thinking(mock_context):
"""Test that SummarizerNode successfully generates summary from both explicit and native thoughts."""
decisions = json.dumps([{"action": "tap", "coordinates": [500, 600]}])
raw_thinking = "Explicit monologue."
native_thinking = "Silent core reasoning."
result = {"status": "success"}
state = DummyState(
structured_decisions=decisions,
operator_raw_thinking=raw_thinking,
operator_native_thinking=native_thinking,
last_execution_result=result,
current_step_id="12345678-1234-5678-1234-567812345678",
)
mock_response = Mock()
mock_response.content = "Summary result."
mock_llm = MagicMock()
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
with (
patch(
"artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm
) as mock_get_llm,
patch("asyncio.create_task") as mock_create_task,
):
node = SummarizerNode(mock_context)
await node(state)
# Verify task was registered in context
assert len(mock_context.background_tasks) == 1
assert mock_context.background_tasks[0] == mock_create_task.return_value
background_coroutine = mock_create_task.call_args[0][0]
await background_coroutine
assert mock_llm.astream.called
invoked_messages = mock_llm.astream.call_args[0][0]
human_msg = invoked_messages[1].content
assert "Operator explicit thoughts:" in human_msg
assert "Explicit monologue." in human_msg
assert "Operator native thoughts:" in human_msg
assert "Silent core reasoning." in human_msg
@pytest.mark.asyncio
async def test_summarizer_with_chronological_trace(mock_context):
"""Test that SummarizerNode generates a prompt containing the chronological step trace when available."""
step_id = "12345678-1234-5678-1234-567812345678"
state = DummyState(
current_step_id=step_id,
)
mock_step_data = {
"step_id": step_id,
"step_number": 1,
"relative_time": "5.0s",
"action_taken": [{"action": "click", "target_text": "Search"}],
"last_execution_result": {"status": "success"},
"interleaved_events": [
{"type": "thought", "content": "Let's click search."},
],
}
mock_context.data_engine.get_agent_friendly_steps.return_value = [mock_step_data]
# Mock LLM response
mock_response = Mock()
mock_response.content = "Summary using chronological trace."
mock_llm = MagicMock()
async def mock_astream(*args, **kwargs):
yield mock_response
mock_llm.astream.side_effect = mock_astream
with (
patch("artemis.agents.summarizer.summarizer.get_llm", return_value=mock_llm),
patch("asyncio.create_task") as mock_create_task,
):
node = SummarizerNode(mock_context)
await node(state)
# Verify task was registered in context
assert len(mock_context.background_tasks) == 1
assert mock_context.background_tasks[0] == mock_create_task.return_value
background_coroutine = mock_create_task.call_args[0][0]
await background_coroutine
assert mock_llm.astream.called
invoked_messages = mock_llm.astream.call_args[0][0]
human_msg = invoked_messages[1].content
assert "--- Execution History ---" in human_msg
assert "Let's click search." in human_msg
+913
View File
@@ -0,0 +1,913 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import AsyncMock, MagicMock, Mock, patch
from langchain_core.messages import AIMessage
from artemis.agents.validator.failure_analyzer import ValidationErrorCategory
from artemis.agents.validator.validator import ValidatorNode
from artemis.context import ArtemisContext
import pytest
class DummyState:
def __init__(
self,
structured_decisions,
validator_messages=[],
current_step_id=None,
latest_screenshot=None,
):
self.structured_decisions = structured_decisions
self.validator_messages = validator_messages
self.current_step_id = current_step_id
self.latest_screenshot = latest_screenshot
async def asanitize_update(self, ctx, update, agent):
return update
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
ctx.data_engine.get_relative_time.return_value = 1.0
return ctx
@pytest.fixture
def temp_screenshot(tmp_path):
p = tmp_path / "screenshot.png"
p.write_bytes(b"dummy_data")
return str(p)
@pytest.fixture
def mock_mcp():
with (
patch("mcp.client.stdio.stdio_client") as mock_stdio,
patch("artemis.agents.validator.validator.ClientSession") as mock_session_cls,
):
mock_ctx = AsyncMock()
mock_ctx.__aenter__.return_value = (Mock(), Mock())
mock_stdio.return_value = mock_ctx
mock_session = AsyncMock()
mock_session.initialize = AsyncMock()
mock_session_cls.return_value = mock_session
yield mock_session
@pytest.mark.asyncio
async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode executes actions successfully."""
# Mock MCP calls: screenshot, tap, screenshot
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # pre-screenshot
Mock(content=[Mock(text="Success")]), # tap action
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
node = ValidatorNode(mock_context)
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert "attempts" not in report["execution"][0]
@pytest.mark.asyncio
async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode triggers failure analysis on error and handles
cannot_fix via tool.
"""
tap_count = 0
def mock_call_tool(name, args):
nonlocal tap_count
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "tap":
tap_count += 1
return Mock(content=[Mock(text="Error: Element not found")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Analysis: Coordinates are likely wrong.",
}
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert report["execution"][0]["attempts"] == [
"Error: Element not found",
"Error: Element not found",
]
assert report["execution"][0]["repair"] == "Analysis: Coordinates are likely wrong."
@pytest.mark.asyncio
async def test_validator_repair_success(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode terminates successfully and clears remaining actions
after successful repair.
"""
tap_count = 0
def mock_call_tool(name, args):
nonlocal tap_count
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "tap":
tap_count += 1
if tap_count <= 2: # First tap fails
return Mock(content=[Mock(text="Error: Element not found")])
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
# Two actions planned initially
decisions = json.dumps(
[
{"action": "tap", "coordinates": [105, 205]},
{"action": "tap", "coordinates": [205, 305]},
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "fixed",
"analysis": "Fixed by updating actions",
}
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 1 # Only the first failed action is in report
assert report["execution"][0]["coordinates"] == [105, 205]
assert report["execution"][0]["attempts"] == [
"Error: Element not found",
"Error: Element not found",
]
assert report["execution"][0]["repair"] == "Fixed by updating actions"
@pytest.mark.asyncio
async def test_validator_wait_for_delay(mock_mcp, mock_context, temp_screenshot):
"""Test that wait_for_delay executes successfully."""
def mock_call(name, args):
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
mock_mcp.call_tool.side_effect = mock_call
decisions = json.dumps([{"action": "wait_for_delay", "time_in_ms": 10}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
node = ValidatorNode(mock_context)
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "wait_for_delay"
@pytest.mark.asyncio
async def test_validator_focus_and_clear_text_no_ui_change(mock_mcp, mock_context, temp_screenshot):
"""Test that focus_and_clear_text executes successfully even if no UI change is detected."""
def mock_call(name, args):
if name == "focus_and_clear_text":
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
mock_mcp.call_tool.side_effect = mock_call
decisions = json.dumps([{"action": "focus_and_clear_text", "coordinates": [400, 500]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
with patch("artemis.utils.image_diff.check_ui_change", return_value=False):
node = ValidatorNode(mock_context)
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "focus_and_clear_text"
@pytest.mark.asyncio
async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_context, temp_screenshot):
"""Test that silent failure (exec succeeds but no UI change) is treated as
success and does not trigger FailureAnalyzer.
"""
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "tap":
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.validator.VALIDATOR_POLL_TIMEOUT", 0.1),
patch("artemis.agents.validator.validator.VALIDATOR_POLL_INTERVAL", 0.01),
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False), # No UI change
):
result = await node(state)
# Assert FailureAnalyzer was NOT called
mock_analyze.assert_not_called()
# Assert the action is considered executed successfully
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert "attempts" not in report["execution"][0]
@pytest.mark.asyncio
async def test_validator_records_skipped_actions_on_cannot_fix(
mock_mcp, mock_context, temp_screenshot
):
"""Test that ValidatorNode marks unexecuted actions as Skipped when repair fails."""
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
return Mock(content=[Mock(text="Error: Connection lost")])
mock_mcp.call_tool.side_effect = mock_call_tool
# Two actions planned
decisions = json.dumps(
[
{"action": "tap", "coordinates": [100, 200]},
{"action": "press_key", "keycode": "ENTER"},
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Cannot fix: System crashed.",
}
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert "execution" in report
assert len(report["execution"]) == 2
# First action failed
assert report["execution"][0]["action"] == "tap"
assert report["execution"][0]["attempts"] == [
"Error: Connection lost",
"Error: Connection lost",
]
assert report["execution"][0]["repair"] == "Cannot fix: System crashed."
# Second action was skipped
assert report["execution"][1]["action"] == "press_key"
assert report["execution"][1]["attempts"] == ["Skipped"]
@pytest.mark.asyncio
async def test_validator_pre_execution_validation_and_repair(
mock_mcp, mock_context, temp_screenshot
):
"""Test that pre-execution validation fails when element changes, and is
repaired by FailureAnalyzer.
"""
mock_live_elements = [
{
"text": "Sign Up",
"bounds": "[100,200][300,300]",
"resource-id": "btn_signup",
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
if name == "tap":
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
"action": "tap",
"coordinates": [200, 250],
"target_text": "Login",
"target_bounds": [100, 200, 300, 300],
"target_resource_id": "btn_login",
}
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
new_callable=AsyncMock,
) as mock_pixel,
):
mock_pixel.return_value = (
False,
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"Pixel check failed",
)
mock_analyze.return_value = {
"status": "fixed",
"analysis": ("Pre-exec failure matched and resolved by switching to Sign Up button."),
}
result = await node(state)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert "Pre-execution validation failed" in args[2]
assert "Login" in args[2]
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
assert "last_execution_result" in result
report = result["last_execution_result"]
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert "Pre-execution validation failed" in report["execution"][0]["attempts"][0]
assert (
report["execution"][0]["repair"]
== "Pre-exec failure matched and resolved by switching to Sign Up"
" button."
)
@pytest.mark.asyncio
async def test_validator_pre_execution_validation_self_healing(
mock_mcp, mock_context, temp_screenshot
):
"""Test that pre-execution validation succeeds and self-heals when element shifts slightly."""
mock_live_elements = [
{
"text": "Login",
"bounds": "[100,220][300,320]",
"resource-id": "btn_login",
}
]
tapped_coordinates = None
def mock_call_tool(name, args):
nonlocal tapped_coordinates
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
if name == "tap":
tapped_coordinates = args.get("coordinates")
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
"action": "tap",
"coordinates": [200, 250],
"target_text": "Login",
"target_bounds": [100, 200, 300, 300],
"target_resource_id": "btn_login",
}
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
await node(state)
# Center of [100, 220][300, 320] is [200, 270]
assert tapped_coordinates == [200, 270]
@pytest.mark.asyncio
async def test_validator_pre_execution_validation_anonymous_occupant(
mock_mcp, mock_context, temp_screenshot
):
"""Test that pre-execution validation correctly flags TARGET_OCCUPIED
when coordinates are blocked by an anonymous clickable view.
"""
# The live screen contains an anonymous clickable view covering
# target coordinates [200, 250]
mock_live_elements = [
{
"text": "",
"bounds": "[150,200][250,300]",
"clickable": True,
"resource-id": "",
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
"action": "tap",
"coordinates": [200, 250],
"target_text": "Login",
"target_bounds": [100, 200, 300, 300],
"target_resource_id": "btn_login",
}
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
new_callable=AsyncMock,
) as mock_pixel,
):
mock_pixel.return_value = (
False,
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"Pixel check failed",
)
mock_analyze.return_value = {
"status": "fixed",
"analysis": "Pre-exec failure bypassed.",
}
await node(state)
# Assert that the validator correctly classified this as TARGET_OCCUPIED
# (due to the clickable anonymous view blocking the click)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
assert "occupied/intercepted by a different element: interactive anonymous element" in args[2]
@pytest.mark.asyncio
async def test_validator_pixel_validation_success(mock_mcp, mock_context, temp_screenshot):
"""Test that the pixel safety net succeeds when Gemini reports target is present."""
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # live screenshot for pixel check
Mock(content=[Mock(text="Success")]), # tap action
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
mock_agent_cfg = Mock()
mock_agent_cfg.provider = "google"
mock_agent_cfg.model = "gemini-3.5-flash-lite"
mock_context.llm_config.get_agent.return_value = mock_agent_cfg
mock_response = Mock()
mock_response.text = json.dumps(
{
"is_present": True,
"reason": "Button is clearly visible and clickable.",
"confidence": 0.95,
}
)
mock_llm = MagicMock()
mock_llm.ainvoke = AsyncMock(return_value=AIMessage(content=mock_response.text))
# Mock crop and draw helpers to return dummy bytes
with (
patch(
"artemis.utils.visualization.crop_and_annotate_target",
return_value=b"dummy_bytes",
),
patch("artemis.agents.validator.validator.get_llm", return_value=mock_llm),
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
):
node = ValidatorNode(mock_context)
result = await node(state)
assert "last_execution_result" in result
report = result["last_execution_result"]
assert len(report["execution"]) == 1
assert "attempts" not in report["execution"][0]
@pytest.mark.asyncio
async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_screenshot):
"""Test that the pixel safety net fails and triggers FailureAnalyzer
when Gemini reports target is missing.
"""
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
mock_agent_cfg = Mock()
mock_agent_cfg.provider = "google"
mock_agent_cfg.model = "gemini-3.5-flash-lite"
mock_context.llm_config.get_agent.return_value = mock_agent_cfg
mock_response = Mock()
mock_response.text = json.dumps(
{
"is_present": False,
"reason": "The button has completely disappeared from the screen.",
"confidence": 0.90,
}
)
mock_llm = MagicMock()
mock_llm.ainvoke = AsyncMock(return_value=AIMessage(content=mock_response.text))
with (
patch(
"artemis.utils.visualization.crop_and_annotate_target",
return_value=b"dummy_bytes",
),
patch("artemis.agents.validator.validator.get_llm", return_value=mock_llm),
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Pixel safety net flagged missing target.",
}
node = ValidatorNode(mock_context)
await node(state)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert "Pixel-level validation failed" in args[2]
assert kwargs.get("error_category") == ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED
@pytest.mark.asyncio
@patch("artemis.agents.validator.validator.find_package")
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
async def test_validator_launch_app_passes_use_fallback_mcp(
mock_launch_app, mock_find_package, mock_mcp, mock_context, temp_screenshot
):
"""Test that ValidatorNode calls find_package with use_fallback=False
for launch_app action in MCP mode.
"""
mock_find_package.return_value = "com.example.app"
mock_launch_app.return_value = (True, None)
# Mock MCP calls: screenshot (pre), screenshot (post)
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # pre-screenshot
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps([{"action": "launch_app", "app_name": "My App"}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
node = ValidatorNode(mock_context)
result = await node(state)
mock_find_package.assert_called_once_with(mock_context, "My App", use_fallback=False)
mock_launch_app.assert_called_once_with(mock_context, "com.example.app")
assert "last_execution_result" in result
@pytest.mark.asyncio
@patch("artemis.agents.validator.validator.find_package")
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
async def test_validator_launch_app_failure_no_retry(
mock_launch_app, mock_find_package, mock_mcp, mock_context, temp_screenshot
):
"""Test that ValidatorNode does NOT retry launch_app action on failure."""
mock_find_package.return_value = "com.example.app"
mock_launch_app.return_value = (False, "Error: Force close")
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps([{"action": "launch_app", "app_name": "My App"}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
with (
patch("artemis.agents.validator.validator.VALIDATOR_POLL_TIMEOUT", 0.05),
patch("artemis.agents.validator.validator.VALIDATOR_POLL_INTERVAL", 0.01),
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=False),
):
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Cannot launch app.",
}
node = ValidatorNode(mock_context)
result = await node(state)
# Verify find_package was called only once
mock_find_package.assert_called_once_with(mock_context, "My App", use_fallback=False)
# Verify launch_app_with_retries was called only once
# (no retry by ValidatorNode itself since it relies on launch_app_with_retries)
mock_launch_app.assert_called_once_with(mock_context, "com.example.app")
# Verify attempts list has only 1 element
assert "last_execution_result" in result
report = result["last_execution_result"]
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "launch_app"
assert report["execution"][0]["attempts"] == ["Error: Force close"]
@pytest.mark.asyncio
async def test_validator_pre_execution_validation_disappeared_not_shifted(
mock_mcp, mock_context, temp_screenshot
):
"""Test that a disappeared small button matching text in a giant container
is classified as TARGET_DISAPPEARED, not TARGET_SHIFTED.
"""
# Live hierarchy only contains a giant background/container
# containing the text "Sign Up"
mock_live_elements = [
{
"text": "Sign Up",
"bounds": "[0,0][1080,2400]",
"resource-id": "root_container",
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
"action": "tap",
"coordinates": [200, 250],
"target_text": "Sign Up",
"target_bounds": [100, 200, 300, 300],
"target_resource_id": "btn_signup",
}
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
new_callable=AsyncMock,
) as mock_pixel,
):
mock_pixel.return_value = (
False,
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"Pixel check failed",
)
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Sign Up button disappeared.",
}
await node(state)
# Assert that the validator correctly classified this as TARGET_DISAPPEARED
# (since size mismatch prevents shift matching)
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_DISAPPEARED
@pytest.mark.asyncio
async def test_validator_pre_execution_validation_ocr_direct_to_pixel(
mock_mcp, mock_context, temp_screenshot
):
"""Test that OCR-derived elements bypass XML validation and
directly use pixel-based validation.
"""
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # live screenshot for pixel check
Mock(content=[Mock(text="Success")]), # tap action
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps(
[
{
"action": "tap",
"coordinates": [200, 250],
"target_text": "Login",
"target_bounds": [100, 200, 300, 300],
"target_class": "android.view.View [OCR]",
}
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch.object(
ValidatorNode,
"_validate_action_precondition",
new_callable=AsyncMock,
) as mock_xml,
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
new_callable=AsyncMock,
) as mock_pixel,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
):
mock_pixel.return_value = (True, ValidationErrorCategory.NONE, "")
result = await node(state)
# Assert XML validation was NOT called
mock_xml.assert_not_called()
# Assert Pixel validation WAS called
mock_pixel.assert_called_once()
assert "last_execution_result" in result
report = result["last_execution_result"]
assert report["status"] == "success"
@pytest.mark.asyncio
async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_bypassed(
mock_mcp, mock_context, temp_screenshot
):
"""Test that when XML validation fails AND Pixel validation is bypassed
(PIXEL_BYPASSED), the XML failure is NOT overridden.
"""
mock_live_elements = [
{
"text": "Sign Up",
"bounds": "[100,200][300,300]",
"resource-id": "btn_signup",
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
"action": "tap",
"coordinates": [200, 250],
"target_text": "Login",
"target_bounds": [100, 200, 300, 300],
"target_resource_id": "btn_login",
}
]
)
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
node = ValidatorNode(mock_context)
with (
patch("artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze") as mock_analyze,
patch("artemis.utils.image_diff.check_ui_change", return_value=True),
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
new_callable=AsyncMock,
) as mock_pixel,
):
# Pixel check was bypassed due to VLM error or low confidence
mock_pixel.return_value = (
True,
ValidationErrorCategory.PIXEL_BYPASSED,
"",
)
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": ("XML validation failed and pixel bypass did not override."),
}
await node(state)
# FailureAnalyzer SHOULD be called because XML validation failed
# and PIXEL_BYPASSED did not override it
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert "Pre-execution validation failed" in args[2]
+456
View File
@@ -0,0 +1,456 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.agents.video_analyzer.video_analyzer import VideoAnalyzer
from artemis.context import ArtemisContext
import pytest
@pytest.mark.asyncio
async def test_video_analyzer_run():
# Mock context
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.llm_config = MagicMock()
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.6-flash"
# Mock Gemini Client
mock_client = MagicMock()
mock_client.aio = MagicMock()
mock_client.aio.models = MagicMock()
mock_chunk = MagicMock()
mock_chunk.text = "Final analysis: The action succeeded."
mock_chunk.function_calls = []
async def mock_stream():
yield mock_chunk
mock_client.aio.models.generate_content_stream = AsyncMock(return_value=mock_stream())
# Mock file reading for prompts
mock_prompt = "# Dynamic Video Analyzer..."
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
return_value=mock_client,
),
patch(
"pathlib.Path.exists",
autospec=True,
side_effect=lambda self: not str(self).endswith(".jpg"),
),
patch("pathlib.Path.read_text", return_value=mock_prompt),
patch(
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
AsyncMock(),
),
):
analyzer = VideoAnalyzer(mock_ctx)
result, status = await analyzer.run(
time_description="from 10s to 15s", purpose="Verify action"
)
assert result == "Final analysis: The action succeeded."
assert status == "success"
assert mock_client.aio.models.generate_content_stream.called
@pytest.mark.asyncio
async def test_video_analyzer_preserves_thought_signature():
# Mock context
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.llm_config = MagicMock()
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.6-flash"
# Mock MobileDeviceController
mock_controller = MagicMock()
mock_controller.extract_segment_metadata = AsyncMock(
return_value=MagicMock(
success=True,
video_path="/tmp/video.mp4",
actual_start_relative_time=10.0,
)
)
from google.genai import types
dummy_signature = b"dummy_thought_signature_bytes_56789"
# Turn 1: Model calls spawn_sub_agent
spawn_call = types.FunctionCall(
name="spawn_sub_agent",
args={
"start_time": 10.0,
"end_time": 15.0,
"specific_query": "Verify login screen",
},
)
spawn_part = types.Part(
function_call=spawn_call,
thought_signature=dummy_signature,
)
mock_chunk_1 = MagicMock()
mock_chunk_1.text = ""
mock_chunk_1.function_calls = [spawn_call]
mock_chunk_1.candidates = [
types.Candidate(content=types.Content(role="model", parts=[spawn_part]))
]
# Turn 2: Model returns final answer
mock_chunk_2 = MagicMock()
mock_chunk_2.text = "Final analysis: The spawn succeeded and login button is visible."
mock_chunk_2.function_calls = []
mock_chunk_2.candidates = [
types.Candidate(
content=types.Content(
role="model",
parts=[types.Part.from_text(text=mock_chunk_2.text)],
)
)
]
# We mock generate_content_stream
call_count = 0
captured_contents = []
async def mock_stream_1():
yield mock_chunk_1
async def mock_stream_2():
yield mock_chunk_2
async def mock_generate_content_stream(model, contents, config):
nonlocal call_count
has_submit_answer = False
if config.tools:
for t in config.tools:
fd_list = getattr(t, "function_declarations", None)
if fd_list:
for fd in fd_list:
if fd.name == "submit_answer":
has_submit_answer = True
break
if not has_submit_answer:
captured_contents.append(list(contents)) # copy history contents passed to call
call_count += 1
if call_count == 1:
return mock_stream_1()
else:
return mock_stream_2()
else:
submit_call = types.FunctionCall(
name="submit_answer",
args={
"confidence_score": 0.9,
"summary": "Sub-agent verification finished",
"analysis": "Analysis here",
"verification_timestamp_secs": 10.0,
},
)
mock_sub_chunk = MagicMock()
mock_sub_chunk.text = ""
mock_sub_chunk.function_calls = [submit_call]
async def sub_stream():
yield mock_sub_chunk
return sub_stream()
# Mock Gemini Client
mock_client = MagicMock()
mock_client.aio = MagicMock()
mock_client.aio.files = MagicMock()
mock_file = MagicMock()
mock_file.name = "dummy_cloud_file_123"
mock_file.uri = "https://gemini-file-api/dummy_cloud_file_123"
mock_file.mime_type = "video/mp4"
mock_client.aio.files.upload = AsyncMock(return_value=mock_file)
mock_f_state = MagicMock()
mock_f_state.state.name = "ACTIVE"
mock_client.aio.files.get = AsyncMock(return_value=mock_f_state)
mock_client.aio.files.delete = AsyncMock()
mock_client.aio.models = MagicMock()
mock_client.aio.models.generate_content_stream = AsyncMock(
side_effect=mock_generate_content_stream
)
# Mock file reading for prompts
mock_prompt = "# Dynamic Video Analyzer..."
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
return_value=mock_client,
),
patch(
"artemis.agents.video_analyzer.video_analyzer.get_controller",
return_value=mock_controller,
),
patch(
"pathlib.Path.exists",
autospec=True,
side_effect=lambda self: not str(self).endswith(".jpg"),
),
patch(
"pathlib.Path.stat",
return_value=os.stat_result((0o100644, 0, 0, 0, 0, 0, 1024 * 1024, 0, 0, 0)),
),
patch("pathlib.Path.read_text", return_value=mock_prompt),
patch(
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
AsyncMock(),
),
patch(
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
AsyncMock(return_value=Path("/tmp/video.mp4")),
),
):
analyzer = VideoAnalyzer(mock_ctx)
result, status = await analyzer.run(
time_description="from 10s to 15s", purpose="Verify login"
)
assert status == "success"
assert "spawn succeeded" in result
assert call_count == 2
# Verify that the second model call's history contains the model turn with the thought_signature
second_call_history = captured_contents[1]
assert len(second_call_history) >= 4
model_turn = second_call_history[1]
assert model_turn.role == "model"
assert len(model_turn.parts) == 1
assert model_turn.parts[0].function_call.name == "spawn_sub_agent"
assert model_turn.parts[0].thought_signature == dummy_signature
@pytest.mark.asyncio
async def test_video_analyzer_sub_agent_confidence_validation():
# Mock context
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.llm_config = MagicMock()
mock_ctx.llm_config.utils.video_analyzer = MagicMock()
mock_ctx.llm_config.utils.video_analyzer.model = "gemini-3.6-flash"
# Mock MobileDeviceController
mock_controller = MagicMock()
mock_controller.extract_segment_metadata = AsyncMock(
return_value=MagicMock(
success=True,
video_path="/tmp/video.mp4",
actual_start_relative_time=10.0,
)
)
from google.genai import types
# Main agent responses
# Turn 1: Calls spawn_sub_agent
spawn_call = types.FunctionCall(
name="spawn_sub_agent",
args={
"start_time": 10.0,
"end_time": 15.0,
"specific_query": "Verify login screen",
},
)
mock_main_chunk_1 = MagicMock()
mock_main_chunk_1.text = ""
mock_main_chunk_1.function_calls = [spawn_call]
# Turn 2: Returns final answer
mock_main_chunk_2 = MagicMock()
mock_main_chunk_2.text = "Final blackboard status checked."
mock_main_chunk_2.function_calls = []
# Sub-agent responses
# Turn 1: Calls submit_answer with invalid confidence_score (string)
invalid_submit_call = types.FunctionCall(
name="submit_answer",
args={
"timeline_events": [
{
"confidence_score": "HIGH",
"transcription": "Invalid confidence score call",
"verification_timestamp_secs": 10.0,
}
],
"summary": "Invalid confidence score call",
"analysis": "Some analysis",
},
)
sub_dummy_signature = b"sub_dummy_signature_9999"
sub_part_1 = types.Part(
function_call=invalid_submit_call,
thought_signature=sub_dummy_signature,
)
mock_sub_chunk_1 = MagicMock()
mock_sub_chunk_1.text = ""
mock_sub_chunk_1.function_calls = [invalid_submit_call]
mock_sub_chunk_1.candidates = [
types.Candidate(content=types.Content(role="model", parts=[sub_part_1]))
]
# Turn 2: Calls submit_answer with valid confidence_score
valid_submit_call = types.FunctionCall(
name="submit_answer",
args={
"timeline_events": [
{
"confidence_score": 0.8,
"transcription": "Valid confidence score call",
"verification_timestamp_secs": 10.0,
}
],
"summary": "Valid confidence score call",
"analysis": "Some analysis",
},
)
mock_sub_chunk_2 = MagicMock()
mock_sub_chunk_2.text = ""
mock_sub_chunk_2.function_calls = [valid_submit_call]
mock_sub_chunk_2.candidates = [
types.Candidate(
content=types.Content(role="model", parts=[types.Part(function_call=valid_submit_call)])
)
]
# Stream generator mocks
main_calls = 0
sub_calls = 0
captured_sub_contents = []
async def mock_main_stream():
nonlocal main_calls
if main_calls == 0:
main_calls += 1
yield mock_main_chunk_1
else:
yield mock_main_chunk_2
async def mock_sub_stream():
nonlocal sub_calls
if sub_calls == 0:
sub_calls += 1
yield mock_sub_chunk_1
else:
yield mock_sub_chunk_2
async def mock_generate_content_stream(model, contents, config):
system_instruction = config.system_instruction
if "Dynamic Video Analyzer" in str(system_instruction):
return mock_main_stream()
else:
captured_sub_contents.append(list(contents))
return mock_sub_stream()
# Mock Gemini Client
mock_client = MagicMock()
mock_client.aio = MagicMock()
mock_client.aio.files = MagicMock()
mock_file = MagicMock()
mock_file.name = "dummy_cloud_file_123"
mock_file.uri = "https://gemini-file-api/dummy_cloud_file_123"
mock_file.mime_type = "video/mp4"
mock_client.aio.files.upload = AsyncMock(return_value=mock_file)
mock_f_state = MagicMock()
mock_f_state.state.name = "ACTIVE"
mock_client.aio.files.get = AsyncMock(return_value=mock_f_state)
mock_client.aio.files.delete = AsyncMock()
mock_client.aio.models = MagicMock()
mock_client.aio.models.generate_content_stream = AsyncMock(
side_effect=mock_generate_content_stream
)
# Mock file reading for prompts
mock_main_prompt = "# Dynamic Video Analyzer..."
mock_sub_prompt = "# Sub Agent..."
def mock_read_text(self, encoding="utf-8"):
if "video_analyzer.md" in str(self):
return mock_main_prompt
elif "video_sub_agent.md" in str(self):
return mock_sub_prompt
return ""
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.genai.Client",
return_value=mock_client,
),
patch(
"artemis.agents.video_analyzer.video_analyzer.get_controller",
return_value=mock_controller,
),
patch(
"pathlib.Path.exists",
autospec=True,
side_effect=lambda self: not str(self).endswith(".jpg"),
),
patch(
"pathlib.Path.stat",
return_value=os.stat_result((0o100644, 0, 0, 0, 0, 0, 1024 * 1024, 0, 0, 0)),
),
patch("pathlib.Path.read_text", mock_read_text),
patch("pathlib.Path.mkdir"),
patch("pathlib.Path.touch"),
patch("pathlib.Path.is_dir", return_value=True),
patch(
"artemis.agents.video_analyzer.video_analyzer.cleanup_abandoned_gemini_files",
AsyncMock(),
),
patch(
"artemis.agents.video_analyzer.video_analyzer.compress_video_for_api",
AsyncMock(return_value=Path("/tmp/video.mp4")),
),
patch(
"asyncio.create_subprocess_exec",
AsyncMock(return_value=MagicMock(communicate=AsyncMock(return_value=(b"", b"")))),
),
):
analyzer = VideoAnalyzer(mock_ctx)
result, status = await analyzer.run(
time_description="from 10s to 15s", purpose="Verify login"
)
assert status == "success"
assert sub_calls == 1
assert len(analyzer.blackboard_entries) == 1
entry = analyzer.blackboard_entries[0]
assert entry["confidence_score"] == 0.8
assert entry["summary"] == "Valid confidence score call"
# Verify that sub-agent turn 2 received the model turn with the thought_signature
assert len(captured_sub_contents) >= 2
sub_turn_2_contents = captured_sub_contents[1]
model_turns = [c for c in sub_turn_2_contents if c.role == "model"]
assert len(model_turns) == 1
assert model_turns[0].parts[0].thought_signature == sub_dummy_signature
@@ -0,0 +1,234 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Universal VideoAnalyzer dual-path execution engine."""
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import pytest
from langchain_core.messages import AIMessage
from artemis.agents.video_analyzer.video_analyzer import (
UNIVERSAL_MAIN_TOOLS,
UNIVERSAL_SUBMIT_ANSWER_TOOL,
VideoAnalyzer,
)
from artemis.context import ArtemisContext
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
ctx.llm_config = Mock()
mock_llm_cfg = Mock()
mock_llm_cfg.model = "claude-3-7-sonnet"
mock_llm_cfg.temperature = 0.2
ctx.llm_config.utils = Mock()
ctx.llm_config.utils.video_analyzer = mock_llm_cfg
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
return ctx
def test_video_analyzer_universal_engine_detection(mock_context):
"""Test that VideoAnalyzer routes to Universal Engine when non-Gemini model is set."""
from artemis.config import settings
with patch.object(settings, "GOOGLE_API_KEY", None):
agent = VideoAnalyzer(mock_context)
assert agent.use_native_gemini is False
assert agent.model_name == "claude-3-7-sonnet"
def test_universal_tool_schemas():
"""Verify schema structures for Universal tools."""
assert UNIVERSAL_SUBMIT_ANSWER_TOOL["type"] == "function"
assert UNIVERSAL_SUBMIT_ANSWER_TOOL["function"]["name"] == "submit_answer"
assert len(UNIVERSAL_MAIN_TOOLS) == 3
tool_names = [t["function"]["name"] for t in UNIVERSAL_MAIN_TOOLS]
assert "extract_segment_metadata" in tool_names
assert "spawn_sub_agent" in tool_names
assert "analyze_audio_only" in tool_names
@pytest.mark.asyncio
async def test_exec_single_chunk_universal(mock_context, tmp_path):
"""Test keyframe extraction and LangChain tool invocation in universal sub-agent."""
agent = VideoAnalyzer(mock_context)
agent.use_native_gemini = False
fake_video = tmp_path / "compressed.mp4"
fake_video.write_bytes(b"video_bytes")
raw_video = tmp_path / "raw.mp4"
raw_video.write_bytes(b"raw_bytes")
fake_keyframes = [
(0.0, b"frame0"),
(1.0, b"frame1"),
(2.0, b"frame2"),
]
mock_llm = MagicMock()
mock_response = AIMessage(
content="Sub-agent analyzed frames.",
tool_calls=[
{
"name": "submit_answer",
"args": {
"summary": "Detected settings button tap.",
"analysis": "At 1.0s user clicked Settings icon.",
"timeline_events": [
{
"start_time": 0.5,
"end_time": 1.5,
"transcription": "Tap on Settings",
"confidence_score": 0.95,
"verification_timestamp_secs": 1.0,
}
],
},
"id": "sub_call_1",
}
],
)
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(return_value=mock_response)
mock_llm.bind_tools.return_value = mock_bound
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.extract_keyframes_from_video",
return_value=fake_keyframes,
),
patch(
"artemis.agents.video_analyzer.video_analyzer.get_llm",
return_value=mock_llm,
),
):
result = await agent._exec_single_chunk_universal(
compressed_path=fake_video,
raw_path=raw_video,
start_time=0.0,
end_time=3.0,
actual_start=0.0,
prompt_with_context="Find when settings was opened.",
specific_query="settings icon",
)
assert "Detected settings button tap" in result
assert "At 1.0s user clicked Settings icon" in result
assert len(agent.blackboard_entries) == 1
assert agent.blackboard_entries[0]["summary"] == "Tap on Settings"
assert agent.blackboard_entries[0]["confidence_score"] == 0.95
@pytest.mark.asyncio
async def test_exec_analyze_audio_universal(mock_context, tmp_path):
"""Test audio analysis in universal mode."""
agent = VideoAnalyzer(mock_context)
agent.use_native_gemini = False
fake_audio = tmp_path / "audio.aac"
fake_audio.write_bytes(b"audio_bytes")
mock_llm = MagicMock()
mock_response = AIMessage(
content="",
tool_calls=[
{
"name": "submit_answer",
"args": {
"summary": "Heard notification sound.",
"analysis": "Notification bell sounded at 2.0s.",
},
"id": "audio_call_1",
}
],
)
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(return_value=mock_response)
mock_llm.bind_tools.return_value = mock_bound
with patch(
"artemis.agents.video_analyzer.video_analyzer.get_llm",
return_value=mock_llm,
):
result = await agent._exec_analyze_audio_universal(
audio_path=fake_audio,
start_time=1.0,
end_time=4.0,
actual_start=1.0,
prompt_with_context="Listen for notification sound.",
specific_query="notification bell",
)
assert "Heard notification sound" in result
assert len(agent.blackboard_entries) == 1
assert agent.blackboard_entries[0]["summary"] == "Heard notification sound."
@pytest.mark.asyncio
async def test_run_universal_multi_turn_reasoning(mock_context):
"""Test main reasoning loop of VideoAnalyzer in universal mode with tool dispatch."""
agent = VideoAnalyzer(mock_context)
agent.use_native_gemini = False
# Turn 1: Call extract_segment_metadata
msg_turn1 = AIMessage(
content="Let me check the video metadata first.",
tool_calls=[
{
"name": "extract_segment_metadata",
"args": {"start_time": 0.0, "end_time": 10.0},
"id": "main_call_1",
}
],
)
# Turn 2: Final response
msg_turn2 = AIMessage(
content="The video shows the app launched successfully at 3.5s.",
tool_calls=[],
)
mock_llm = MagicMock()
mock_bound = MagicMock()
mock_bound.ainvoke = AsyncMock(side_effect=[msg_turn1, msg_turn2])
mock_llm.bind_tools.return_value = mock_bound
with (
patch(
"artemis.agents.video_analyzer.video_analyzer.get_llm",
return_value=mock_llm,
),
patch.object(
agent,
"exec_extract_segment_metadata",
new_callable=AsyncMock,
return_value='{"duration_seconds": 10.0, "file_size_mb": 2.5}',
),
):
outcome, status = await agent._run_universal(
time_description="from 0s to 10s",
purpose="Verify app launch",
system_prompt="You are a video analyzer.",
)
assert status == "success"
assert "launched successfully at 3.5s" in outcome
assert agent.exec_extract_segment_metadata.call_count == 1
@@ -0,0 +1,167 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
import shutil
import tempfile
from unittest.mock import MagicMock
from uuid import uuid4
from langchain_core.messages import AIMessage, HumanMessage
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
from artemis.data_engine.trace import CURRENT_TRACE_ID, DataEngineCallbackHandler
from artemis.utils.task_tree import build_plan_and_history
import pytest
@pytest.fixture
def temp_workspace():
tmpdir = tempfile.mkdtemp()
yield Path(tmpdir)
shutil.rmtree(tmpdir)
@pytest.mark.asyncio
async def test_end_to_end_thoughts_non_duplication(temp_workspace):
# 1. Setup mock context and DataEngine
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(temp_workspace / "traces")
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
mock_ctx.llm_config = MagicMock()
engine = DataEngine(mock_ctx)
mock_ctx.data_engine = engine
# Start session
engine.start_session("Test initial goal")
# 2. Allocate step ID
step_id = engine.allocate_step_id()
# 3. Simulate Operator Agent run starts
operator_trace_id = uuid4()
engine.record_trace(
type="agent",
name="operator",
payload={"args": {}},
trace_id=operator_trace_id,
step_id=step_id,
status="running",
)
# Set context variables for parenting
token = CURRENT_TRACE_ID.set(operator_trace_id)
try:
# 4. Simulate LLM Call callback sequence
handler = DataEngineCallbackHandler(mock_ctx)
llm_run_id = uuid4()
# Start model
handler.on_chat_model_start(
serialized={"name": "ChatGoogleGenerativeAI"},
messages=[[HumanMessage(content="What should I do?")]],
run_id=llm_run_id,
parent_run_id=None,
)
# End model (generating thought + action)
class MockGeneration:
def __init__(self, message):
self.message = message
class MockLLMResult:
def __init__(self, generations):
self.generations = generations
response_message = AIMessage(
content=[
{
"type": "thinking",
"thinking": "I need to tap the search bar to find books.",
},
{"type": "text", "text": "Tapping search bar."},
]
)
mock_response = MockLLMResult(generations=[[MockGeneration(response_message)]])
handler.on_llm_end(mock_response, run_id=llm_run_id)
finally:
CURRENT_TRACE_ID.reset(token)
# 5. Finish Operator Agent trace
engine.record_trace(
type="agent",
name="operator",
payload={"result": "Success"},
trace_id=operator_trace_id,
step_id=step_id,
status="success",
)
# 6. Record step
engine.record_step(
pre_screenshot_bytes=b"pre_screenshot",
ui_tree=[],
ocr_result=[],
action_taken=[{"action": "tap", "coordinates": [100, 200]}],
operator_raw_thinking="I need to tap the search bar to find books.",
operator_native_thinking="I need to tap the search bar to find books.",
last_execution_result={"status": "success"},
summary="Tapped search bar",
)
# Flush any pending tasks
await engine.shutdown()
# 7. Retrieve agent-friendly steps
steps = engine.get_agent_friendly_steps()
# Assert step was retrieved
assert len(steps) == 1
step_data = steps[0]
# Assert thoughts are present in interleaved events (since we associated step_id now!)
events = step_data.get("interleaved_events", [])
thought_events = [e for e in events if e.get("type") in ("thought", "native_thought")]
assert len(thought_events) > 0
assert any(
e["content"] == "I need to tap the search bar to find books." for e in thought_events
)
# 8. Build history and check for duplications
plan = "- [ ] Search for books"
history = build_plan_and_history(
plan, steps, "default", last_n_detailed=1, for_failure_analyzer=True
)
print("\n--- Generated History Output ---")
print(history)
print("--------------------------------")
# Assert thought is printed
assert "I need to tap the search bar to find books." in history
# Assert thought is NOT printed multiple times
# Count occurrences of the thought string
count = history.count("I need to tap the search bar to find books.")
# The thought might appear in native thinking block and raw thinking block if both were stored.
# In our test we set BOTH operator_raw_thinking and operator_native_thinking to the same string,
# AND the LLM trace generated both "thinking" block and "text" block.
# Let's count how many times it was actually rendered in the Operator Decision Loop.
# It should appear at most 2 times (once as Native Thought, once as Thought/Raw Thought).
# If there was duplicate appending due to fallback failure, it would appear 4+ times.
assert count <= 2
+185
View File
@@ -0,0 +1,185 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
def test_get_or_create_image_updates_missing_data(tmp_path):
# Setup mock context
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
# Initialize DataEngine
engine = DataEngine(mock_ctx)
image_bytes = b"fake_image_bytes"
# 1. Save image without OCR/UI tree
image_name1 = engine.get_or_create_image(image_bytes)
# Verify it was saved with None for OCR/UI tree
record1 = engine.storage.get_image(image_name1)
assert record1 is not None
assert record1.ocr_result is None
assert record1.ui_tree is None
# 2. Save same image with OCR/UI tree
fake_ocr = [{"text": "hello", "box": [0, 0, 10, 10]}]
fake_ui = [{"resource-id": "button1", "text": "click me"}]
image_name2 = engine.get_or_create_image(image_bytes, ui_tree=fake_ui, ocr_result=fake_ocr)
assert image_name1 == image_name2
# Verify it now has OCR/UI tree
record2 = engine.storage.get_image(image_name2)
assert record2 is not None
assert record2.ocr_result == fake_ocr
assert record2.ui_tree == fake_ui
def test_create_image_duplicate_handling(tmp_path):
from artemis.data_engine.storage import StorageManager
from artemis.data_engine.models import ImageRecord
storage = StorageManager(tmp_path / "data_engine.db", tmp_path)
image_record = ImageRecord(
image_name="test_image_hash",
timestamp=123.456,
ocr_result=None,
ui_tree=None,
extra_metadata={},
)
# First insert
storage.create_image(image_record)
# Second insert with same image_name should not raise IntegrityError due to INSERT OR IGNORE
storage.create_image(image_record)
record = storage.get_image("test_image_hash")
assert record is not None
assert record.image_name == "test_image_hash"
def test_video_recording_persistence(tmp_path):
from uuid import uuid4
from artemis.data_engine.storage import StorageManager
from artemis.data_engine.models import VideoRecordingRecord
storage = StorageManager(tmp_path / "data_engine.db", tmp_path)
video_id = uuid4()
session_id = uuid4()
record = VideoRecordingRecord(
video_id=video_id,
session_id=session_id,
device_id="emulator-5554",
start_time=100.0,
end_time=None,
local_video_path="/tmp/recording.mp4",
)
# Test insert
storage.create_video_recording(record)
persisted = storage.get_video_recording(video_id)
assert persisted is not None
assert persisted.video_id == video_id
assert persisted.session_id == session_id
assert persisted.device_id == "emulator-5554"
assert persisted.start_time == 100.0
assert persisted.end_time is None
assert persisted.local_video_path == "/tmp/recording.mp4"
# Test update
record.end_time = 150.0
record.local_video_path = "/tmp/recording_final.mp4"
storage.update_video_recording(record)
updated = storage.get_video_recording(video_id)
assert updated is not None
assert updated.end_time == 150.0
assert updated.local_video_path == "/tmp/recording_final.mp4"
def test_record_step_suppresses_identical_post_screenshot(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
engine = DataEngine(mock_ctx)
engine.start_session("test_goal")
same_image_bytes = b"identical_screen_bytes_123"
step_id = engine.record_step(
pre_screenshot_bytes=same_image_bytes,
post_screenshot_bytes=same_image_bytes,
summary="Test step with identical screenshot",
)
for t in list(engine._pending_threads):
t.join()
step_record = engine.storage.get_steps(engine.current_session_id)[0]
assert step_record is not None
assert step_record.pre_image_name is not None
# post_image_name must be None when no visual transition occurred
assert step_record.post_image_name is None
def test_update_step_execution_result_suppresses_identical_post_screenshot(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
engine = DataEngine(mock_ctx)
engine.start_session("test_goal")
image_bytes = b"pre_screen_bytes_456"
step_id = engine.record_step(
pre_screenshot_bytes=image_bytes,
summary="Initial step",
)
step_record = engine.storage.get_steps(engine.current_session_id)[0]
assert step_record.post_image_name is None
# Update with post_image_name matching pre_image_name
engine.storage.update_step_execution_result(
step_id,
{"status": "success"},
post_image_name=step_record.pre_image_name,
)
updated_step = engine.storage.get_steps(engine.current_session_id)[0]
assert updated_step.post_image_name is None
# Update with distinct post_image_name
engine.storage.update_step_execution_result(
step_id,
{"status": "success"},
post_image_name="distinct_new_hash_789",
)
updated_step_distinct = engine.storage.get_steps(engine.current_session_id)[0]
assert updated_step_distinct.post_image_name == "distinct_new_hash_789"
@@ -0,0 +1,566 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import base64
import hashlib
import json
import logging
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from langchain_core.messages import HumanMessage
from artemis.context import ArtemisContext
from artemis.data_engine.engine import DataEngine
from artemis.graph.state import State
from artemis.utils.logger import DataEngineHandler
import pytest
@pytest.fixture
def mock_artemis_ctx(tmp_path):
mock_ctx = MagicMock(spec=ArtemisContext)
mock_execution_setup = MagicMock()
mock_execution_setup.traces_path = str(tmp_path)
mock_ctx.execution_setup = mock_execution_setup
mock_ctx.device = None
mock_ctx.task_plan_snapshot = None
return mock_ctx
@pytest.mark.asyncio
async def test_data_engine_handler_queue_buffering_and_background_worker(
mock_artemis_ctx,
):
"""Verify DataEngineHandler.emit buffers log records via in-memory queue without spawning OS threads per log line."""
engine = DataEngine(mock_artemis_ctx)
session_id = engine.start_session("Test Log Queue")
handler = DataEngineHandler()
with patch("sys.modules") as mock_modules:
mock_engine_mod = MagicMock()
mock_engine_mod._CURRENT_DATA_ENGINE = engine
mock_modules.get.return_value = mock_engine_mod
# Emit 50 rapid log records
for i in range(50):
record = logging.LogRecord(
name=f"test.logger.{i}",
level=logging.INFO,
pathname="test.py",
lineno=10 + i,
msg=f"Async queue test log message {i}",
args=(),
exc_info=None,
)
handler.emit(record)
# Wait for the background worker thread of DataEngineHandler and DataEngine tasks to drain
await asyncio.sleep(0.4)
if engine._pending_tasks:
await asyncio.gather(*engine._pending_tasks)
# Verify that traces were recorded and persisted accurately via direct SQLite query
with engine.storage._get_connection() as conn:
cursor = conn.execute(
"SELECT * FROM traces WHERE session_id = ? AND type = 'log'",
(str(session_id),),
)
log_rows = cursor.fetchall()
assert len(log_rows) >= 50
messages = {json.loads(row["payload"])["message"] for row in log_rows}
for i in range(50):
assert f"Async queue test log message {i}" in messages
@pytest.mark.asyncio
async def test_update_step_action_and_metadata_background_offloading(mock_artemis_ctx, tmp_path):
"""Verify that update_step_* methods publish SSE events instantly in memory and offload SQLite & disk I/O to background."""
engine = DataEngine(mock_artemis_ctx)
session_id = engine.start_session("Test Step Offloading")
step_id = engine.allocate_step_id()
# Create the initial step
engine.record_step(summary="Initial step summary")
if engine._pending_tasks:
await asyncio.gather(*engine._pending_tasks)
# Mock _publish to verify immediate real-time SSE delivery
published_events = []
engine._publish = lambda event_type, payload: published_events.append((event_type, payload))
fake_action = {"action": "tap", "coordinates": [100, 200]}
fake_post_jpg = b"fake_post_screenshot_jpg_bytes_12345"
# Call update_step_action with post screenshot bytes
engine.update_step_action(fake_action, post_screenshot_bytes=fake_post_jpg)
# Verify SSE was published immediately before/during task execution
assert len(published_events) == 1
assert published_events[0][0] == "step_updated"
assert published_events[0][1]["action_taken"] == fake_action
# Call other update_step_* methods
engine.update_step_summary(step_id, "Updated concise summary")
engine.update_step_thinking(step_id, "Raw thinking process...")
engine.update_step_native_thinking(step_id, "Native thinking process...")
engine.update_step_execution_result(
step_id, {"status": "success"}, post_image_name="hash_post_123"
)
# Now await all background pending tasks
if engine._pending_tasks:
await asyncio.gather(*engine._pending_tasks)
# Verify SQLite record has been correctly updated by background tasks
steps = engine.storage.get_steps(session_id)
step_record = next((s for s in steps if s.step_id == step_id), None)
assert step_record is not None
assert step_record.action_taken == fake_action
assert step_record.summary == "Updated concise summary"
assert step_record.operator_raw_thinking == "Raw thinking process..."
assert step_record.operator_native_thinking == "Native thinking process..."
assert step_record.last_execution_result == {"status": "success"}
assert step_record.post_image_name == "hash_post_123"
# Verify post.jpg was written to disk inside step directory
post_jpg_path = engine.current_step_dir / "post.jpg"
assert post_jpg_path.exists()
assert post_jpg_path.read_bytes() == fake_post_jpg
@pytest.mark.asyncio
async def test_record_step_background_image_and_step_persistence(
mock_artemis_ctx,
):
"""Verify record_step immediately calculates SHA-256 hashes and offloads get_or_create_image + create_step to background."""
engine = DataEngine(mock_artemis_ctx)
session_id = engine.start_session("Test Record Step Image Offloading")
step_id = engine.allocate_step_id()
pre_img_bytes = b"fake_pre_screenshot_data_001"
post_img_bytes = b"fake_post_screenshot_data_002"
expected_pre_hash = hashlib.sha256(pre_img_bytes).hexdigest()
expected_post_hash = hashlib.sha256(post_img_bytes).hexdigest()
returned_step_id = engine.record_step(
pre_screenshot_bytes=pre_img_bytes,
post_screenshot_bytes=post_img_bytes,
summary="Testing background image persistence",
)
assert returned_step_id == step_id
# Await background storage tasks
if engine._pending_tasks:
await asyncio.gather(*engine._pending_tasks)
# Verify image records were created in SQLite
pre_record = engine.storage.get_image(expected_pre_hash)
post_record = engine.storage.get_image(expected_post_hash)
assert pre_record is not None
assert pre_record.image_name == expected_pre_hash
assert post_record is not None
assert post_record.image_name == expected_post_hash
# Verify image files exist on disk
pre_path = engine.get_image_path(expected_pre_hash)
post_path = engine.get_image_path(expected_post_hash)
assert pre_path.exists() and pre_path.read_bytes() == pre_img_bytes
assert post_path.exists() and post_path.read_bytes() == post_img_bytes
# Verify StepRecord has correct hashes and summary
steps = engine.storage.get_steps(session_id)
step_record = next((s for s in steps if s.step_id == step_id), None)
assert step_record is not None
assert step_record.pre_image_name == expected_pre_hash
assert step_record.post_image_name == expected_post_hash
assert step_record.summary == "Testing background image persistence"
@pytest.mark.asyncio
async def test_perception_node_async_offloading(mock_artemis_ctx, tmp_path):
"""Verify perception_node offloads injected_instruction reading/unlinking and get_or_create_image to background."""
from artemis.graph.perception import perception_node
engine = DataEngine(mock_artemis_ctx)
engine.start_session("Test Perception Node Offloading")
mock_artemis_ctx.data_engine = engine
# Create an injected_instruction.json file
instruction_file = Path(engine.base_dir) / "injected_instruction.json"
instruction_file.write_text(json.dumps({"instruction": "Avoid clicking Ads"}), encoding="utf-8")
# Prepare mock controller
fake_screenshot_bytes = b"fake_screenshot_for_perception_node_123"
fake_screenshot_b64 = base64.b64encode(fake_screenshot_bytes).decode("utf-8")
expected_hash = hashlib.sha256(fake_screenshot_bytes).hexdigest()
mock_screen_data = MagicMock()
mock_screen_data.base64 = fake_screenshot_b64
mock_screen_data.elements = [{"resource-id": "app_bar", "bounds": "[0,0][1080,200]"}]
mock_screen_data.width = 1080
mock_screen_data.height = 2400
with (
patch("artemis.graph.perception.UnifiedMobileController") as mock_controller_cls,
patch("artemis.graph.perception.perform_ocr", new_callable=AsyncMock) as mock_ocr,
patch("artemis.graph.perception._detect_status_bar_height", return_value=0),
patch("artemis.graph.perception._should_skip_settling", return_value=True),
):
mock_controller = mock_controller_cls.return_value
mock_controller.get_screen_data = AsyncMock(return_value=mock_screen_data)
mock_ocr.return_value = [{"text": "Home", "box": [100, 100, 200, 200]}]
state = State(
messages=[HumanMessage(content="test")],
initial_goal="test goal",
complete_subgoals_by_ids=[],
validator_messages=[],
latest_ui_hierarchy=[],
latest_screenshot=None,
focused_app_info=None,
device_date=None,
structured_decisions=None,
)
update = await perception_node(state, mock_artemis_ctx)
# Verify perception_node returned expected update immediately
assert update["injected_instruction"] == "Avoid clicking Ads"
assert update["latest_screenshot"] == str(engine.get_image_path(expected_hash))
assert update["operator_raw_data"]["width"] == 1080
# Verify injected_instruction.json was unlinked
assert not instruction_file.exists()
# Await background tasks triggered by perception_node
await asyncio.sleep(0.1)
if engine._pending_tasks:
await asyncio.gather(*engine._pending_tasks)
# Verify image record and file were created in background by perception_node
img_record = engine.storage.get_image(expected_hash)
assert img_record is not None
assert img_record.image_name == expected_hash
assert engine.get_image_path(expected_hash).exists()
@pytest.mark.asyncio
async def test_snapshot_copy_and_restore_offloading(mock_artemis_ctx, tmp_path):
"""Verify operator create_snapshot and graph rollback/restore execute asynchronously without blocking."""
from artemis.utils.file import create_snapshot, restore_snapshot
notes_dir = tmp_path / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
(notes_dir / "task_plan.md").write_text("# Plan\n- Step 1", encoding="utf-8")
(notes_dir / "verification_chat_1.json").write_text('{"messages": ["hello"]}', encoding="utf-8")
snapshot_dir = tmp_path / "notes_snapshot"
# Test async offloading of create_snapshot (operator pattern)
await asyncio.to_thread(create_snapshot, notes_dir, snapshot_dir)
assert snapshot_dir.exists()
assert (snapshot_dir / "task_plan.md").read_text(encoding="utf-8") == "# Plan\n- Step 1"
# Modify live task_plan.md
(notes_dir / "task_plan.md").write_text("# Plan\n- Step 1 (Modified)", encoding="utf-8")
# Test async offloading of restore_snapshot (graph pattern when checker fails)
def _restore_and_preserve():
chat_contents = {}
for p in notes_dir.glob("verification_chat_*.json"):
chat_contents[p.name] = p.read_text(encoding="utf-8")
restore_snapshot(snapshot_dir, notes_dir)
for name, content in chat_contents.items():
(notes_dir / name).write_text(content, encoding="utf-8")
await asyncio.to_thread(_restore_and_preserve)
assert (notes_dir / "task_plan.md").read_text(encoding="utf-8") == "# Plan\n- Step 1"
assert (notes_dir / "verification_chat_1.json").read_text(
encoding="utf-8"
) == '{"messages": ["hello"]}'
@pytest.mark.asyncio
async def test_ocr_api_persistent_http_client_singleton_and_tls_reuse():
"""Verify get_http_client returns a persistent singleton httpx.AsyncClient that reuses TLS connections across OCR calls."""
import httpx
from artemis.utils import ocr_api
# Reset singleton to test fresh creation & persistence
ocr_api._HTTP_CLIENT = None
client1 = ocr_api.get_http_client()
client2 = ocr_api.get_http_client()
assert isinstance(client1, httpx.AsyncClient)
assert client1 is client2
# Verify perform_ocr uses this exact persistent client
fake_ocr_resp = MagicMock()
fake_ocr_resp.status_code = 200
fake_ocr_resp.json.return_value = {
"responses": [
{
"textAnnotations": [
{
"description": "Combined text block",
"boundingPoly": {"vertices": []},
},
{
"description": "Hello World",
"boundingPoly": {"vertices": [{"x": 10, "y": 20}, {"x": 100, "y": 200}]},
},
]
}
]
}
with patch.object(client1, "post", new_callable=AsyncMock) as mock_post:
mock_post.return_value = fake_ocr_resp
result = await ocr_api.perform_ocr(screenshot_b64="ZHVtbXlfYjY0")
mock_post.assert_called_once()
assert len(result) == 1
assert result[0]["text"] == "Hello World"
@pytest.mark.asyncio
async def test_ui_filter_and_ui_automator_client_pre_parsed_bounds_o1_lookup():
"""Verify XML parsing pre-populates parsed_bounds and _parse_bounds hits this cache in O(1) without regex matches."""
import re
from artemis.clients.ui_automator_client import _parse_hierarchy_xml_to_elements
from artemis.utils.ui_filter import _parse_bounds
# 1. Test ingestion pre-parsing in _parse_hierarchy_xml_to_elements
xml_data = '<hierarchy><node text="Login Btn" bounds="[15,25][350,120]"/></hierarchy>'
elements = _parse_hierarchy_xml_to_elements(xml_data)
assert len(elements) == 1
assert elements[0]["parsed_bounds"] == {
"left": 15,
"top": 25,
"right": 350,
"bottom": 120,
}
# 2. Test _parse_bounds O(1) hit vs fallback
with patch.object(re, "match", wraps=re.match) as spy_match:
# Hit pre-parsed bounds -> re.match MUST NOT be called
hit_result = _parse_bounds(elements[0])
assert hit_result == {
"left": 15,
"top": 25,
"right": 350,
"bottom": 120,
}
spy_match.assert_not_called()
# Fallback raw string -> re.match is called exactly once
fallback_result = _parse_bounds("[50,60][150,160]")
assert fallback_result == {
"left": 50,
"top": 60,
"right": 150,
"bottom": 160,
}
assert spy_match.call_count == 1
@pytest.mark.asyncio
async def test_validator_pre_execution_loop_reverted_to_exact_safety_contract(
mock_artemis_ctx, tmp_path
):
"""Verify Validator pre-execution check runs VLM on XML failure, but retains XML error if VLM fails (reverted contract)."""
from artemis.agents.validator.validator import ValidatorNode, ValidationErrorCategory
# Set mock context properties required by tracing decorators
mock_artemis_ctx.data_engine = MagicMock()
node = ValidatorNode(mock_artemis_ctx)
session = MagicMock()
session.call_tool = AsyncMock(
return_value=MagicMock(content=[MagicMock(text="ZHVtbXlfZGF0YQ==")])
)
decisions = json.dumps(
[
{
"action": "tap",
"coordinates": [200, 250],
"target_text": "Login",
"target_bounds": [100, 200, 300, 300],
"target_resource_id": "btn_login",
}
]
)
state = State(
messages=[],
initial_goal="test",
complete_subgoals_by_ids=[],
validator_messages=[],
latest_ui_hierarchy=[],
latest_screenshot="dummy.png",
structured_decisions=decisions,
focused_app_info=None,
device_date=None,
)
with (
patch.object(
ValidatorNode, "_get_initial_screenshot", new_callable=AsyncMock
) as mock_get_screen,
patch.object(
ValidatorNode, "_validate_action_precondition", new_callable=AsyncMock
) as mock_xml,
patch.object(
ValidatorNode,
"_validate_action_precondition_pixel",
new_callable=AsyncMock,
) as mock_pixel,
patch(
"artemis.agents.validator.failure_analyzer.FailureAnalyzer.analyze",
new_callable=AsyncMock,
) as mock_analyze,
):
mock_get_screen.return_value = ("dummy_b64", "dummy_stem")
# 1. XML check returns failure (TARGET_OCCUPIED by 'Sign Up' button)
mock_xml.return_value = (
False,
ValidationErrorCategory.TARGET_OCCUPIED,
"Occupied by Sign Up button",
)
# 2. VLM check returns failure (PIXEL_TARGET_DISAPPEARED)
mock_pixel.return_value = (
False,
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
"VLM target missing",
)
mock_analyze.return_value = {
"status": "cannot_fix",
"analysis": "Unrepairable drift",
}
result = await node(state)
# Verify both XML check and VLM check were called (reverted double-check contract)
mock_xml.assert_called_once()
mock_pixel.assert_called_once()
# Verify FailureAnalyzer received the EXACT XML error (TARGET_OCCUPIED), NOT the VLM error
mock_analyze.assert_called_once()
args, kwargs = mock_analyze.call_args
assert kwargs.get("error_category") == ValidationErrorCategory.TARGET_OCCUPIED
assert "Occupied by Sign Up button" in args[2]
@pytest.mark.asyncio
async def test_failure_analyzer_prune_intermediate_screenshots(mock_artemis_ctx):
"""Verify FailureAnalyzer._prune_intermediate_screenshots keeps initial reference images and the latest live image, while pruning intermediate images."""
from google.genai import types
from artemis.agents.validator.failure_analyzer import FailureAnalyzer
analyzer = FailureAnalyzer(mock_artemis_ctx)
# Build simulated contents history:
# 0: User (Initial reference screenshots + prompt)
# 1: Model
# 2: User (Intermediate tool results)
# 3: User (Intermediate observation screenshot - SHOULD BE PRUNED)
# 4: Model
# 5: User (Intermediate tool results 2)
# 6: User (Latest observation screenshot - SHOULD BE PRESERVED)
contents = [
types.Content(
role="user",
parts=[
types.Part.from_text(text="Reference screenshot 1"),
types.Part.from_bytes(data=b"initial_pre_image_bytes", mime_type="image/jpeg"),
types.Part.from_bytes(data=b"initial_post_image_bytes", mime_type="image/jpeg"),
],
),
types.Content(role="model", parts=[types.Part.from_text(text="Clicked something")]),
types.Content(
role="user",
parts=[types.Part.from_text(text="Result text")],
),
types.Content(
role="user",
parts=[
types.Part.from_text(text="Intermediate observation"),
types.Part.from_bytes(
data=b"intermediate_image_bytes_to_prune",
mime_type="image/jpeg",
),
],
),
types.Content(role="model", parts=[types.Part.from_text(text="Clicking again")]),
types.Content(
role="user",
parts=[types.Part.from_text(text="Result text 2")],
),
types.Content(
role="user",
parts=[
types.Part.from_text(text="Latest live observation"),
types.Part.from_bytes(
data=b"latest_live_image_bytes_to_keep",
mime_type="image/jpeg",
),
],
),
]
analyzer._prune_intermediate_screenshots(contents)
# 1. Verify index 0 images are untouched
assert contents[0].parts[1].inline_data.data == b"initial_pre_image_bytes"
assert contents[0].parts[2].inline_data.data == b"initial_post_image_bytes"
# 2. Verify index 3 (intermediate image) is replaced by text placeholder
assert getattr(contents[3].parts[1], "inline_data", None) is None
assert "[Screenshot of intermediate step omitted for performance]" in contents[3].parts[1].text
# 3. Verify index 6 (latest image) is fully preserved
assert contents[6].parts[1].inline_data.data == b"latest_live_image_bytes_to_keep"
@pytest.mark.asyncio
async def test_record_step_pre_allocated_id_reset(mock_artemis_ctx):
"""Verify that after record_step is called, the pre-allocated current_step_id is reset to None so subsequent steps generate distinct IDs."""
from artemis.data_engine.engine import DataEngine
engine = DataEngine(mock_artemis_ctx)
engine.start_session("Pre-allocated ID Reset Test")
# 1. Pre-allocate ID for step 1
step_id_1 = engine.allocate_step_id()
assert engine.current_step_id == step_id_1
# 2. Record step 1 (should use step_id_1 and reset current_step_id to None)
returned_id_1 = engine.record_step(summary="Step 1")
assert returned_id_1 == step_id_1
assert engine.current_step_id is None
# 3. Pre-allocate ID for step 2 (should get a distinct new ID)
step_id_2 = engine.allocate_step_id()
assert step_id_2 != step_id_1
assert engine.current_step_id == step_id_2
# 4. Record step 2 (should use step_id_2 and reset current_step_id to None)
returned_id_2 = engine.record_step(summary="Step 2")
assert returned_id_2 == step_id_2
assert returned_id_2 != returned_id_1
assert engine.current_step_id is None
@@ -0,0 +1,189 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import MagicMock
from artemis.context import ArtemisContext
from artemis.data_engine.trace import (
TraceSpan,
smart_serialize,
)
import pytest
def test_smart_serialize():
# Test dict
d = {"a": 1, "b": "string"}
assert smart_serialize(d) == json.dumps(d, ensure_ascii=False)
# Test list
items_list = [1, 2, "three"]
assert smart_serialize(items_list) == json.dumps(items_list, ensure_ascii=False)
# Test bytes
b = b"hello world"
import hashlib
expected_sha = hashlib.sha256(b).hexdigest()[:8]
assert smart_serialize(b) == f"<Bytes length={len(b)} sha256={expected_sha}>"
# Test other types
assert smart_serialize(123) == "123"
assert smart_serialize("just a string") == "just a string"
@pytest.mark.asyncio
async def test_trace_span():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
with TraceSpan(name="test_span", ctx=mock_ctx) as span:
span.result = "span_result"
# Simulate work
# TraceSpan writes twice: start (running) and end (success)
assert mock_ctx.data_engine.record_trace.call_count == 2
# Verify the final success call (which is the last call)
args, kwargs = mock_ctx.data_engine.record_trace.call_args
assert kwargs["type"] == "span"
assert kwargs["name"] == "test_span"
assert kwargs["status"] == "success"
assert kwargs["payload"]["result"] == "span_result"
@pytest.mark.asyncio
async def test_trace_span_failure():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
try:
with TraceSpan(name="failed_span", ctx=mock_ctx):
raise ValueError("Span error")
except ValueError:
pass
# TraceSpan writes twice: start (running) and end (failed)
assert mock_ctx.data_engine.record_trace.call_count == 2
# Verify the final failed call (which is the last call)
args, kwargs = mock_ctx.data_engine.record_trace.call_args
assert kwargs["type"] == "span"
assert kwargs["name"] == "failed_span"
assert kwargs["status"] == "failed"
assert "error" in kwargs["payload"]
assert "Span error" in kwargs["payload"]["error"]
def test_data_engine_callback_handler():
from uuid import uuid4
from langchain_core.messages import HumanMessage, AIMessage
from artemis.data_engine.trace import DataEngineCallbackHandler
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_step_id = uuid4()
mock_ctx.data_engine.current_step_id = mock_step_id
handler = DataEngineCallbackHandler(mock_ctx)
run_id = uuid4()
# 1. Simulate on_chat_model_start
serialized = {"name": "test_model"}
messages = [[HumanMessage(content="Hello LLM")]]
handler.on_chat_model_start(serialized, messages, run_id=run_id)
assert mock_ctx.data_engine.record_trace.call_count == 1
call1_args, call1_kwargs = mock_ctx.data_engine.record_trace.call_args_list[0]
assert call1_kwargs["type"] == "llm_call"
assert call1_kwargs["name"] == "test_model"
assert call1_kwargs["status"] == "running"
assert call1_kwargs["step_id"] == mock_step_id
assert call1_kwargs["payload"]["messages"][0]["content"] == "Hello LLM"
# 2. Simulate on_llm_end (successful complete response)
class MockGeneration:
def __init__(self, message):
self.message = message
class MockLLMResult:
def __init__(self, generations):
self.generations = generations
response_message = AIMessage(content="Hello User")
mock_response = MockLLMResult(generations=[[MockGeneration(response_message)]])
handler.on_llm_end(mock_response, run_id=run_id)
assert mock_ctx.data_engine.record_trace.call_count == 3
call2_args, call2_kwargs = mock_ctx.data_engine.record_trace.call_args_list[1]
assert call2_kwargs["type"] == "llm_call"
assert call2_kwargs["status"] == "success"
assert call2_kwargs["step_id"] == mock_step_id
assert call2_kwargs["payload"]["messages"][0]["content"] == "Hello LLM"
assert call2_kwargs["payload"]["response"][0]["content"] == "Hello User"
assert isinstance(call2_kwargs.get("duration"), float)
assert call2_kwargs["duration"] >= 0.0
call3_args, call3_kwargs = mock_ctx.data_engine.record_trace.call_args_list[2]
assert call3_kwargs["type"] == "raw_thinking"
assert call3_kwargs["parent_trace_id"] == run_id
assert call3_kwargs["payload"]["thought"] == "Hello User"
def test_callback_handler_chain_parenting():
from uuid import uuid4
from langchain_core.messages import HumanMessage
from artemis.data_engine.trace import DataEngineCallbackHandler, CURRENT_TRACE_ID
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
handler = DataEngineCallbackHandler(mock_ctx)
agent_run_id = uuid4()
chain1_run_id = uuid4()
chain2_run_id = uuid4()
llm_run_id = uuid4()
token = CURRENT_TRACE_ID.set(agent_run_id)
try:
# Simulate chain start (top level, parent is agent_run_id)
handler.on_chain_start({}, {}, run_id=chain1_run_id, parent_run_id=None)
# Simulate nested chain start (parent is chain1_run_id)
handler.on_chain_start({}, {}, run_id=chain2_run_id, parent_run_id=chain1_run_id)
# Simulate LLM start inside nested chain (parent is chain2_run_id)
messages = [[HumanMessage(content="Hello Nested LLM")]]
handler.on_chat_model_start(
{"name": "test_nested_model"},
messages,
run_id=llm_run_id,
parent_run_id=chain2_run_id,
)
# Verify parent trace ID resolves to agent_run_id (since chains are not recorded)
assert mock_ctx.data_engine.record_trace.call_count == 1
args, kwargs = mock_ctx.data_engine.record_trace.call_args
assert kwargs["parent_trace_id"] == agent_run_id
# Cleanup
handler.on_chain_end({}, run_id=chain2_run_id)
handler.on_chain_end({}, run_id=chain1_run_id)
assert len(handler.chain_parents) == 0
finally:
CURRENT_TRACE_ID.reset(token)
+297
View File
@@ -0,0 +1,297 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.graph.graph import wrap_note_tool, wrap_update_note_tool
import pytest
@pytest.mark.asyncio
async def test_wrap_update_note_tool_valid_status_change(tmp_path):
base_dir = tmp_path
notes_dir = base_dir / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
task_plan_path = notes_dir / "task_plan.md"
original_plan = """# Test Plan
- [ ] Milestone 1
- [ ] Substep 1.1
- [ ] Milestone 2
"""
task_plan_path.write_text(original_plan, encoding="utf-8")
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = str(base_dir)
mock_ctx.execution_setup = None
mock_original_tool = MagicMock()
mock_original_tool.name = "update_note"
mock_original_tool.description = "Update a note"
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
content = task_plan_path.read_text(encoding="utf-8")
updated = content.replace(args["target"], args["replacement"])
task_plan_path.write_text(updated, encoding="utf-8")
return "Success"
target = "- [ ] Milestone 1"
replacement = "- [x] Milestone 1"
with patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=fake_invoke_tool,
):
# Instantiate inside patch context to capture the mocked import locally
wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool)
result = await wrapped_tool.ainvoke(
{
"key": "task_plan",
"target": target,
"replacement": replacement,
"tool_call_id": "test_call_id",
}
)
assert result == "Success"
updated_content = task_plan_path.read_text(encoding="utf-8")
assert "- [x] Milestone 1" in updated_content
assert "- [ ] Milestone 2" in updated_content
@pytest.mark.asyncio
async def test_wrap_update_note_tool_invalid_milestone_text_change(tmp_path):
base_dir = tmp_path
notes_dir = base_dir / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
task_plan_path = notes_dir / "task_plan.md"
original_plan = """# Test Plan
- [ ] Milestone 1
- [ ] Substep 1.1
- [ ] Milestone 2
"""
task_plan_path.write_text(original_plan, encoding="utf-8")
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = str(base_dir)
mock_ctx.execution_setup = None
mock_original_tool = MagicMock()
mock_original_tool.name = "update_note"
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
content = task_plan_path.read_text(encoding="utf-8")
updated = content.replace(args["target"], args["replacement"])
task_plan_path.write_text(updated, encoding="utf-8")
return "Success"
target = "- [ ] Milestone 1"
replacement = "- [ ] Milestone Renamed"
mock_validation = AsyncMock(return_value={"status": "failed", "feedback": "rejected"})
with (
patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=fake_invoke_tool,
),
patch(
"artemis.agents.planner.planner.run_async_planner_validation",
mock_validation,
),
):
wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool)
result = await wrapped_tool.ainvoke(
{
"key": "task_plan",
"target": target,
"replacement": replacement,
"tool_call_id": "test_call_id",
}
)
assert result == "Success"
assert mock_ctx.planner_task is not None
@pytest.mark.asyncio
async def test_wrap_update_note_tool_allow_subgoal_changes(tmp_path):
base_dir = tmp_path
notes_dir = base_dir / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
task_plan_path = notes_dir / "task_plan.md"
original_plan = """# Test Plan
- [ ] Milestone 1
- [ ] Substep 1.1
- [ ] Milestone 2
"""
task_plan_path.write_text(original_plan, encoding="utf-8")
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = str(base_dir)
mock_ctx.execution_setup = None
mock_original_tool = MagicMock()
mock_original_tool.name = "update_note"
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
content = task_plan_path.read_text(encoding="utf-8")
updated = content.replace(args["target"], args["replacement"])
task_plan_path.write_text(updated, encoding="utf-8")
return "Success"
target = " - [ ] Substep 1.1"
replacement = " - [x] Substep 1.1 Completed"
with patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=fake_invoke_tool,
):
wrapped_tool = wrap_update_note_tool(mock_ctx, mock_original_tool)
result = await wrapped_tool.ainvoke(
{
"key": "task_plan",
"target": target,
"replacement": replacement,
"tool_call_id": "test_call_id",
}
)
assert result == "Success"
updated_content = task_plan_path.read_text(encoding="utf-8")
assert "Substep 1.1 Completed" in updated_content
assert "- [ ] Milestone 1" in updated_content
@pytest.mark.asyncio
async def test_wrap_note_tool_invalid_rewrite(tmp_path):
base_dir = tmp_path
notes_dir = base_dir / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
task_plan_path = notes_dir / "task_plan.md"
original_plan = """# Test Plan
- [ ] Milestone 1
- [ ] Substep 1.1
- [ ] Milestone 2
"""
task_plan_path.write_text(original_plan, encoding="utf-8")
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = str(base_dir)
mock_ctx.execution_setup = None
mock_original_tool = MagicMock()
mock_original_tool.name = "save_note"
mock_original_tool.ainvoke = AsyncMock(return_value="Success")
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
task_plan_path.write_text(args["content"], encoding="utf-8")
return "Success"
corrupted_plan = """# Test Plan
- [ ] Milestone 1 (Altered)
- [ ] New Milestone Added
"""
mock_validation = AsyncMock(return_value={"status": "failed", "feedback": "rejected"})
with (
patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=fake_invoke_tool,
),
patch(
"artemis.agents.planner.planner.run_async_planner_validation",
mock_validation,
),
):
wrapped_tool = wrap_note_tool(mock_ctx, mock_original_tool)
result = await wrapped_tool.ainvoke(
{
"key": "task_plan",
"content": corrupted_plan,
"tool_call_id": "test_call_id",
}
)
assert result == "Success"
assert mock_ctx.planner_task is not None
@pytest.mark.asyncio
async def test_wrap_note_tool_valid_lexical_drift(tmp_path):
from langgraph.types import Command
from langchain_core.messages import ToolMessage
from artemis.constants import VALIDATOR_MESSAGES_KEY
base_dir = tmp_path
notes_dir = base_dir / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
task_plan_path = notes_dir / "task_plan.md"
original_plan = """# Test Plan
- [ ] Open the Chrome browser and navigate to youtube.com
"""
task_plan_path.write_text(original_plan, encoding="utf-8")
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = str(base_dir)
mock_ctx.execution_setup = None
# Create mock Command result
tool_msg = ToolMessage(tool_call_id="test_call_id", content="Success outcome", status="success")
mock_command = Command(update={VALIDATOR_MESSAGES_KEY: [tool_msg]})
mock_original_tool = MagicMock()
mock_original_tool.name = "save_note"
mock_original_tool.ainvoke = AsyncMock(return_value=mock_command)
async def fake_invoke_tool(tool, args, tool_call_id, state, record_trace=None):
task_plan_path.write_text(args["content"], encoding="utf-8")
return mock_command
# Minor wording change that is semantically valid (similarity ~92%)
minor_drift_plan = """# Test Plan
- [ ] Open Chrome browser and navigate to youtube.com
"""
with patch(
"artemis.graph.graph.invoke_tool_with_injection",
side_effect=fake_invoke_tool,
):
wrapped_tool = wrap_note_tool(mock_ctx, mock_original_tool)
result = await wrapped_tool.ainvoke(
{
"key": "task_plan",
"content": minor_drift_plan,
"tool_call_id": "test_call_id",
}
)
assert isinstance(result, Command)
assert not hasattr(mock_ctx, "planner_task") or mock_ctx.planner_task is None
# Verify that the drift plan is actually saved
saved_content = task_plan_path.read_text(encoding="utf-8")
assert "Open Chrome browser" in saved_content
+146
View File
@@ -0,0 +1,146 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Base Agent Protocol & Lifecycle Registry."""
import pytest
from typing import Any
from artemis.agents.base import AgentConfig, AgentRegistry, BaseAgent
from artemis.agents.validator.failure_analyzer_agent import FailureAnalyzerAgent
from artemis.agents.validator.recovery_strategies import (
RecoveryStrategyRegistry,
)
from artemis.drivers.mock.mock_driver import MockDeviceDriver
class DummyTestAgent(BaseAgent):
"""Simple test agent implementing BaseAgent protocol."""
def __init__(self, config: AgentConfig | None = None, ctx: Any | None = None, **kwargs: Any):
super().__init__(
name="dummy_test_agent",
role="Test Worker",
description="Agent used for testing lifecycle transitions.",
config=config,
ctx=ctx,
)
async def process(self, state: Any) -> dict[str, Any]:
return {"status": "success", "processed_state": True}
@pytest.mark.asyncio
async def test_base_agent_lifecycle():
"""Verify BaseAgent initialization, execution, and cleanup lifecycle."""
agent = DummyTestAgent()
assert agent.name == "dummy_test_agent"
assert not agent._is_initialized
# 1. Initialize
mock_ctx = object()
await agent.initialize(mock_ctx)
assert agent._is_initialized
assert agent.ctx is mock_ctx
# 2. Process
result = await agent.process({})
assert result.get("status") == "success"
assert result.get("processed_state") is True
# 3. Cleanup
await agent.cleanup()
assert not agent._is_initialized
def test_agent_registry():
"""Verify AgentRegistry registers and instantiates agents correctly."""
registered = AgentRegistry.list_agents()
assert "planner" in registered
assert "summarizer" in registered
assert "operator" in registered
assert "failure_analyzer" in registered
# Instantiate PlannerAgent via registry
planner = AgentRegistry.create_agent("planner", ctx=None)
assert planner.name == "planner"
assert planner.role == "Strategic Task Planner"
# Instantiate SummarizerAgent via registry
summarizer = AgentRegistry.create_agent("summarizer", ctx=None)
assert summarizer.name == "summarizer"
assert summarizer.role == "Execution History Summarizer"
# Instantiate OperatorAgent via registry
operator = AgentRegistry.create_agent("operator", ctx=None)
assert operator.name == "operator"
assert operator.role == "Mobile UI Interaction Operator"
# Instantiate FailureAnalyzerAgent via registry
fa = AgentRegistry.create_agent("failure_analyzer", ctx=None)
assert fa.name == "failure_analyzer"
assert fa.role == "Autonomous Self-Healing Failure Analyzer"
def test_agent_config():
"""Verify AgentConfig parameters can be customized."""
config = AgentConfig(
model_name="gemini-2.5-pro",
temperature=0.7,
max_tokens=2048,
enabled_tools=["click", "swipe"],
)
agent = DummyTestAgent(config=config)
assert agent.config.model_name == "gemini-2.5-pro"
assert agent.config.temperature == 0.7
assert agent.config.enabled_tools == ["click", "swipe"]
@pytest.mark.asyncio
async def test_recovery_strategies_and_failure_analyzer():
"""Verify self-healing recovery strategies and FailureAnalyzerAgent execution."""
driver = MockDeviceDriver(device_id="mock-rec-1")
# 1. Test dialog dismissal strategy
dialog_ctx = {"category": "ui_obstruction", "error_message": "Unexpected permission popup"}
dialog_res = await RecoveryStrategyRegistry.attempt_recovery(driver, dialog_ctx)
assert dialog_res is not None
assert dialog_res.success
assert dialog_res.action_type.value == "dismiss_dialog"
assert driver.action_history[-1].get("key") == "back"
# 2. Test navigate back strategy
nav_ctx = {"category": "wrong_screen"}
nav_res = await RecoveryStrategyRegistry.attempt_recovery(driver, nav_ctx)
assert nav_res is not None
assert nav_res.success
assert nav_res.action_type.value == "navigate_back"
# 3. Test scroll and search strategy
scroll_ctx = {"category": "element_not_found", "scroll_direction": "down"}
scroll_res = await RecoveryStrategyRegistry.attempt_recovery(driver, scroll_ctx)
assert scroll_res is not None
assert scroll_res.success
assert scroll_res.action_type.value == "scroll_and_retry"
# 4. Test FailureAnalyzerAgent process
class MockContext:
pass
ctx = MockContext()
setattr(ctx, "_active_driver", driver)
fa_agent = FailureAnalyzerAgent(ctx=ctx)
fa_result = await fa_agent.process({"failure_context": {"category": "wrong_screen"}})
assert fa_result["status"] == "recovered"
assert fa_result["recovery_action"] == "navigate_back"
+30
View File
@@ -0,0 +1,30 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
async def f():
return [1, 2]
async def m():
try:
val = await f()[-1]
print(val)
except Exception as e:
print(repr(e))
asyncio.run(m())
+65
View File
@@ -0,0 +1,65 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for ARTEMIS Unified CLI application."""
from typer.testing import CliRunner
from artemis.interfaces.cli.main import app
runner = CliRunner()
def test_cli_help():
"""Verify top-level CLI help returns status 0 and lists core subcommands."""
result = runner.invoke(app, ["--help"])
assert result.exit_code == 0
assert "Artemis: Autonomous Multimodal Android Agent" in result.output
assert "run" in result.output
assert "batch" in result.output
assert "bench" in result.output
assert "server" in result.output
assert "trace" in result.output
assert "mcp" in result.output
def test_cli_version():
"""Verify --version returns version banner."""
result = runner.invoke(app, ["--version"])
assert result.exit_code == 0
assert "Artemis Agent Platform" in result.output
def test_cli_run_help():
"""Verify 'artemis run --help' displays execution options."""
result = runner.invoke(app, ["run", "--help"])
assert result.exit_code == 0
assert "--profile" in result.output
assert "--locked-app" in result.output
assert "--traces-path" in result.output
def test_cli_batch_help():
"""Verify 'artemis batch --help' displays batch options."""
result = runner.invoke(app, ["batch", "--help"])
assert result.exit_code == 0
assert "--file" in result.output
assert "--delay" in result.output
def test_cli_trace_help():
"""Verify 'artemis trace --help' lists trace subcommands."""
result = runner.invoke(app, ["trace", "--help"])
assert result.exit_code == 0
assert "list" in result.output
assert "view" in result.output
+541
View File
@@ -0,0 +1,541 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for the unified artemis.config package."""
from pathlib import Path
import tempfile
from unittest.mock import MagicMock
import pytest
from pydantic import SecretStr
from artemis.config import (
CONFIG_DIR,
ROOT_DIR,
AgentGlobalConfig,
LLMConfig,
OutputConfig,
Settings,
cleanup_temp_dir,
clear_ipc_port,
clear_ls_address,
deep_merge_llm_config,
get_app_dir,
get_config_path,
get_data_engine_db_path,
get_default_llm_config,
get_default_traces_path,
get_temp_dir,
get_traces_dir,
load_agent_config,
read_ipc_port,
read_ls_address,
record_events,
write_ipc_port,
write_ls_address,
)
def test_paths_and_directories():
"""Verify central path resolution."""
assert ROOT_DIR.exists()
assert (ROOT_DIR / "artemis").exists()
assert CONFIG_DIR.exists()
assert (CONFIG_DIR / "artemis.jsonc").exists()
app_dir = get_app_dir()
assert isinstance(app_dir, Path)
traces_path = get_default_traces_path()
assert isinstance(traces_path, Path)
assert get_traces_dir() == traces_path
db_path = get_data_engine_db_path()
assert db_path.name == "data_engine.db"
temp_dir = get_temp_dir("test_subfolder")
assert temp_dir.exists()
assert temp_dir.name == "test_subfolder"
def test_config_file_resolver():
"""Test get_config_path for unified artemis.jsonc."""
artemis_path = get_config_path("artemis.jsonc")
assert artemis_path.exists()
with pytest.raises(FileNotFoundError):
get_config_path("non_existent_config_12345.json")
def test_settings_and_api_key_fallbacks(monkeypatch):
"""Test Settings model, API key fallbacks, and get/set methods."""
monkeypatch.delenv("GOOGLE_API_KEY", raising=False)
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
monkeypatch.delenv("GCP_API_KEY", raising=False)
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
custom_settings = Settings(
_env_file=None,
GOOGLE_API_KEY=None,
GEMINI_API_KEY=SecretStr("gemini_key_123"),
OPENAI_API_KEY=SecretStr("openai_key_456"),
OCR_API_KEY=SecretStr("ocr_key_789"),
)
# Automatic fallback should populate GOOGLE_API_KEY from GEMINI_API_KEY
assert custom_settings.GOOGLE_API_KEY is not None
assert custom_settings.GOOGLE_API_KEY.get_secret_value() == "gemini_key_123"
assert custom_settings.get_api_key("ocr").get_secret_value() == "ocr_key_789"
assert custom_settings.get_api_key("google").get_secret_value() == "gemini_key_123"
assert custom_settings.get_api_key("gemini").get_secret_value() == "gemini_key_123"
assert custom_settings.get_api_key("openai").get_secret_value() == "openai_key_456"
assert custom_settings.get_api_key("nonexistent") is None
# Test dynamic set_api_key
custom_settings.set_api_key("xai", "xai_test_key", persist_to_env=False)
assert custom_settings.get_api_key("xai").get_secret_value() == "xai_test_key"
def test_llm_config_parsing_and_merging():
"""Test LLMConfig parsing, agent querying, and deep merging."""
llm_cfg = get_default_llm_config()
assert isinstance(llm_cfg, LLMConfig)
assert llm_cfg.planner.provider in ("google", "openai", "openrouter", "xai", "vertexai")
assert llm_cfg.get_agent("planner") is not None
assert llm_cfg.get_utils("hopper") is not None
# Deep merge overrides
overrides = {
"planner": {
"model": "custom-model-v1",
"temperature": 0.7,
}
}
merged = deep_merge_llm_config(llm_cfg, overrides)
assert merged.planner.model == "custom-model-v1"
assert merged.planner.temperature == 0.7
def test_agent_config_loading():
"""Test AgentGlobalConfig parsing from agent_config.json / artemis.jsonc."""
agent_cfg = load_agent_config()
assert isinstance(agent_cfg, AgentGlobalConfig)
assert "operator" in agent_cfg.explorer_versions
assert agent_cfg.explorer.default_version == "flash"
assert agent_cfg.explorer.flash_mode == "flash"
assert agent_cfg.explorer.pro_mode == "flash"
assert agent_cfg.explorer.caching is True
assert "explorer" in agent_cfg.blacklisted_tools
assert agent_cfg.video_analyzer.enable_ledger is True
assert agent_cfg.planner_validation.enabled is False
assert agent_cfg.planner_validation.similarity_threshold == 0.85
assert agent_cfg.committee.enabled is False
assert agent_cfg.committee.debate_rounds == 2
assert agent_cfg.checker.enabled is False
assert agent_cfg.checker.max_iterations == 20
assert agent_cfg.checker.max_chat_rounds == 4
assert agent_cfg.outputter.enabled is True
assert agent_cfg.outputter.force_synthesis is False
assert agent_cfg.flash.max_turns == 30
assert agent_cfg.flash.explorer_mode == "flash"
assert agent_cfg.pro.explorer.mode == "flash"
assert agent_cfg.pro.checker.enabled is False
assert agent_cfg.pro.committee.enabled is False
assert agent_cfg.pro.planner_validation.enabled is False
assert agent_cfg.pro.video_analyzer.enable_ledger is True
def test_output_config_and_recording():
"""Test OutputConfig behavior and event recording."""
cfg = OutputConfig(output_description="Output a valid JSON summary")
assert cfg.needs_structured_format() is True
cfg_disabled = OutputConfig(output_description="Output JSON", enable_outputter=False)
assert cfg_disabled.needs_structured_format() is False
cfg_forced = OutputConfig(force_synthesis=True)
assert cfg_forced.needs_structured_format() is True
with tempfile.TemporaryDirectory() as tmpdir:
output_file = Path(tmpdir) / "events.json"
events_data = [{"event": "test_event", "status": "ok"}]
record_events(output_file, events_data)
assert output_file.exists()
assert "test_event" in output_file.read_text(encoding="utf-8")
def test_runtime_state_and_ipc():
"""Test IPC port and LS address state helpers."""
# Test IPC port
write_ipc_port(49152)
assert read_ipc_port() == 49152
clear_ipc_port()
# Test LS address
write_ls_address("localhost:12345")
assert read_ls_address() == "localhost:12345"
clear_ls_address()
# Test temp dir cleanup
test_temp_sub = get_temp_dir("test_cleanup_dir")
test_temp_file = test_temp_sub / "temp_marker.txt"
test_temp_file.write_text("temporary data")
assert test_temp_file.exists()
deleted = cleanup_temp_dir("test_cleanup_dir")
assert deleted >= 1
assert not test_temp_file.exists()
def test_planner_validation_builder_and_milestones():
"""Test AgentConfigBuilder methods for planner validation and validate_milestones threshold."""
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
from artemis.graph.graph import validate_milestones
# Default builder inherits from artemis.jsonc (enabled=False, similarity_threshold=0.85)
builder = AgentConfigBuilder()
cfg = builder.build()
assert cfg.disable_planner_validation is True
assert cfg.planner_validation_threshold == 0.85
# Fluent enabling
cfg_enabled = (
AgentConfigBuilder()
.with_planner_validation(enabled=True, similarity_threshold=0.90)
.build()
)
assert cfg_enabled.disable_planner_validation is False
assert cfg_enabled.planner_validation_threshold == 0.90
# Fluent disabling
cfg_disabled = AgentConfigBuilder().with_disable_planner_validation(True).build()
assert cfg_disabled.disable_planner_validation is True
# Test validate_milestones with custom thresholds
before = "- [ ] Tap Login button\n- [ ] Enter password"
after_minor = "- [ ] Tap the Login button\n- [ ] Enter password"
# Minor edit has ~0.94 similarity
assert validate_milestones(before, after_minor, similarity_threshold=0.85) is False
assert validate_milestones(before, after_minor, similarity_threshold=0.98) is True
@pytest.mark.asyncio
async def test_committee_builder_and_graph_mounting():
"""Test AgentConfigBuilder committee methods and graph mounting."""
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform, ExecutionSetup
from artemis.graph.graph import get_graph
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
# Default: disabled
cfg_default = AgentConfigBuilder().build()
assert cfg_default.enable_committee is False
assert cfg_default.committee_debate_rounds == 2
# Enabled via builder
cfg_enabled = AgentConfigBuilder().with_committee(enabled=True, debate_rounds=3).build()
assert cfg_enabled.enable_committee is True
assert cfg_enabled.committee_debate_rounds == 3
# Test mounting in graph
device = DeviceContext(
host_platform="LINUX",
mobile_platform=DevicePlatform.ANDROID,
device_id="dummy",
device_width=1080,
device_height=2400,
)
ctx_disabled = ArtemisContext(
device=device,
execution_setup=ExecutionSetup(enable_committee=False),
)
graph_disabled = await get_graph(ctx_disabled)
op_node_disabled = graph_disabled.nodes.get("operator")
assert op_node_disabled is not None
# Verify ask_committee is not mounted when disabled
op_tools_disabled = [t.name for t in op_node_disabled.bound.afunc.tools]
assert "ask_committee" not in op_tools_disabled
ctx_enabled = ArtemisContext(
device=device,
execution_setup=ExecutionSetup(enable_committee=True, committee_debate_rounds=3),
)
graph_enabled = await get_graph(ctx_enabled)
op_node_enabled = graph_enabled.nodes.get("operator")
assert op_node_enabled is not None
# Verify ask_committee is mounted when enabled
op_tools_enabled = [t.name for t in op_node_enabled.bound.afunc.tools]
assert "ask_committee" in op_tools_enabled
def test_checker_builder_and_context_propagation():
"""Test AgentConfigBuilder methods and context propagation for checker."""
from unittest.mock import MagicMock
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
from artemis.sdk.agent import Agent
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
# Default builder inherits from artemis.jsonc (enabled=False, max_iterations=20, max_chat_rounds=4)
builder = AgentConfigBuilder()
cfg = builder.build()
assert cfg.disable_checker is True
assert cfg.checker_max_iterations == 20
assert cfg.checker_max_chat_rounds == 4
# Fluent enabling
cfg_enabled = (
AgentConfigBuilder()
.with_checker(enabled=True, max_iterations=25, max_chat_rounds=5)
.build()
)
assert cfg_enabled.disable_checker is False
assert cfg_enabled.checker_max_iterations == 25
assert cfg_enabled.checker_max_chat_rounds == 5
# Fluent disabling
cfg_disabled = AgentConfigBuilder().with_disable_checker(True).build()
assert cfg_disabled.disable_checker is True
# Test propagation to ExecutionSetup via Agent._prepare_tracing
agent = Agent(config=cfg_enabled)
device = DeviceContext(
host_platform="LINUX",
mobile_platform=DevicePlatform.ANDROID,
device_id="dummy",
device_width=1080,
device_height=2400,
)
ctx = ArtemisContext(device=device)
mock_task = MagicMock()
mock_task.get_name.return_value = "test_task"
mock_task.request.record_trace = False
mock_task.request.name = "test_task"
mock_task.request.profile = None
mock_task.request.goal = "Test goal"
agent._prepare_tracing(mock_task, ctx)
assert ctx.execution_setup is not None
assert ctx.execution_setup.disable_checker is False
assert ctx.execution_setup.checker_max_iterations == 25
assert ctx.execution_setup.checker_max_chat_rounds == 5
def test_explorer_builder_and_resolution(monkeypatch):
"""Test AgentConfigBuilder explorer methods and multi-tier resolution logic."""
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
from artemis.sdk.agent import Agent
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
from artemis.tools.explorer_tool import resolve_explorer_version
# Default builder inherits from artemis.jsonc (default="flash", flash_mode="flash", pro_mode="flash", caching=True)
builder = AgentConfigBuilder()
cfg = builder.build()
assert cfg.explorer.default_version == "flash"
assert cfg.explorer.flash_mode == "flash"
assert cfg.explorer.pro_mode == "flash"
assert cfg.explorer.caching is True
# Fluent configuration with with_explorer
cfg_custom = (
AgentConfigBuilder()
.with_explorer(
version="pro",
flash_mode="flash",
pro_mode="ultra",
caching=False,
versions={"operator": "ultra", "validator": "pro"},
)
.build()
)
assert cfg_custom.explorer.default_version == "pro"
assert cfg_custom.explorer.flash_mode == "flash"
assert cfg_custom.explorer.pro_mode == "ultra"
assert cfg_custom.explorer.caching is False
assert cfg_custom.explorer_versions["operator"] == "ultra"
assert cfg_custom.explorer_versions["validator"] == "pro"
# Fluent configuration with with_explorer_version shorthand
cfg_shorthand = AgentConfigBuilder().with_explorer_version("ultra").build()
assert cfg_shorthand.explorer.default_version == "ultra"
# Context setup for resolution tests
device = DeviceContext(
host_platform="LINUX",
mobile_platform=DevicePlatform.ANDROID,
device_id="dummy",
device_width=1080,
device_height=2400,
)
ctx = ArtemisContext(device=device, agent_config=cfg_custom)
# 1. Explicit override takes highest priority
assert resolve_explorer_version(ctx, explicit_version="flash") == "flash"
assert resolve_explorer_version(ctx, explicit_version="pro") == "pro"
assert resolve_explorer_version(ctx, explicit_version="ultra") == "ultra"
# 2. Environment variable override
monkeypatch.setenv("ARTEMIS_EXPLORER_VERSION", "ultra")
assert resolve_explorer_version(ctx) == "ultra"
monkeypatch.delenv("ARTEMIS_EXPLORER_VERSION", raising=False)
# 3. Per-agent mapping in explorer_versions
assert resolve_explorer_version(ctx, agent_or_profile_name="operator") == "ultra"
assert resolve_explorer_version(ctx, agent_or_profile_name="validator") == "pro"
# 4. Pro profile vs Flash profile resolution
cfg_profiled = (
AgentConfigBuilder()
.with_explorer(flash_mode="flash", pro_mode="pro", default_version="flash", versions={})
.build()
)
ctx_profiled = ArtemisContext(device=device, agent_config=cfg_profiled)
assert resolve_explorer_version(ctx_profiled, agent_or_profile_name="flash") == "flash"
assert resolve_explorer_version(ctx_profiled, agent_or_profile_name="flash_runner") == "flash"
assert resolve_explorer_version(ctx_profiled, agent_or_profile_name="pro") == "pro"
assert resolve_explorer_version(ctx_profiled, agent_or_profile_name="operator") == "pro"
# 5. Propagation to ExecutionSetup via Agent._prepare_tracing
agent = Agent(config=cfg_custom)
mock_task = MagicMock()
mock_task.get_name.return_value = "explorer_test_task"
mock_task.request.record_trace = False
mock_task.request.name = "explorer_test_task"
mock_task.request.profile = None
mock_task.request.goal = "Test explorer propagation"
agent._prepare_tracing(mock_task, ctx)
assert ctx.execution_setup is not None
assert ctx.execution_setup.explorer_version == "pro"
assert ctx.execution_setup.explorer_flash_mode == "flash"
assert ctx.execution_setup.explorer_pro_mode == "ultra"
assert ctx.execution_setup.explorer_caching is False
def test_outputter_builder_and_context_propagation():
"""Test AgentConfigBuilder outputter methods and propagation to ExecutionSetup."""
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
from artemis.sdk.agent import Agent
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
# Default builder inherits from artemis.jsonc (enabled=True, force_synthesis=False)
builder = AgentConfigBuilder()
cfg = builder.build()
assert cfg.disable_outputter is False
assert cfg.outputter.enabled is True
assert cfg.outputter.force_synthesis is False
# Fluent configuration with with_outputter
cfg_custom = AgentConfigBuilder().with_outputter(enabled=True, force_synthesis=True).build()
assert cfg_custom.disable_outputter is False
assert cfg_custom.outputter.enabled is True
assert cfg_custom.outputter.force_synthesis is True
# Fluent disabling with with_disable_outputter
cfg_disabled = AgentConfigBuilder().with_disable_outputter(True).build()
assert cfg_disabled.disable_outputter is True
assert cfg_disabled.outputter.enabled is False
# Test propagation to ExecutionSetup via Agent._prepare_tracing
device = DeviceContext(
host_platform="LINUX",
mobile_platform=DevicePlatform.ANDROID,
device_id="dummy",
device_width=1080,
device_height=2400,
)
ctx = ArtemisContext(device=device)
agent = Agent(config=cfg_custom)
mock_task = MagicMock()
mock_task.get_name.return_value = "outputter_test_task"
mock_task.request.record_trace = False
mock_task.request.name = "outputter_test_task"
mock_task.request.profile = None
mock_task.request.goal = "Test outputter propagation"
agent._prepare_tracing(mock_task, ctx)
assert ctx.execution_setup is not None
assert ctx.execution_setup.disable_outputter is False
assert ctx.execution_setup.outputter.enabled is True
assert ctx.execution_setup.outputter.force_synthesis is True
def test_categorized_flash_and_pro_profile_builders():
"""Test with_flash_config and with_pro_config fluent builders and bidirectional sync."""
from artemis.config.agent import AgentGlobalConfig
from artemis.sdk.builders.agent_config_builder import AgentConfigBuilder
# 1. Test with_flash_config
cfg_flash = AgentConfigBuilder().with_flash_config(max_turns=15, explorer_mode="flash").build()
assert cfg_flash.flash.max_turns == 15
assert cfg_flash.flash.explorer_mode == "flash"
assert cfg_flash.explorer.flash_mode == "flash"
# 2. Test with_pro_config
cfg_pro = (
AgentConfigBuilder()
.with_pro_config(
explorer_mode="ultra",
planner_validation=True,
committee=True,
checker=True,
video_ledger=False,
)
.build()
)
assert cfg_pro.pro.explorer.mode == "ultra"
assert cfg_pro.explorer.pro_mode == "ultra"
assert cfg_pro.disable_planner_validation is False
assert cfg_pro.enable_committee is True
assert cfg_pro.disable_checker is False
assert cfg_pro.enable_video_ledger is False
# 3. Test AgentGlobalConfig bidirectional synchronization
raw_dict = {
"flash": {"max_turns": 25, "explorer_mode": "flash"},
"pro": {
"explorer": {"mode": "pro", "caching": False},
"checker": {"enabled": True, "max_iterations": 10},
"committee": {"enabled": True, "debate_rounds": 4},
},
}
synced = AgentGlobalConfig.model_validate(raw_dict)
assert synced.flash.max_turns == 25
assert synced.explorer.flash_mode == "flash"
assert synced.explorer.pro_mode == "pro"
assert synced.explorer.caching is False
assert synced.checker.enabled is True
assert synced.checker.max_iterations == 10
assert synced.committee.enabled is True
assert synced.committee.debate_rounds == 4
def test_agent_config_environment_variable_overrides(monkeypatch):
"""Test environment variable overrides for AgentGlobalConfig switches."""
from artemis.config.agent import load_agent_config
monkeypatch.setenv("ARTEMIS_CHECKER_ENABLED", "true")
monkeypatch.setenv("ARTEMIS_COMMITTEE_ENABLED", "1")
monkeypatch.setenv("ARTEMIS_PLANNER_VALIDATION_ENABLED", "yes")
monkeypatch.setenv("ARTEMIS_OUTPUTTER_ENABLED", "false")
monkeypatch.setenv("ARTEMIS_VIDEO_LEDGER_ENABLED", "0")
cfg = load_agent_config()
assert cfg.checker.enabled is True
assert cfg.pro.checker.enabled is True
assert cfg.committee.enabled is True
assert cfg.pro.committee.enabled is True
assert cfg.planner_validation.enabled is True
assert cfg.pro.planner_validation.enabled is True
assert cfg.outputter.enabled is False
assert cfg.video_analyzer.enable_ledger is False
assert cfg.pro.video_analyzer.enable_ledger is False
+30
View File
@@ -0,0 +1,30 @@
import asyncio
import pytest
from artemis.context import ArtemisContext, DeviceContext, DevicePlatform
@pytest.mark.asyncio
async def test_artemis_context_waits_for_background_tasks():
device = DeviceContext(
host_platform="LINUX",
mobile_platform=DevicePlatform.ANDROID,
device_id="dummy",
device_width=1080,
device_height=2400,
)
context = ArtemisContext(device=device)
task_run = False
async def dummy_task():
nonlocal task_run
await asyncio.sleep(0.1)
task_run = True
async with context:
task = asyncio.create_task(dummy_task())
context.background_tasks.append(task)
# After exiting the context, the task should have run to completion
assert task_run is True
+91
View File
@@ -0,0 +1,91 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Unified Base Device Driver and implementations."""
import pytest
from unittest.mock import MagicMock
from artemis.drivers.base import KeyCode, ScreenData, SwipeDirection
from artemis.drivers.mock.mock_driver import MockDeviceDriver
from artemis.drivers.android.adb_driver import AndroidAdbDriver
@pytest.mark.asyncio
async def test_mock_driver_actions():
"""Verify MockDeviceDriver records and performs all core actions."""
driver = MockDeviceDriver(device_id="test-mock", width=1080, height=2400)
assert driver.device_id == "test-mock"
assert driver.screen_size == (1080, 2400)
# 1. Screen data
screen_data = await driver.get_screen_data()
assert isinstance(screen_data, ScreenData)
assert screen_data.width == 1080
assert screen_data.height == 2400
assert screen_data.screenshot_base64 is not None
# 2. Tap
assert await driver.tap(500, 1000) is True
assert driver.action_history[-1]["action"] == "tap"
assert driver.action_history[-1]["x"] == 500
# 3. Swipe
assert await driver.swipe(100, 200, 300, 400) is True
assert driver.action_history[-1]["action"] == "swipe"
# 4. Swipe direction
assert await driver.swipe_direction(SwipeDirection.UP) is True
assert driver.action_history[-1]["direction"] == "up"
# 5. Input text
assert await driver.input_text("Hello Artemis") is True
assert driver.action_history[-1]["text"] == "Hello Artemis"
# 6. Press key
assert await driver.press_key(KeyCode.HOME) is True
assert driver.action_history[-1]["key"] == "home"
# 7. App lifecycle
assert await driver.launch_app("com.android.settings") is True
assert await driver.get_current_package() == "com.android.settings"
assert await driver.stop_app("com.android.settings") is True
@pytest.mark.asyncio
async def test_android_driver_with_mock_adb():
"""Verify AndroidAdbDriver formats shell commands properly."""
mock_adb_client = MagicMock()
mock_adb_device = MagicMock()
mock_adb_client.device.return_value = mock_adb_device
driver = AndroidAdbDriver(
device_id="emulator-5554",
adb_client=mock_adb_client,
width=1080,
height=2400,
)
# Test tap
await driver.tap(540, 1200)
mock_adb_device.shell.assert_called_with("input tap 540 1200")
# Test multi-tap
await driver.tap(540, 1200, times=3, delay_ms=100)
expected_multi = "input tap 540 1200 && sleep 0.100 && input tap 540 1200 && sleep 0.100 && input tap 540 1200"
mock_adb_device.shell.assert_called_with(expected_multi)
# Test press key
await driver.press_key(KeyCode.BACK)
mock_adb_device.shell.assert_called_with("input keyevent 4")
+67
View File
@@ -0,0 +1,67 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from google import genai
from google.genai import types
from google.genai.errors import APIError
import pytest
@pytest.fixture
def client():
"""Initializes the Gemini client. Requires GEMINI_API_KEY in environment."""
try:
return genai.Client()
except Exception as e:
pytest.skip(f"Could not initialize genai.Client: {e}")
def test_gemini_role_tool_fails(client):
"""Confirm that using role='tool' results in a 400 APIError from Gemini."""
contents = [
types.Content(
role="tool",
parts=[types.Part.from_text(text="This is a test payload for tool execution.")],
)
]
with pytest.raises(APIError) as exc_info:
client.models.generate_content(
model="gemini-3.6-flash",
contents=contents,
)
# Assert it failed specifically due to the invalid role string
error_msg = str(exc_info.value)
assert "Invalid role string" in error_msg or "400" in error_msg
def test_gemini_role_user_succeeds(client):
"""Confirm that using role='user' (which is the correct fix) succeeds."""
contents = [
types.Content(
role="user",
parts=[
types.Part.from_text(text="Acknowledge this message with 'ok' and nothing else.")
],
)
]
# This should not raise any exceptions
response = client.models.generate_content(
model="gemini-3.6-flash",
contents=contents,
)
assert response.text is not None
+211
View File
@@ -0,0 +1,211 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Dynamic Model Router."""
from artemis.llm.router import ModelEndpoint, ModelProvider, ModelRouter
def test_model_router_default_routes():
"""Verify default role routing returns expected models."""
router = ModelRouter()
# Planner route
planner_ep = router.get_endpoint("planner")
assert planner_ep.provider == ModelProvider.GEMINI
assert "pro" in planner_ep.model_name
# Operator route
operator_ep = router.get_endpoint("operator")
assert operator_ep.provider == ModelProvider.GEMINI
assert "flash" in operator_ep.model_name
def test_model_router_custom_route_override():
"""Verify dynamic role route override."""
router = ModelRouter()
custom_ep = ModelEndpoint(
provider=ModelProvider.OPENAI,
model_name="gpt-4o",
api_base="https://custom.api.endpoint/v1",
)
router.set_role_route("operator", custom_ep)
ep = router.get_endpoint("operator")
assert ep.provider == ModelProvider.OPENAI
assert ep.model_name == "gpt-4o"
assert ep.api_base == "https://custom.api.endpoint/v1"
def test_model_router_fallbacks():
"""Verify fallback endpoint configuration."""
router = ModelRouter()
fallbacks = [
ModelEndpoint(provider=ModelProvider.GEMINI, model_name="gemini-2.5-flash"),
ModelEndpoint(provider=ModelProvider.OPENAI, model_name="gpt-4o-mini"),
]
router.set_role_route("custom_worker", "gemini-2.5-pro", fallbacks=fallbacks)
fb = router.get_fallbacks("custom_worker")
assert len(fb) == 2
assert fb[0].model_name == "gemini-2.5-flash"
assert fb[1].model_name == "gpt-4o-mini"
def test_model_router_anthropic_instantiation():
"""Verify Anthropic Claude model instantiation with thinking budget."""
router = ModelRouter()
anth_ep = ModelEndpoint(
provider=ModelProvider.ANTHROPIC,
model_name="claude-3-7-sonnet-20250219",
api_key="sk-ant-test-key",
reasoning_effort="high",
)
model = router.instantiate_model(anth_ep)
assert model.model == "claude-3-7-sonnet-20250219"
assert getattr(model, "thinking", None) == {"type": "enabled", "budget_tokens": 32768}
def test_model_router_openai_instantiation():
"""Verify OpenAI model instantiation with reasoning effort."""
router = ModelRouter()
oai_ep = ModelEndpoint(
provider=ModelProvider.OPENAI,
model_name="o3-mini",
api_key="sk-test-key",
reasoning_effort="medium",
)
model = router.instantiate_model(oai_ep)
assert model.model_name == "o3-mini"
assert getattr(model, "reasoning_effort", None) == "medium"
def test_robust_chat_model_wrapper_grounding_google():
"""Verify RobustChatModelWrapper auto-injects google_search and server-side tool config for Gemini."""
from artemis.services.llm import RobustChatModelWrapper
from langchain_core.tools import tool
@tool
def dummy_tool(query: str) -> str:
"""A test tool."""
return query
class MockGeminiModel:
def __init__(self):
self.bound_tools = []
self.bound_kwargs = {}
def bind_tools(self, tools, *args, **kwargs):
self.bound_tools = tools
self.bound_kwargs = kwargs
return self
# Case 1: enable_grounding=True automatically attaches google_search and tool_config
ep_google = ModelEndpoint(
provider=ModelProvider.GOOGLE,
model_name="gemini-2.5-flash",
enable_grounding=True,
)
wrapper = RobustChatModelWrapper(MockGeminiModel(), endpoint=ep_google)
bound = wrapper.bind_tools([dummy_tool])
assert any(isinstance(t, dict) and "google_search" in t for t in bound.base_model.bound_tools)
assert bound.base_model.bound_kwargs.get("tool_config") == {
"include_server_side_tool_invocations": True
}
# Case 2: Explicit google_search in tools automatically ensures tool_config
ep_google_no_ground = ModelEndpoint(
provider=ModelProvider.GOOGLE,
model_name="gemini-2.5-flash",
enable_grounding=False,
)
wrapper_explicit = RobustChatModelWrapper(MockGeminiModel(), endpoint=ep_google_no_ground)
bound_explicit = wrapper_explicit.bind_tools([dummy_tool, {"google_search": {}}])
assert any(
isinstance(t, dict) and "google_search" in t for t in bound_explicit.base_model.bound_tools
)
assert bound_explicit.base_model.bound_kwargs.get("tool_config") == {
"include_server_side_tool_invocations": True
}
def test_robust_chat_model_wrapper_grounding_non_google():
"""Verify RobustChatModelWrapper ignores grounding and strips google_search for non-Google providers."""
from artemis.services.llm import RobustChatModelWrapper
from langchain_core.tools import tool
@tool
def dummy_tool(query: str) -> str:
"""A test tool."""
return query
class MockOpenAIModel:
def __init__(self):
self.bound_tools = []
self.bound_kwargs = {}
def bind_tools(self, tools, *args, **kwargs):
self.bound_tools = tools
self.bound_kwargs = kwargs
return self
ep_openai = ModelEndpoint(
provider=ModelProvider.OPENAI,
model_name="gpt-4o",
enable_grounding=True,
)
wrapper = RobustChatModelWrapper(MockOpenAIModel(), endpoint=ep_openai)
bound = wrapper.bind_tools([dummy_tool, {"google_search": {}}])
# Should ignore grounding and strip {"google_search": {}}
assert len(bound.base_model.bound_tools) == 1
assert bound.base_model.bound_tools[0].name == "dummy_tool"
assert "tool_config" not in bound.base_model.bound_kwargs
def test_chat_google_generative_ai_process_tool_config_patch():
"""Verify patched ChatGoogleGenerativeAI preserves include_server_side_tool_invocations even with tool_choice."""
from artemis.llm.router import ModelFactory
from langchain_google_genai import ChatGoogleGenerativeAI
from langchain_core.tools import tool
@tool
def dummy_tool(query: str) -> str:
"""A test tool."""
return query
# Trigger patching via Google model creation
ep = ModelEndpoint(provider=ModelProvider.GOOGLE, model_name="gemini-2.5-flash", api_key="fake")
model = ModelFactory.create_model(ep)
assert isinstance(model, ChatGoogleGenerativeAI)
formatted_tools = model._format_tools([dummy_tool, {"google_search": {}}], None)
# 1. With tool_choice="auto"
config_auto = model._process_tool_config("auto", None, formatted_tools)
assert config_auto is not None
assert config_auto.include_server_side_tool_invocations is True
# 2. With tool_choice="any"
config_any = model._process_tool_config("any", None, formatted_tools)
assert config_any is not None
assert config_any.include_server_side_tool_invocations is True
# 3. With explicit tool_config
config_explicit = model._process_tool_config(
None, {"include_server_side_tool_invocations": True}, formatted_tools
)
assert config_explicit is not None
assert config_explicit.include_server_side_tool_invocations is True
+46
View File
@@ -0,0 +1,46 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
from pathlib import Path
from artemis.agents.planner.planner import run_async_planner_validation
class DummyDataEngine:
def get_agent_friendly_steps(self):
return []
class DummyCtx:
data_engine = DummyDataEngine()
project_dir = str(Path(__file__).resolve().parent.parent.parent)
async def test():
content_before = "- [/] Click the button\n- [ ] Complete task"
content_after = (
"- [/] Click the button\n- [ ] Complete task\n- [ ] New subgoal appended at bottom"
)
res = await run_async_planner_validation(
ctx=DummyCtx(),
initial_goal="Do the task",
content_before=content_before,
content_after=content_after,
operator_raw_thinking="I need to add a new subgoal",
operator_native_thinking="Thinking...",
)
print(res)
asyncio.run(test())
+56
View File
@@ -0,0 +1,56 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for OperatorPromptBuilder and LoopDetector."""
from artemis.agents.operator.loop_detector import LoopDetector
from artemis.agents.operator.prompt_builder import OperatorPromptBuilder
def test_operator_prompt_builder():
"""Verify prompt rendering formats variables correctly."""
system_msg = OperatorPromptBuilder.build_system_message()
assert "ARTEMIS Operator" in system_msg
assert "click" in system_msg
human_msg = OperatorPromptBuilder.build_human_message(
goal="Open Settings",
sub_goal="Tap Network & internet",
current_turn=2,
history="1. Tapped Settings icon",
)
assert "Open Settings" in human_msg
assert "Tap Network & internet" in human_msg
assert "Current Turn: 2" in human_msg
def test_loop_detector():
"""Verify loop detector flags repeated action sequences."""
detector = LoopDetector(repetition_threshold=3)
assert not detector.is_loop_detected()
# Record distinct actions
detector.record_action("click", {"target": [100, 200]})
detector.record_action("swipe", {"action": "up"})
assert not detector.is_loop_detected()
# Record 3 identical actions
detector.record_action("click", {"target": [500, 500]})
detector.record_action("click", {"target": [500, 500]})
detector.record_action("click", {"target": [500, 500]})
assert detector.is_loop_detected()
# Reset
detector.reset()
assert not detector.is_loop_detected()
+66
View File
@@ -0,0 +1,66 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for the official Python SDK (ArtemisClient, Task, TaskResult)."""
import pytest
from artemis import ArtemisClient, Task, TaskResult
@pytest.mark.asyncio
async def test_artemis_client_run_mock():
"""Verify ArtemisClient runs a task through the execution pipeline."""
client = ArtemisClient(device_id="mock-test-dev", default_profile="pro")
assert client.device_id == "mock-test-dev"
assert client.default_profile == "pro"
result = await client.run(goal="Open Settings and enable Dark Theme")
assert isinstance(result, TaskResult)
assert result.status in ("completed", "running", "success")
assert result.trace_id.startswith("trace_") or len(result.trace_id) > 0
def test_task_model_validation():
"""Verify Task validation and default arguments."""
task = Task(goal="Search for flight prices", profile="flash", device_id="emulator-5554")
assert task.goal == "Search for flight prices"
assert task.profile == "flash"
assert task.device_id == "emulator-5554"
assert task.max_turns == 30
assert task.locked_package is None
@pytest.mark.asyncio
async def test_artemis_client_stream_run():
"""Verify ArtemisClient.stream_run yields real-time StreamEvents."""
from artemis import StreamEvent, StreamEventType
client = ArtemisClient(device_id="mock-stream-dev", default_profile="flash")
collected_events: list[StreamEvent] = []
def on_event(ev: StreamEvent):
collected_events.append(ev)
events: list[StreamEvent] = []
async for event in client.stream_run(
goal="Test streaming UI action flow",
profile="flash",
callbacks=[on_event],
):
events.append(event)
assert len(events) >= 2
assert events[0].event_type == StreamEventType.STATUS
assert events[0].payload.get("status") == "starting"
assert len(collected_events) == len(events)
+55
View File
@@ -0,0 +1,55 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Telemetry & Span Lifecycle Tracing."""
import pytest
from pathlib import Path
from artemis.telemetry.models import SpanType
from artemis.telemetry.tracer import TelemetryTracer
@pytest.mark.asyncio
async def test_telemetry_span_lifecycle(tmp_path: Path):
"""Verify TelemetryTracer properly records and persists nested spans."""
tracer = TelemetryTracer(trace_id="test-session-001", traces_dir=tmp_path)
async with tracer.start_span("parent_agent_step", span_type=SpanType.AGENT) as parent_span:
assert parent_span.status == "running"
assert parent_span.name == "parent_agent_step"
async with tracer.start_span(
"child_device_action", span_type=SpanType.DEVICE_ACTION, payload={"action": "click"}
) as child_span:
assert child_span.parent_id == parent_span.span_id
assert child_span.payload["action"] == "click"
# Spans should be persisted in JSONL
assert len(tracer.spans) == 2
child_recorded = tracer.spans[0]
parent_recorded = tracer.spans[1]
assert child_recorded.name == "child_device_action"
assert child_recorded.status == "success"
assert child_recorded.duration_ms is not None
assert parent_recorded.name == "parent_agent_step"
assert parent_recorded.status == "success"
# File existence
log_file = tmp_path / "telemetry_spans.jsonl"
assert log_file.exists()
content = log_file.read_text(encoding="utf-8")
assert "child_device_action" in content
assert "parent_agent_step" in content
+70
View File
@@ -0,0 +1,70 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for Universal Tool Protocol and Actions."""
import pytest
from artemis.drivers.mock.mock_driver import MockDeviceDriver
from artemis.tools.base import ToolRegistry
import artemis.tools.actions.device_actions # noqa: F401 - Register action tools
@pytest.mark.asyncio
async def test_tool_registry_registration():
"""Verify standard device actions are registered in ToolRegistry."""
click_tool = ToolRegistry.get("click")
assert click_tool is not None
assert click_tool.name == "click"
assert click_tool.category == "action"
swipe_tool = ToolRegistry.get("swipe")
assert swipe_tool is not None
assert swipe_tool.name == "swipe"
@pytest.mark.asyncio
async def test_tool_execution_with_driver():
"""Verify tool execution correctly dispatches to MockDeviceDriver."""
driver = MockDeviceDriver(width=1000, height=2000)
# Click normalized [500, 500] -> pixel (500, 1000)
result = await ToolRegistry.execute("click", {"target": [500, 500]}, driver=driver, ctx=None)
assert "Click" in result or "Clicked" in result
assert driver.action_history[-1]["action"] == "tap"
assert driver.action_history[-1]["x"] == 500
assert driver.action_history[-1]["y"] == 1000
# Swipe cardinal "up"
result = await ToolRegistry.execute("swipe", {"action": "up"}, driver=driver, ctx=None)
assert "Swipe" in result or "Swiped" in result
assert driver.action_history[-1]["action"] == "swipe_direction"
def test_tool_to_genai_declaration():
"""Verify tools export to Google GenAI FunctionDeclaration properly."""
declarations = ToolRegistry.get_genai_declarations(["click", "swipe", "input_text"])
assert len(declarations) == 3
decl_names = [d.name for d in declarations]
assert "click" in decl_names
assert "swipe" in decl_names
assert "input_text" in decl_names
def test_tool_to_langchain_tool():
"""Verify tools export to LangChain BaseTool."""
lc_tools = ToolRegistry.get_langchain_tools(ctx=None, names=["click", "press_key"])
assert len(lc_tools) == 2
assert lc_tools[0].name == "click"
assert lc_tools[1].name == "press_key"
+279
View File
@@ -0,0 +1,279 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for UnifiedMobileController video recording and playback features."""
import time
from unittest.mock import AsyncMock, MagicMock, patch
from uuid import uuid4
import pytest
from artemis.context import ArtemisContext
from artemis.controllers.unified_controller import UnifiedMobileController
from artemis.drivers.mock.mock_driver import MockDeviceDriver
from artemis.utils.video import (
RecordingSession,
get_active_session,
remove_active_session,
set_active_session,
)
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_id = "emulator-5554"
ctx.device.mobile_platform = "android"
# DataEngine mock
ctx.data_engine = MagicMock()
ctx.data_engine.current_session_id = uuid4()
ctx.data_engine.session_start_time = time.time()
ctx.data_engine.storage = MagicMock()
# Mock driver
mock_driver = MockDeviceDriver(device_id="emulator-5554")
ctx._active_driver = mock_driver
return ctx
@pytest.mark.asyncio
async def test_unified_controller_start_recording(mock_ctx, tmp_path):
controller = UnifiedMobileController(mock_ctx)
remove_active_session("emulator-5554")
# Mock scrcpy subprocess
mock_proc = MagicMock()
mock_proc.returncode = None
with patch("asyncio.create_subprocess_exec", AsyncMock(return_value=mock_proc)):
with patch("asyncio.sleep", AsyncMock()):
res = await controller.start_video_recording(output_dir=tmp_path)
assert res.success is True
assert get_active_session("emulator-5554") is not None
assert mock_ctx.data_engine.record_video_start.called
# Calling start again should report already in progress
res2 = await controller.start_video_recording(output_dir=tmp_path)
assert res2.success is False
assert "already in progress" in res2.message
remove_active_session("emulator-5554")
@pytest.mark.asyncio
async def test_unified_controller_stop_recording(mock_ctx, tmp_path):
controller = UnifiedMobileController(mock_ctx)
remove_active_session("emulator-5554")
# Create dummy recording.mkv
mkv_path = tmp_path / "recording.mkv"
mkv_path.write_bytes(b"dummy mkv content")
mock_proc = MagicMock()
mock_proc.returncode = None
mock_proc.terminate = MagicMock()
mock_proc.wait = AsyncMock()
session = RecordingSession(
video_id=uuid4(),
device_id="emulator-5554",
start_time=time.time() - 10.0,
data_engine_start_time=time.time() - 10.0,
local_video_path=mkv_path,
process=mock_proc,
)
set_active_session("emulator-5554", session)
with patch.object(
controller,
"_convert_mkv_to_mp4",
AsyncMock(side_effect=lambda src, dst: dst.write_bytes(b"mp4 content") or True),
):
res = await controller.stop_video_recording()
assert res.success is True
assert res.video_path is not None
assert str(res.video_path).endswith("recording.mp4")
assert get_active_session("emulator-5554") is None
assert mock_ctx.data_engine.record_video_stop.called
@pytest.mark.asyncio
async def test_unified_controller_extract_segment_metadata(mock_ctx, tmp_path):
controller = UnifiedMobileController(mock_ctx)
remove_active_session("emulator-5554")
mkv_path = tmp_path / "recording.mkv"
mkv_path.write_bytes(b"dummy mkv content")
mock_proc = MagicMock()
mock_proc.returncode = None
session = RecordingSession(
video_id=uuid4(),
device_id="emulator-5554",
start_time=time.time() - 20.0,
data_engine_start_time=time.time() - 20.0,
local_video_path=mkv_path,
process=mock_proc,
)
set_active_session("emulator-5554", session)
with patch(
"artemis.controllers.unified_controller.trim_video",
AsyncMock(side_effect=lambda src, s, e, dst: dst.write_bytes(b"segment mp4") or True),
):
res = await controller.extract_segment_metadata(start_time=2.0, end_time=8.0)
assert res.success is True
assert res.video_path is not None
assert res.video_path.exists()
# Ensure session is still active (not stopped by extraction!)
assert get_active_session("emulator-5554") is not None
remove_active_session("emulator-5554")
def test_data_engine_video_lifecycle(tmp_path):
from artemis.data_engine.engine import DataEngine
from artemis.context import ArtemisContext
mock_c = MagicMock(spec=ArtemisContext)
mock_c.execution_setup = MagicMock(traces_path=str(tmp_path / "traces"))
mock_c.device = None
engine = DataEngine(mock_c)
session_id = engine.start_session(goal="Test Video Engine")
vid = uuid4()
vpath = tmp_path / "recording.mp4"
vpath.write_bytes(b"video bytes")
# Start video recording
engine.record_video_start(vid, "device-1", vpath)
rec = engine.storage.get_video_recording(vid)
assert rec is not None
assert rec.device_id == "device-1"
# Stop video recording
engine.record_video_stop(vid, "device-1", vpath, time.time() - 10.0, time.time())
sess = engine.storage.get_session(session_id)
assert sess.video_filepath == str(vpath)
# Move video path on finalize
new_vpath = tmp_path / "archived" / "recording.mp4"
new_vpath.parent.mkdir(parents=True, exist_ok=True)
new_vpath.write_bytes(b"video bytes")
engine.update_video_path(new_vpath)
sess_updated = engine.storage.get_session(session_id)
assert sess_updated.video_filepath == str(new_vpath)
@pytest.mark.asyncio
async def test_unified_controller_crash_recovery_and_multi_segment(mock_ctx, tmp_path):
"""Verify that if recording is interrupted, segments are auto-recovered and concatenated seamlessly."""
controller = UnifiedMobileController(mock_ctx)
remove_active_session("emulator-5554")
seg1 = tmp_path / "recording_0.mkv"
seg1.write_bytes(b"segment 0")
seg2 = tmp_path / "recording_1.mkv"
seg2.write_bytes(b"segment 1")
mock_proc = MagicMock()
mock_proc.returncode = 1 # Simulated crash
session = RecordingSession(
video_id=uuid4(),
device_id="emulator-5554",
start_time=time.time() - 30.0,
data_engine_start_time=time.time() - 30.0,
local_video_path=seg2,
android_video_segments=[seg1],
android_segment_index=1,
process=mock_proc,
)
set_active_session("emulator-5554", session)
with (
patch(
"artemis.controllers.unified_controller.concatenate_videos",
AsyncMock(side_effect=lambda segs, out: out.write_bytes(b"combined mkv") or True),
),
patch.object(
controller,
"_convert_mkv_to_mp4",
AsyncMock(side_effect=lambda src, dst: dst.write_bytes(b"final mp4") or True),
),
):
res = await controller.stop_video_recording()
assert res.success is True
assert res.video_path is not None
assert res.video_path.exists()
assert str(res.video_path).endswith("recording.mp4")
assert get_active_session("emulator-5554") is None
@pytest.mark.asyncio
async def test_unified_controller_timeline_alignment(mock_ctx, tmp_path):
"""Verify that relative time offset between DataEngine T0 and video start is precisely calculated."""
controller = UnifiedMobileController(mock_ctx)
remove_active_session("emulator-5554")
mkv_path = tmp_path / "recording.mkv"
mkv_path.write_bytes(b"dummy mkv content")
now = time.time()
# Assume session started 15s ago, and video started 12s ago (offset = 3.0s)
session_start_t0 = now - 15.0
video_start_t = now - 12.0
session = RecordingSession(
video_id=uuid4(),
device_id="emulator-5554",
start_time=video_start_t,
data_engine_start_time=session_start_t0,
local_video_path=mkv_path,
process=MagicMock(returncode=None),
)
set_active_session("emulator-5554", session)
trimmed_ranges = []
async def mock_trim(src, s_time, e_time, dst):
trimmed_ranges.append((s_time, e_time))
dst.write_bytes(b"trimmed")
return True
with patch(
"artemis.controllers.unified_controller.trim_video", AsyncMock(side_effect=mock_trim)
):
# Agent asks for system time range [5.0s, 10.0s]
res = await controller.extract_segment_metadata(start_time=5.0, end_time=10.0)
assert res.success is True
# Since offset = 3.0s, video_start_relative_time should be 5.0 - 3.0 = 2.0s
# and video_end_relative_time should be 10.0 - 3.0 = 7.0s
assert len(trimmed_ranges) == 1
s_time, e_time = trimmed_ranges[0]
assert abs(s_time - 2.0) < 0.1
assert abs(e_time - 7.0) < 0.1
# actual_start_relative_time reported back to agent must align with system T0 (5.0s)
assert abs(res.actual_start_relative_time - 5.0) < 0.1
remove_active_session("emulator-5554")
+36
View File
@@ -0,0 +1,36 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from artemis.utils.notes import update_note_content
base_dir = "/tmp/artemis_test"
os.makedirs(f"{base_dir}/notes", exist_ok=True)
task_plan = """- [x] Initial task
- [/] Active task
- [ ] Future task"""
with open(f"{base_dir}/notes/task_plan.md", "w") as f:
f.write(task_plan)
try:
res = update_note_content(
base_dir,
"task_plan",
"- [/] Active task",
"- [/] Active task\n - [ ] New subgoal",
)
print("Update successful!")
print(open(f"{base_dir}/notes/task_plan.md").read())
except Exception as e:
print("Error:", e)
@@ -0,0 +1,132 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest.mock import MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
AppendNoteArgs,
AppendNotePure,
AppendNotePureTool,
append_note_pure,
get_append_note_tool_pure,
)
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_append_note_pure_tool_subclass_and_registry():
"""Verify AppendNotePureTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(AppendNotePureTool, ArtemisTool)
assert issubclass(AppendNotePure, ArtemisTool)
assert isinstance(append_note_pure, ArtemisTool)
assert isinstance(append_note_pure, AppendNotePureTool)
assert append_note_pure.name == "append_note_pure"
assert append_note_pure.category == "memory"
assert append_note_pure.args_schema == AppendNoteArgs
# Registry lookup
reg_tool = ToolRegistry.get("append_note_pure")
assert reg_tool is not None
assert isinstance(reg_tool, AppendNotePureTool)
# GenAI FunctionDeclaration export
declaration = append_note_pure.to_genai_declaration()
assert declaration.name == "append_note_pure"
assert "key" in declaration.parameters.properties
assert "content" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_append_note_pure_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of AppendNotePureTool.execute appends to note file."""
# First append creates note
result1 = await append_note_pure.execute(
ctx=mock_ctx,
key="pure_append",
content="First line",
)
assert result1 == "Successfully appended note to pure_append.md."
note_path = Path(tmp_path) / "notes" / "pure_append.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "First line\n"
# Second append appends line
result2 = await append_note_pure.execute(
ctx=mock_ctx,
key="pure_append",
content="Second line",
)
assert result2 == "Successfully appended note to pure_append.md."
assert note_path.read_text(encoding="utf-8") == "First line\nSecond line\n"
@pytest.mark.asyncio
async def test_append_note_pure_callable_execution(mock_ctx, tmp_path):
"""Verify invoking append_note_pure directly as a callable."""
result = await append_note_pure(
ctx=mock_ctx,
key="callable_pure_append",
content="Pure callable append",
)
assert result == "Successfully appended note to callable_pure_append.md."
note_path = Path(tmp_path) / "notes" / "callable_pure_append.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Pure callable append\n"
@pytest.mark.asyncio
async def test_append_note_pure_execution_failure(mock_ctx):
"""Verify error handling when append_note_content fails."""
with patch(
"artemis.tools.scratchpad.append_note_content",
side_effect=PermissionError("Permission denied"),
):
result = await append_note_pure.execute(
ctx=mock_ctx,
key="fail_append",
content="Some content",
)
assert "Failed to append note fail_append.md: Permission denied" in result
@pytest.mark.asyncio
async def test_get_append_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_append_note_tool_pure exports a LangChain tool named 'append_note'."""
pure_tool = get_append_note_tool_pure(mock_ctx)
assert pure_tool.name == "append_note"
result = await pure_tool.ainvoke(
{
"key": "lc_pure_append",
"content": "Saved via pure append tool",
}
)
assert result == "Successfully appended note to lc_pure_append.md."
note_path = Path(tmp_path) / "notes" / "lc_pure_append.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Saved via pure append tool\n"
+190
View File
@@ -0,0 +1,190 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
AppendNote,
AppendNoteArgs,
AppendNoteTool,
get_append_note_tool,
get_append_note_tool_pure,
append_note,
append_note_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_append_note_tool_subclass_and_registry():
"""Verify AppendNoteTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(AppendNoteTool, ArtemisTool)
assert issubclass(AppendNote, ArtemisTool)
assert isinstance(append_note, ArtemisTool)
assert isinstance(append_note, AppendNoteTool)
assert append_note.name == "append_note"
assert append_note.category == "memory"
assert append_note.args_schema == AppendNoteArgs
# Registry lookup
reg_tool = ToolRegistry.get("append_note")
assert reg_tool is not None
assert isinstance(reg_tool, AppendNoteTool)
# GenAI FunctionDeclaration export
declaration = append_note.to_genai_declaration()
assert declaration.name == "append_note"
assert "key" in declaration.parameters.properties
assert "content" in declaration.parameters.properties
# Wrapper check
assert append_note_wrapper is not None
assert append_note_wrapper.tool_fn_getter == get_append_note_tool
@pytest.mark.asyncio
async def test_append_note_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of AppendNoteTool.execute appends to note file."""
# First append creates the note
result1 = await append_note.execute(
ctx=mock_ctx,
key="test_append",
content="First line",
)
assert result1 == "Successfully appended note to test_append.md."
note_path = Path(tmp_path) / "notes" / "test_append.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "First line\n"
# Second append appends with newline
result2 = await append_note.execute(
ctx=mock_ctx,
key="test_append",
content="Second line",
)
assert result2 == "Successfully appended note to test_append.md."
assert note_path.read_text(encoding="utf-8") == "First line\nSecond line\n"
@pytest.mark.asyncio
async def test_append_note_callable_execution(mock_ctx, tmp_path):
"""Verify invoking append_note directly as a callable."""
result = await append_note(
ctx=mock_ctx,
key="callable_append",
content="Callable append test",
)
assert result == "Successfully appended note to callable_append.md."
note_path = Path(tmp_path) / "notes" / "callable_append.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Callable append test\n"
@pytest.mark.asyncio
async def test_append_note_with_state_command(mock_ctx, tmp_path):
"""Verify AppendNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
mock_state = MagicMock(spec=State)
mock_state.current_agent = "operator"
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
result = await append_note.execute(
ctx=mock_ctx,
key="state_append_note",
content="Appended with state",
state=mock_state,
tool_call_id="call_append_999",
)
assert isinstance(result, Command)
assert VALIDATOR_MESSAGES_KEY in result.update
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
assert len(tool_messages) == 1
assert tool_messages[0].tool_call_id == "call_append_999"
assert tool_messages[0].content == "Successfully appended note to state_append_note.md."
assert tool_messages[0].status == "success"
mock_state.asanitize_update.assert_called_once()
note_path = Path(tmp_path) / "notes" / "state_append_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Appended with state\n"
@pytest.mark.asyncio
async def test_append_note_execution_failure(mock_ctx):
"""Verify error handling when appending note fails."""
with patch(
"artemis.tools.scratchpad.append_note_content",
side_effect=PermissionError("Permission denied"),
):
result = await append_note.execute(
ctx=mock_ctx,
key="fail_append_note",
content="Fail content",
)
assert "Failed to append note fail_append_note.md: Permission denied" in result
@pytest.mark.asyncio
async def test_get_append_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_append_note_tool exports a LangChain tool that works with ainvoke."""
lc_tool = get_append_note_tool(mock_ctx)
assert lc_tool.name == "append_note"
result = await lc_tool.ainvoke(
{
"key": "lc_append_note",
"content": "Appended via LangChain BaseTool",
}
)
assert result == "Successfully appended note to lc_append_note.md."
note_path = Path(tmp_path) / "notes" / "lc_append_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Appended via LangChain BaseTool\n"
@pytest.mark.asyncio
async def test_get_append_note_tool_pure(mock_ctx, tmp_path):
"""Verify get_append_note_tool_pure returns a pure LangChain tool."""
pure_tool = get_append_note_tool_pure(mock_ctx)
assert pure_tool.name == "append_note"
result = await pure_tool.ainvoke(
{
"key": "pure_append_note",
"content": "Appended via pure tool",
}
)
assert result == "Successfully appended note to pure_append_note.md."
note_path = Path(tmp_path) / "notes" / "pure_append_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Appended via pure tool\n"
+158
View File
@@ -0,0 +1,158 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.exec_tools import (
Click,
ClickArgs,
ClickTool,
click,
click_wrapper,
get_click_tool,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
driver.screen_size = (1080, 2400)
driver.tap = AsyncMock(return_value=True)
return driver
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def test_click_tool_subclass_and_registry():
"""Verify ClickTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(ClickTool, ArtemisTool)
assert issubclass(Click, ArtemisTool)
assert isinstance(click, ArtemisTool)
assert isinstance(click, ClickTool)
assert click.name == "click"
assert click.category == "action"
assert click.args_schema == ClickArgs
# Registry lookup
reg_tool = ToolRegistry.get("click")
assert reg_tool is not None
assert isinstance(reg_tool, ClickTool)
# GenAI FunctionDeclaration export
declaration = click.to_genai_declaration()
assert declaration.name == "click"
assert "target" in declaration.parameters.properties
# Wrapper check
assert click_wrapper is not None
assert click_wrapper.tool_fn_getter == get_click_tool
@pytest.mark.asyncio
async def test_click_direct_execution_with_driver(mock_driver):
"""Verify direct execution of ClickTool with BaseDeviceDriver."""
result = await click.execute(driver=mock_driver, target=[500, 500])
mock_driver.tap.assert_called_once_with(540, 1200)
assert "Clicked at [500, 500] (normalized) successfully." in result
@pytest.mark.asyncio
async def test_click_direct_execution_with_ctx(mock_ctx):
"""Verify direct execution of ClickTool with ArtemisContext."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_cls.return_value = mock_controller_inst
result = await click.execute(ctx=mock_ctx, target=[250, 750])
mock_controller_inst.tap_at.assert_called_once_with(270, 1800)
assert "Clicked at [250, 750] (normalized) successfully." in result
@pytest.mark.asyncio
async def test_click_with_state_command(mock_ctx, mock_state):
"""Verify ClickTool returns Command when state is provided."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_cls.return_value = mock_controller_inst
cmd = await click.execute(
ctx=mock_ctx,
target=[100, 200],
tool_call_id="call_click_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_click_1"
assert messages[0].status == "success"
assert "Clicked at [100, 200]" in messages[0].content
@pytest.mark.asyncio
async def test_click_callable_execution(mock_driver):
"""Verify invoking click directly as a callable."""
result = await click(driver=mock_driver, target=[100, 100])
assert "Clicked at [100, 100] (normalized) successfully." in result
@pytest.mark.asyncio
async def test_click_invalid_target(mock_ctx):
"""Verify error handling on invalid target coordinates."""
result = await click.execute(ctx=mock_ctx, target=[])
assert "Error during click: Invalid target coordinates" in result
@pytest.mark.asyncio
async def test_get_click_tool_langchain_ainvoke(mock_ctx):
"""Verify get_click_tool exports a LangChain tool that works with ainvoke."""
click_tool = get_click_tool(mock_ctx)
assert click_tool.name == "click"
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_cls.return_value = mock_controller_inst
result = await click_tool.ainvoke({"target": [500, 500]})
assert "Clicked at [500, 500] (normalized) successfully." in result
+603
View File
@@ -0,0 +1,603 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.command_tool import (
_BACKGROUND_TASKS,
_PERSISTENT_ENVIRONMENTS,
get_manage_task_tool,
get_run_adb_command_tool,
get_run_short_adb_command_tool,
)
import pytest
class MockProcess:
def __init__(
self,
output_bytes: bytes,
exit_code: int = 0,
pid: int = 12345,
delay: float = 0,
):
self.output_bytes = output_bytes
self.exit_code = exit_code
self.pid = pid
self.returncode = None
self._read_offset = 0
self.delay = delay
self.stdin = MagicMock()
self.stdin.write = MagicMock()
self.stdin.drain = AsyncMock()
self.stdin.close = MagicMock()
self.stdout = MagicMock()
self.stdout.readline = AsyncMock(side_effect=self._readline)
async def _readline(self) -> bytes:
if self.delay > 0:
await asyncio.sleep(self.delay)
self.delay = 0 # Only delay the first read
if self._read_offset >= len(self.output_bytes):
self.returncode = self.exit_code
return b""
idx = self.output_bytes.find(b"\n", self._read_offset)
if idx == -1:
line = self.output_bytes[self._read_offset :]
self._read_offset = len(self.output_bytes)
else:
line = self.output_bytes[self._read_offset : idx + 1]
self._read_offset = idx + 1
return line
async def wait(self) -> int:
self.returncode = self.exit_code
return self.exit_code
def terminate(self):
self.returncode = -15
self.exit_code = -15
async def communicate(self) -> tuple[bytes, bytes]:
lines = []
while True:
line = await self._readline()
if not line:
break
lines.append(line)
self.returncode = self.exit_code
return b"".join(lines), b""
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = None
ctx.device = MagicMock()
ctx.device.device_id = "test_device_1234"
return ctx
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_run_command_sync(mock_exec, mock_ctx):
# Mock subprocess output
mock_process = MockProcess(
output_bytes=b"hello world\n===EXIT_CODE===0\n===ENV_START===\n",
exit_code=0,
)
mock_exec.return_value = mock_process
run_command = get_run_adb_command_tool(mock_ctx)
result = await run_command.ainvoke(
{
"CommandLine": "echo 'hello world'",
"Cwd": "/data/local/tmp",
"RunPersistent": False,
"WaitMsBeforeAsync": 500,
}
)
assert "completed with exit code 0" in result
assert "hello world" in result
mock_exec.assert_called_once()
# Check that adb was called with device ID and "shell"
args, kwargs = mock_exec.call_args
assert args[0] == "adb"
assert args[1] == "-s"
assert args[2] == "test_device_1234"
assert args[3] == "shell"
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_run_command_persistent_env(mock_exec, mock_ctx):
run_command = get_run_adb_command_tool(mock_ctx)
terminal_id = "test_term_android"
# 1. Set environment variable in persistent terminal
mock_process_1 = MockProcess(
output_bytes=(b"===EXIT_CODE===0\n===ENV_START===\nARTEMIS_TEST_ENV=coffee\n"),
exit_code=0,
)
mock_exec.return_value = mock_process_1
res1 = await run_command.ainvoke(
{
"CommandLine": "export ARTEMIS_TEST_ENV=coffee",
"Cwd": "/data/local/tmp",
"RunPersistent": True,
"RequestedTerminalID": terminal_id,
"WaitMsBeforeAsync": 500,
}
)
assert "completed with exit code 0" in res1
assert terminal_id in _PERSISTENT_ENVIRONMENTS
assert _PERSISTENT_ENVIRONMENTS[terminal_id].get("ARTEMIS_TEST_ENV") == "coffee"
# 2. Query environment variable in subsequent command on same terminal
mock_process_2 = MockProcess(
output_bytes=b"coffee\n===EXIT_CODE===0\n===ENV_START===\nARTEMIS_TEST_ENV=coffee\n",
exit_code=0,
)
mock_exec.return_value = mock_process_2
res2 = await run_command.ainvoke(
{
"CommandLine": "echo $ARTEMIS_TEST_ENV",
"Cwd": "/data/local/tmp",
"RunPersistent": True,
"RequestedTerminalID": terminal_id,
"WaitMsBeforeAsync": 500,
}
)
assert "completed with exit code 0" in res2
assert "coffee" in res2
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_run_command_async_and_status(mock_exec, mock_ctx):
# Mock a long running command by delaying stdout response
mock_process = MockProcess(
output_bytes=(b"background task finished\n===EXIT_CODE===0\n===ENV_START===\n"),
exit_code=0,
delay=1.0, # 1.0s delay will trigger timeout
)
mock_exec.return_value = mock_process
run_command = get_run_adb_command_tool(mock_ctx)
manage_task = get_manage_task_tool(mock_ctx)
_BACKGROUND_TASKS.clear()
res = await run_command.ainvoke(
{
"CommandLine": "sleep 2 && echo 'background task finished'",
"Cwd": "/data/local/tmp",
"RunPersistent": False,
"WaitMsBeforeAsync": 100, # 100ms wait threshold
}
)
assert "sent to the background as a task" in res
assert "TaskId: task_" in res
lines = res.splitlines()
task_id_line = [line for line in lines if line.startswith("TaskId: ")][0]
task_id = task_id_line.split(": ")[1]
assert task_id in _BACKGROUND_TASKS
# Check status (should be running)
status_res = await manage_task.ainvoke(
{
"Action": "status",
"TaskId": task_id,
}
)
assert "Status: running" in status_res
# Wait for the background listener task to finish reading delayed output
await asyncio.sleep(1.2)
# Status should be completed (and task removed from active map)
assert task_id not in _BACKGROUND_TASKS
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_run_command_kill(mock_exec, mock_ctx):
mock_process = MockProcess(output_bytes=b"", exit_code=0, delay=5.0)
mock_exec.return_value = mock_process
run_command = get_run_adb_command_tool(mock_ctx)
manage_task = get_manage_task_tool(mock_ctx)
_BACKGROUND_TASKS.clear()
res = await run_command.ainvoke(
{
"CommandLine": "sleep 10",
"Cwd": "/data/local/tmp",
"RunPersistent": False,
"WaitMsBeforeAsync": 100,
}
)
lines = res.splitlines()
task_id_line = [line for line in lines if line.startswith("TaskId: ")][0]
task_id = task_id_line.split(": ")[1]
# Kill the task
kill_res = await manage_task.ainvoke(
{
"Action": "kill",
"TaskId": task_id,
}
)
assert f"Task {task_id} successfully terminated" in kill_res
assert task_id not in _BACKGROUND_TASKS
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_run_short_command_success(mock_exec, mock_ctx):
mock_process = MockProcess(output_bytes=b"short command success output", exit_code=0)
mock_exec.return_value = mock_process
run_short_command = get_run_short_adb_command_tool(mock_ctx)
result = await run_short_command.ainvoke(
{
"CommandLine": "echo 'short command'",
}
)
assert "completed with exit code 0" in result
assert "short command success output" in result
mock_exec.assert_called_once()
args, kwargs = mock_exec.call_args
assert args[0] == "adb"
assert args[1] == "-s"
assert args[2] == "test_device_1234"
assert args[3] == "shell"
# Verify it cd'd to /data/local/tmp by default
assert "cd /data/local/tmp" in args[4]
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_run_short_command_timeout(mock_exec, mock_ctx):
mock_process = MockProcess(output_bytes=b"some output", exit_code=0, delay=5.0)
mock_exec.return_value = mock_process
run_short_command = get_run_short_adb_command_tool(mock_ctx)
result = await run_short_command.ainvoke(
{
"CommandLine": "sleep 5",
}
)
assert "Error: Command timed out after 3 seconds." in result
@pytest.mark.asyncio
@patch("asyncio.create_subprocess_exec")
async def test_run_command_sync_long_output(mock_exec, mock_ctx):
# Generate long output (> 500 lines)
long_output = "\n".join(f"Output line {i}" for i in range(1, 600))
mock_process = MockProcess(
output_bytes=long_output.encode() + b"\n===EXIT_CODE===0\n===ENV_START===\n",
exit_code=0,
)
mock_exec.return_value = mock_process
run_command = get_run_adb_command_tool(mock_ctx)
result = await run_command.ainvoke(
{
"CommandLine": "generate_long_output",
"Cwd": "/data/local/tmp",
"RunPersistent": False,
"WaitMsBeforeAsync": 500,
}
)
assert "ADB command completed with exit code 0." in result
assert "has been truncated" in result
assert "Output line 599" in result
assert "Output line 1" not in result # since it was truncated to last 200 lines
assert "analyze_task_output" in result
assert "TaskId: task_sync_" in result
@pytest.mark.asyncio
@patch("artemis.agents.log_analyzer.output_analyzer.get_llm")
async def test_analyze_task_output_tool(mock_get_llm, mock_ctx):
# Set up mock LLM
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.content = "This is the analysis result: everything looks good."
mock_response.tool_calls = []
mock_llm.bind_tools.return_value = mock_llm
mock_llm.ainvoke = AsyncMock(return_value=mock_response)
mock_get_llm.return_value = mock_llm
# Set up finished tasks logs cache manually
from artemis.tools.command_tool import (
_FINISHED_TASKS_LOGS,
_register_finished_task,
get_analyze_task_output_tool,
)
task_id = "test_task_long_output"
_register_finished_task(
task_id=task_id,
command="cat long_log.txt",
cwd="/data/local/tmp",
terminal_id=None,
status="completed",
exit_code=0,
output="line1\nline2\n" + "\n".join(f"line{i}" for i in range(3, 50)),
)
analyze_tool = get_analyze_task_output_tool(mock_ctx)
result = await analyze_tool.ainvoke({"TaskId": task_id, "Query": "Summarize the log"})
assert "Analysis result for Task test_task_long_output:" in result
assert "This is the analysis result: everything looks good." in result
mock_get_llm.assert_called_once_with(ctx=mock_ctx, name="output_analyzer")
# Clean up registry
if task_id in _FINISHED_TASKS_LOGS:
del _FINISHED_TASKS_LOGS[task_id]
def test_run_adb_command_tool_subclass_and_registry():
"""Verify RunAdbCommandTool is a subclass of ArtemisTool and properly registered."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.command_tool import (
RunAdbCommand,
RunAdbCommandArgs,
RunAdbCommandTool,
run_adb_command,
)
assert issubclass(RunAdbCommandTool, ArtemisTool)
assert issubclass(RunAdbCommand, ArtemisTool)
assert isinstance(run_adb_command, ArtemisTool)
assert isinstance(run_adb_command, RunAdbCommandTool)
assert run_adb_command.name == "run_adb_command"
assert run_adb_command.category == "system"
assert run_adb_command.args_schema == RunAdbCommandArgs
# Registry lookup
reg_tool = ToolRegistry.get("run_adb_command")
assert reg_tool is not None
assert isinstance(reg_tool, RunAdbCommandTool)
# GenAI FunctionDeclaration export
declaration = run_adb_command.to_genai_declaration()
assert declaration.name == "run_adb_command"
assert "CommandLine" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_run_adb_command_tool_mock_driver_execute():
"""Verify RunAdbCommandTool.execute dispatches to MockDeviceDriver when ctx is None."""
from artemis.drivers.mock.mock_driver import MockDeviceDriver
from artemis.tools.command_tool import RunAdbCommandTool
driver = MockDeviceDriver(width=1080, height=2400)
tool_inst = RunAdbCommandTool()
result = await tool_inst.execute(
driver=driver,
ctx=None,
CommandLine="pm list packages",
)
assert "mock_output: pm list packages" in result
assert "pm list packages" in driver.action_history[-1]["command"]
def test_manage_task_tool_subclass_and_registry():
"""Verify ManageTaskTool is a subclass of ArtemisTool and properly registered."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.command_tool import (
ManageTask,
ManageTaskArgs,
ManageTaskTool,
manage_task,
)
assert issubclass(ManageTaskTool, ArtemisTool)
assert issubclass(ManageTask, ArtemisTool)
assert isinstance(manage_task, ArtemisTool)
assert isinstance(manage_task, ManageTaskTool)
assert manage_task.name == "manage_task"
assert manage_task.category == "system"
assert manage_task.args_schema == ManageTaskArgs
# Registry lookup
reg_tool = ToolRegistry.get("manage_task")
assert reg_tool is not None
assert isinstance(reg_tool, ManageTaskTool)
# GenAI FunctionDeclaration export
declaration = manage_task.to_genai_declaration()
assert declaration.name == "manage_task"
assert "Action" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_manage_task_tool_direct_execute(mock_ctx):
"""Verify direct ManageTaskTool.execute execution."""
from artemis.tools.command_tool import (
_BACKGROUND_TASKS,
BackgroundTask,
manage_task,
)
_BACKGROUND_TASKS.clear()
list_res = await manage_task.execute(ctx=mock_ctx, Action="list")
assert "No active background tasks" in list_res
# Create dummy task
dummy_proc = MagicMock()
bg_task = BackgroundTask(
task_id="dummy_task_99",
command="sleep 100",
process=dummy_proc,
terminal_id="term_1",
cwd="/data/local/tmp",
)
_BACKGROUND_TASKS["dummy_task_99"] = bg_task
list_res2 = await manage_task.execute(ctx=mock_ctx, Action="list")
assert "dummy_task_99" in list_res2
assert "sleep 100" in list_res2
# Status check
status_res = await manage_task.execute(ctx=mock_ctx, Action="status", TaskId="dummy_task_99")
assert "Status: running" in status_res
assert "dummy_task_99" in status_res
_BACKGROUND_TASKS.clear()
def test_analyze_task_output_tool_subclass_and_registry():
"""Verify AnalyzeTaskOutputTool is a subclass of ArtemisTool and properly registered."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.command_tool import (
AnalyzeTaskOutput,
AnalyzeTaskOutputArgs,
AnalyzeTaskOutputTool,
analyze_task_output,
)
assert issubclass(AnalyzeTaskOutputTool, ArtemisTool)
assert issubclass(AnalyzeTaskOutput, ArtemisTool)
assert isinstance(analyze_task_output, ArtemisTool)
assert isinstance(analyze_task_output, AnalyzeTaskOutputTool)
assert analyze_task_output.name == "analyze_task_output"
assert analyze_task_output.category == "system"
assert analyze_task_output.args_schema == AnalyzeTaskOutputArgs
# Registry lookup
reg_tool = ToolRegistry.get("analyze_task_output")
assert reg_tool is not None
assert isinstance(reg_tool, AnalyzeTaskOutputTool)
# GenAI FunctionDeclaration export
declaration = analyze_task_output.to_genai_declaration()
assert declaration.name == "analyze_task_output"
assert "TaskId" in declaration.parameters.properties
assert "Query" in declaration.parameters.properties
@pytest.mark.asyncio
@patch("artemis.agents.log_analyzer.output_analyzer.get_llm")
async def test_analyze_task_output_tool_direct_execute(mock_get_llm, mock_ctx):
"""Verify direct AnalyzeTaskOutputTool.execute execution."""
from artemis.tools.command_tool import (
_FINISHED_TASKS_LOGS,
_register_finished_task,
analyze_task_output,
)
mock_llm = MagicMock()
mock_response = MagicMock()
mock_response.content = "Summary of task output."
mock_response.tool_calls = []
mock_llm.bind_tools.return_value = mock_llm
mock_llm.ainvoke = AsyncMock(return_value=mock_response)
mock_get_llm.return_value = mock_llm
task_id = "test_direct_analyze"
_register_finished_task(
task_id=task_id,
command="cat file.txt",
cwd="/data/local/tmp",
terminal_id=None,
status="completed",
exit_code=0,
output="Sample log content",
)
result = await analyze_task_output.execute(
ctx=mock_ctx, TaskId=task_id, Query="What is the content?"
)
assert "Analysis result for Task test_direct_analyze:" in result
assert "Summary of task output." in result
if task_id in _FINISHED_TASKS_LOGS:
del _FINISHED_TASKS_LOGS[task_id]
def test_run_short_adb_command_tool_subclass_and_registry():
"""Verify RunShortAdbCommandTool is a subclass of ArtemisTool and properly registered."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.command_tool import (
RunShortAdbCommand,
RunShortAdbCommandArgs,
RunShortAdbCommandTool,
run_short_adb_command,
)
assert issubclass(RunShortAdbCommandTool, ArtemisTool)
assert issubclass(RunShortAdbCommand, ArtemisTool)
assert isinstance(run_short_adb_command, ArtemisTool)
assert isinstance(run_short_adb_command, RunShortAdbCommandTool)
assert run_short_adb_command.name == "run_short_adb_command"
assert run_short_adb_command.category == "system"
assert run_short_adb_command.args_schema == RunShortAdbCommandArgs
# Registry lookup
reg_tool = ToolRegistry.get("run_short_adb_command")
assert reg_tool is not None
assert isinstance(reg_tool, RunShortAdbCommandTool)
# GenAI FunctionDeclaration export
declaration = run_short_adb_command.to_genai_declaration()
assert declaration.name == "run_short_adb_command"
assert "CommandLine" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_run_short_adb_command_tool_mock_driver_execute():
"""Verify RunShortAdbCommandTool.execute dispatches to MockDeviceDriver when ctx is None."""
from artemis.drivers.mock.mock_driver import MockDeviceDriver
from artemis.tools.command_tool import RunShortAdbCommandTool
driver = MockDeviceDriver(width=1080, height=2400)
tool_inst = RunShortAdbCommandTool()
result = await tool_inst.execute(
driver=driver,
ctx=None,
CommandLine="getprop ro.build.version.release",
)
assert "mock_output: getprop ro.build.version.release" in result
assert "getprop ro.build.version.release" in driver.action_history[-1]["command"]
+222
View File
@@ -0,0 +1,222 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
import shutil
import tempfile
import unittest
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.committee_tool import get_ask_committee_tool
class TestCommitteeTool(unittest.IsolatedAsyncioTestCase):
async def test_ask_committee_tool_with_directive(self):
# Create a temp directory for testing
temp_dir = tempfile.mkdtemp()
try:
notes_dir = Path(temp_dir) / "notes"
notes_dir.mkdir(parents=True, exist_ok=True)
# Create a mock screenshot file
mock_screenshot_path = Path(temp_dir) / "mock_screenshot.jpg"
mock_screenshot_path.write_bytes(b"fake image bytes")
# Create a mock failed plans history file
mock_failed_plans_history_path = notes_dir / "failed_plans_history.md"
mock_failed_plans_history_path.write_text(
"## Failed Plan Attempt #1\n### Plan:\n- [ ] Step 1\n###"
" Failure Reason:\nTimeout waiting for element",
encoding="utf-8",
)
# Mock context and state
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.device = MagicMock()
mock_ctx.device.device_width = 1080
mock_ctx.device.device_height = 2400
mock_ctx.data_engine = MagicMock()
mock_ctx.data_engine.base_dir = temp_dir
mock_ctx.data_engine.get_agent_friendly_steps.return_value = [
{"pre_image_name": "test_image", "step_number": 1}
]
mock_state = MagicMock()
mock_state.initial_goal = "Test Goal"
mock_state.messages = []
mock_state.latest_ui_hierarchy = None
mock_state.latest_screenshot = str(mock_screenshot_path)
mock_state.focused_app_info = None
mock_state.device_date = None
mock_state.structured_decisions = None
mock_state.complete_subgoals_by_ids = []
mock_state.validator_messages = []
mock_state.operator_replan_reason = "Mock Immediate Replan Reason"
# Mock LLM responses for the members
mock_llm_pl = MagicMock()
mock_llm_pl.bind_tools.return_value = mock_llm_pl
m1 = MagicMock(content="Op: I think we should tap.")
m1.tool_calls = []
m2 = MagicMock(content="Op: I agree with Exp.")
m2.tool_calls = []
m3 = MagicMock(content="Op: Final conclusion: Tap at [50, 50]")
m3.tool_calls = []
mock_llm_pl.ainvoke = AsyncMock(side_effect=[m1, m2, m3])
mock_llm_diag = MagicMock()
mock_llm_diag.bind_tools.return_value = mock_llm_diag
m_diag = MagicMock(content="Diag: I see an error.")
m_diag.tool_calls = []
mock_llm_diag.ainvoke = AsyncMock(return_value=m_diag)
# Mock History Analyzer to call a tool first
mock_llm_hist = MagicMock()
mock_llm_hist.bind_tools.return_value = mock_llm_hist
mock_response_tool_call = MagicMock()
mock_response_tool_call.tool_calls = [
{
"name": "get_step_details",
"args": {"start_step": 1, "end_step": 1},
"id": "call_1",
}
]
mock_response_tool_call.content = ""
mock_response_final = MagicMock()
mock_response_final.tool_calls = []
mock_response_final.content = "Hist: I found step 1 details."
m_hist_agree = MagicMock(content="Hist: Still agree.")
m_hist_agree.tool_calls = []
mock_llm_hist.ainvoke = AsyncMock(
side_effect=[
mock_response_tool_call,
mock_response_final,
# For second round
m_hist_agree,
]
)
def mock_get_llm(ctx, name, temperature=None):
if name == "planner_avatar":
return mock_llm_pl
elif name == "diagnoser_expert":
return mock_llm_diag
elif name == "history_analyzer_expert":
return mock_llm_hist
return MagicMock()
# Patch get_llm and the actual tool functions to avoid real calls
with (
patch(
"artemis.tools.committee_tool.get_llm",
side_effect=mock_get_llm,
),
patch(
"artemis.tools.committee_tool.trace_langchain_tool",
side_effect=lambda t, ctx: t,
),
):
tool = get_ask_committee_tool(mock_ctx)
# Invoke tool with avatar_directive
from artemis.data_engine.trace import CURRENT_TRACE_ID
token = CURRENT_TRACE_ID.set("test_trace_id")
try:
result = await tool.ainvoke(
{
"avatar_directive": "Advocate for paying",
"state": mock_state,
}
)
finally:
CURRENT_TRACE_ID.reset(token)
# Verify result
self.assertIn("Final conclusion: Tap at [50, 50]", result)
# Verify file was created and contains content
notes_dir = Path(temp_dir) / "notes"
blackboard_files = list(notes_dir.glob("blackboard_*.md"))
self.assertTrue(len(blackboard_files) > 0, "No blackboard file created")
blackboard_path = blackboard_files[0]
content = blackboard_path.read_text(encoding="utf-8")
self.assertIn("Advocate for paying", content) # Check directive is in blackboard
self.assertIn("Hist: I found step 1 details.", content)
self.assertIn(
"Timeout waiting for element", content
) # Check failed history is in blackboard
self.assertIn("Test Goal", content) # Check initial goal is in blackboard
self.assertIn(
"Mock Immediate Replan Reason", content
) # Check current replan reason is in blackboard
finally:
# Clean up temp directory
shutil.rmtree(temp_dir)
def test_ask_committee_tool_subclass_and_registry(self):
"""Verify AskCommitteeTool is a subclass of ArtemisTool and properly registered."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.committee_tool import (
AskCommittee,
AskCommitteeArgs,
AskCommitteeTool,
AskCommitteeToolAlias,
ask_committee,
ask_committee_wrapper,
get_ask_committee_tool,
)
self.assertTrue(issubclass(AskCommitteeTool, ArtemisTool))
self.assertTrue(issubclass(AskCommittee, ArtemisTool))
self.assertTrue(issubclass(AskCommitteeToolAlias, ArtemisTool))
self.assertIsInstance(ask_committee, ArtemisTool)
self.assertIsInstance(ask_committee, AskCommitteeTool)
self.assertEqual(ask_committee.name, "ask_committee")
self.assertEqual(ask_committee.category, "custom")
self.assertEqual(ask_committee.args_schema, AskCommitteeArgs)
# Registry lookup
reg_tool = ToolRegistry.get("ask_committee")
self.assertIsNotNone(reg_tool)
self.assertIsInstance(reg_tool, AskCommitteeTool)
# GenAI FunctionDeclaration export
declaration = ask_committee.to_genai_declaration()
self.assertEqual(declaration.name, "ask_committee")
self.assertIn("avatar_directive", declaration.parameters.properties)
# Wrapper check
self.assertIsNotNone(ask_committee_wrapper)
self.assertEqual(ask_committee_wrapper.tool_fn_getter, get_ask_committee_tool)
async def test_ask_committee_no_ctx(self):
"""Verify executing without ctx returns an error message."""
from artemis.tools.committee_tool import ask_committee
result = await ask_committee.execute(avatar_directive="Test")
self.assertEqual(result, "Error: ArtemisContext is required for ask_committee.")
if __name__ == "__main__":
unittest.main()
+375
View File
@@ -0,0 +1,375 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from unittest.mock import AsyncMock, MagicMock, mock_open, patch
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.explorer_tool import (
AskExplorer,
AskExplorerArgs,
AskExplorerTool,
AskExplorerToolAlias,
ask_explorer,
ask_explorer_wrapper,
get_ask_explorer_tool,
)
import pytest
def test_ask_explorer_tool_subclass_and_registry():
"""Verify AskExplorerTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(AskExplorerTool, ArtemisTool)
assert issubclass(AskExplorer, ArtemisTool)
assert issubclass(AskExplorerToolAlias, ArtemisTool)
assert isinstance(ask_explorer, ArtemisTool)
assert isinstance(ask_explorer, AskExplorerTool)
assert ask_explorer.name == "ask_explorer"
assert ask_explorer.category == "explorer"
assert ask_explorer.args_schema == AskExplorerArgs
# Registry lookup
reg_tool = ToolRegistry.get("ask_explorer")
assert reg_tool is not None
assert isinstance(reg_tool, AskExplorerTool)
# GenAI FunctionDeclaration export
declaration = ask_explorer.to_genai_declaration()
assert declaration.name == "ask_explorer"
assert "query" in declaration.parameters.properties
# Wrapper check
assert ask_explorer_wrapper is not None
assert ask_explorer_wrapper.tool_fn_getter == get_ask_explorer_tool
@pytest.mark.asyncio
async def test_ask_explorer_multiple_candidates():
# Mock Context and State
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = MagicMock()
mock_ctx.device.device_width = 1080
mock_ctx.device.device_height = 2400
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = [[100, 200]] # Pre-existing point
mock_state.indexed_elements = [
{
"index": 1,
"center": [100, 200],
"text": "Pre-existing",
"bounds": None,
"class": "Test",
"resource_id": None,
"is_ocr": False,
}
]
# Mock Explorer Agent Outcome with multiple candidates and fallback message
explorer_outcome = {
"candidates": [
{
"label": "S1",
"coords": [500, 500],
"description": "First Button",
},
{
"label": "S2",
"coords": [250, 750],
"description": "Second Button",
},
],
"fallback_message": "Please choose carefully.",
}
explorer_output_str = json.dumps(explorer_outcome)
# We need to patch the Explorer class inside the tool
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
with patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
):
# Retrieve the tool
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
result = await ask_explorer_tool.ainvoke(
{
"query": "Find buttons",
"context_feedback": "Attempt 1",
"state": mock_state,
}
)
assert mock_state.indexed_points == [
[100, 200],
[540, 1200],
[270, 1800],
]
assert len(mock_state.indexed_elements) == 3
assert mock_state.indexed_elements[1]["index"] == 2
assert mock_state.indexed_elements[1]["center"] == [540, 1200]
assert mock_state.indexed_elements[1]["text"] == "First Button"
assert mock_state.indexed_elements[2]["index"] == 3
assert mock_state.indexed_elements[2]["center"] == [270, 1800]
assert mock_state.indexed_elements[2]["text"] == "Second Button"
assert isinstance(result, str)
assert "Explorer successfully located the following candidate(s):" in result
assert "- [2] 'First Button' at coordinate [500, 500]" in result
assert "- [3] 'Second Button' at coordinate [250, 750]" in result
assert (
"You can click/act on them directly by calling perform_action with"
" their respective index." in result
)
assert "Additional Notes from Explorer: Please choose carefully." in result
@pytest.mark.asyncio
async def test_ask_explorer_only_fallback_message():
# Mock Context and State
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = []
mock_state.indexed_elements = []
explorer_outcome = {
"candidates": [],
"fallback_message": ("Element not found because it is covered by a keyboard."),
}
explorer_output_str = json.dumps(explorer_outcome)
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
with patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
):
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
result = await ask_explorer_tool.ainvoke(
{
"query": "Find hidden button",
"context_feedback": "",
"state": mock_state,
}
)
assert mock_state.indexed_points == []
assert mock_state.indexed_elements == []
assert (
"Explorer could not locate the element. Message: Element not found"
" because it is covered by a keyboard." in result
)
@pytest.mark.asyncio
async def test_ask_explorer_multimodal_success():
# Mock Context and State
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = MagicMock()
mock_ctx.device.device_width = 1080
mock_ctx.device.device_height = 2400
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = [[100, 200]]
mock_state.indexed_elements = [
{
"index": 1,
"center": [100, 200],
"text": "Pre-existing",
"bounds": None,
"class": "Test",
"resource_id": None,
"is_ocr": False,
}
]
explorer_outcome = {
"candidates": [
{
"label": "S1",
"coords": [500, 500],
"description": "First Button",
},
],
"fallback_message": "",
}
explorer_output_str = json.dumps(explorer_outcome)
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(return_value=explorer_output_str)
m_open = mock_open(read_data=b"fake_annotated_image_bytes")
with (
patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
),
patch("artemis.tools.explorer_tool.draw_dots") as mock_draw_dots,
patch("builtins.open", m_open),
patch("pathlib.Path.mkdir"),
):
ask_explorer_tool = get_ask_explorer_tool(mock_ctx, version="pro")
result = await ask_explorer_tool.ainvoke(
{
"query": "Find buttons",
"context_feedback": "",
"state": mock_state,
}
)
assert isinstance(result, list)
assert len(result) == 2
assert result[0]["type"] == "text"
assert "Explorer successfully located" in result[0]["text"]
assert result[1]["type"] == "image_url"
assert result[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
mock_draw_dots.assert_called_once()
points_arg = mock_draw_dots.call_args[0][1]
assert points_arg == [[100, 200], [540, 1200]]
labels_arg = mock_draw_dots.call_args[0][2]
assert labels_arg == ["1", "2"]
@pytest.mark.asyncio
async def test_ask_explorer_direct_execution():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = None
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = []
mock_state.indexed_elements = []
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(
return_value=json.dumps(
{
"candidates": [],
"fallback_message": "Direct execution not found.",
}
)
)
with patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
):
result = await ask_explorer.execute(
ctx=mock_ctx, state=mock_state, query="Find icon", version="pro"
)
assert "Direct execution not found." in result
@pytest.mark.asyncio
async def test_ask_explorer_callable_execution():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = None
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = []
mock_state.indexed_elements = []
mock_explorer_instance = MagicMock()
mock_explorer_instance.run = AsyncMock(
return_value=json.dumps(
{
"candidates": [],
"fallback_message": "Callable test message.",
}
)
)
with patch(
"artemis.tools.explorer_tool.Explorer",
return_value=mock_explorer_instance,
):
result = await ask_explorer(
ctx=mock_ctx, state=mock_state, query="Search bar", version="pro"
)
assert "Callable test message." in result
@pytest.mark.asyncio
async def test_ask_explorer_flash_mode():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_ctx.agent_config = None
mock_ctx.llm_config = None
mock_ctx.execution_setup = None
mock_ctx.device = MagicMock()
mock_ctx.device.device_width = 1080
mock_ctx.device.device_height = 2400
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test.jpg"
mock_state.indexed_points = []
mock_state.indexed_elements = []
with (
patch(
"artemis.tools.explorer_tool._run_object_detection",
new_callable=AsyncMock,
return_value={
"detected": [
{"label": "Icon", "point": [500, 500]},
],
"failed": [],
},
),
patch(
"builtins.open",
mock_open(read_data=json.dumps({"templates": [], "instructions": ""})),
),
):
result = await ask_explorer.execute(
ctx=mock_ctx, state=mock_state, query="Icon", version="flash"
)
assert "Explorer successfully located" in result
assert "Icon" in result
@@ -0,0 +1,99 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.image_processor_tool import get_ask_image_processor_tool
import pytest
@pytest.mark.asyncio
async def test_ask_vision_coder_success():
# Mock Context and State
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/tmp/test_screen.jpg"
# Mock VisionCoder Agent Outcome
vision_coder_outcome = {
"new_image_path": "/tmp/coder_output.jpg",
"summary": "Found red button at pixel coords (100, 200)",
}
# We need to patch the ImageProcessor class inside the tool
mock_coder_instance = MagicMock()
mock_coder_instance.run = AsyncMock(return_value=vision_coder_outcome)
with patch(
"artemis.agents.image_processor.image_processor.ImageProcessor",
return_value=mock_coder_instance,
):
# Retrieve the tool
ask_vision_coder_tool = get_ask_image_processor_tool(mock_ctx)
# Invoke the tool
result = await ask_vision_coder_tool.ainvoke(
{
"instruction": "Find the red button",
"target_image_id": "img_0",
"state": mock_state,
}
)
# Verify the agent was called with correct arguments
mock_coder_instance.run.assert_called_once_with(
"Find the red button", "/tmp/test_screen.jpg"
)
# Verify the output matches
assert result == vision_coder_outcome
def test_ask_image_processor_tool_subclass_and_registry():
"""Verify AskImageProcessorTool is an ArtemisTool subclass and registered in ToolRegistry."""
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.image_processor_tool import (
AskImageProcessor,
AskImageProcessorTool,
AskVisionCoderArgs,
AskVisionCoderTool,
ImageProcessorTool,
ask_image_processor,
)
assert issubclass(AskImageProcessorTool, ArtemisTool)
assert issubclass(AskImageProcessor, ArtemisTool)
assert issubclass(AskVisionCoderTool, ArtemisTool)
assert issubclass(ImageProcessorTool, ArtemisTool)
assert isinstance(ask_image_processor, ArtemisTool)
assert isinstance(ask_image_processor, AskImageProcessorTool)
assert ask_image_processor.name == "ask_image_processor"
assert ask_image_processor.category == "custom"
assert ask_image_processor.args_schema == AskVisionCoderArgs
# Registry lookup
reg_tool = ToolRegistry.get("ask_image_processor")
assert reg_tool is not None
assert isinstance(reg_tool, AskImageProcessorTool)
# GenAI FunctionDeclaration export
declaration = ask_image_processor.to_genai_declaration()
assert declaration.name == "ask_image_processor"
assert "instruction" in declaration.parameters.properties
assert "target_image_id" in declaration.parameters.properties
+182
View File
@@ -0,0 +1,182 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.exec_tools import (
InputText,
InputTextArgs,
InputTextTool,
get_input_text_tool,
input_text,
input_text_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
driver.screen_size = (1080, 2400)
driver.tap = AsyncMock(return_value=True)
driver.input_text = AsyncMock(return_value=True)
return driver
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def test_input_text_tool_subclass_and_registry():
"""Verify InputTextTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(InputTextTool, ArtemisTool)
assert issubclass(InputText, ArtemisTool)
assert isinstance(input_text, ArtemisTool)
assert isinstance(input_text, InputTextTool)
assert input_text.name == "input_text"
assert input_text.category == "action"
assert input_text.args_schema == InputTextArgs
# Registry lookup
reg_tool = ToolRegistry.get("input_text")
assert reg_tool is not None
assert isinstance(reg_tool, InputTextTool)
# GenAI FunctionDeclaration export
declaration = input_text.to_genai_declaration()
assert declaration.name == "input_text"
assert "text" in declaration.parameters.properties
assert "target" in declaration.parameters.properties
assert "clear_exist" in declaration.parameters.properties
# Wrapper check
assert input_text_wrapper is not None
assert input_text_wrapper.tool_fn_getter == get_input_text_tool
@pytest.mark.asyncio
async def test_input_text_direct_execution_with_driver(mock_driver):
"""Verify direct execution of InputTextTool with BaseDeviceDriver."""
result = await input_text.execute(
driver=mock_driver, text="hello world", target=[500, 500], clear_exist=True
)
mock_driver.tap.assert_called_once_with(540, 1200)
mock_driver.input_text.assert_called_once_with("hello world", clear_existing=True)
assert "Typed 'hello world' successfully." in result
@pytest.mark.asyncio
async def test_input_text_direct_execution_with_ctx(mock_ctx):
"""Verify direct execution of InputTextTool with ArtemisContext."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_inst.erase_text = AsyncMock(return_value=True)
mock_controller_inst.type_text = AsyncMock(return_value=True)
mock_controller_cls.return_value = mock_controller_inst
result = await input_text.execute(
ctx=mock_ctx, text="test input", target=[250, 750], clear_exist=True
)
mock_controller_inst.tap_at.assert_called_once_with(270, 1800)
mock_controller_inst.erase_text.assert_called_once()
mock_controller_inst.type_text.assert_called_once_with("test input", clear_existing=False)
assert "Typed 'test input' successfully." in result
@pytest.mark.asyncio
async def test_input_text_with_state_command(mock_ctx, mock_state):
"""Verify InputTextTool returns Command when state is provided."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_inst.erase_text = AsyncMock(return_value=True)
mock_controller_inst.type_text = AsyncMock(return_value=True)
mock_controller_cls.return_value = mock_controller_inst
cmd = await input_text.execute(
ctx=mock_ctx,
text="query",
target=[100, 200],
tool_call_id="call_it_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_it_1"
assert messages[0].status == "success"
assert "Typed 'query'" in messages[0].content
@pytest.mark.asyncio
async def test_input_text_callable_execution(mock_driver):
"""Verify invoking input_text directly as a callable."""
result = await input_text(driver=mock_driver, text="call text", target=[100, 100])
assert "Typed 'call text' successfully." in result
@pytest.mark.asyncio
async def test_input_text_invalid_target(mock_ctx):
"""Verify error handling on invalid target coordinates."""
result = await input_text.execute(ctx=mock_ctx, text="bad", target=[])
assert "Error during input text: Invalid target coordinates" in result
@pytest.mark.asyncio
async def test_get_input_text_tool_langchain_ainvoke(mock_ctx):
"""Verify get_input_text_tool exports a LangChain tool that works with ainvoke."""
it_tool = get_input_text_tool(mock_ctx)
assert it_tool.name == "input_text"
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_inst.press_key = AsyncMock(return_value=True)
mock_controller_inst.type_text = AsyncMock(return_value=True)
mock_controller_cls.return_value = mock_controller_inst
result = await it_tool.ainvoke(
{
"text": "abc",
"target": [500, 500],
"clear_exist": False,
}
)
mock_controller_inst.press_key.assert_called_once_with("123")
assert "Typed 'abc' successfully." in result
+92
View File
@@ -0,0 +1,92 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock, patch
from artemis.agents.hopper.hopper import HopperOutput
from artemis.context import ArtemisContext
from artemis.tools.mobile.launch_app import find_package
import pytest
@pytest.fixture
def mock_context():
ctx = MagicMock(spec=ArtemisContext)
ctx.package_cache = {}
return ctx
@pytest.mark.asyncio
@patch("artemis.tools.mobile.launch_app.list_packages_async")
@patch("artemis.tools.mobile.launch_app.hopper")
async def test_find_package_cache_hit(mock_hopper, mock_list_packages, mock_context):
mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
# First call: not in cache, calls hopper
mock_hopper.return_value = HopperOutput(
found=True, output="com.example.app1", reason="Found it"
)
pkg1 = await find_package(mock_context, "App 1")
assert pkg1 == "com.example.app1"
assert mock_hopper.call_count == 1
assert mock_context.package_cache["App 1"] == "com.example.app1"
# Second call: cache hit, does not call hopper or list_packages again
mock_hopper.reset_mock()
mock_list_packages.reset_mock()
pkg2 = await find_package(mock_context, "App 1")
assert pkg2 == "com.example.app1"
mock_hopper.assert_not_called()
mock_list_packages.assert_not_called()
@pytest.mark.asyncio
@patch("artemis.tools.mobile.launch_app.list_packages_async")
@patch("artemis.tools.mobile.launch_app.hopper")
async def test_find_package_not_found_caches_none(mock_hopper, mock_list_packages, mock_context):
mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
# Hopper doesn't find it
mock_hopper.return_value = HopperOutput(found=False, output=None, reason="Not found")
pkg1 = await find_package(mock_context, "App 3")
assert pkg1 is None
assert mock_hopper.call_count == 1
assert mock_context.package_cache["App 3"] is None
# Second call: cache hit (None), does not call hopper again
mock_hopper.reset_mock()
pkg2 = await find_package(mock_context, "App 3")
assert pkg2 is None
mock_hopper.assert_not_called()
@pytest.mark.asyncio
@patch("artemis.tools.mobile.launch_app.list_packages_async")
@patch("artemis.tools.mobile.launch_app.hopper")
async def test_find_package_passes_use_fallback(mock_hopper, mock_list_packages, mock_context):
mock_list_packages.return_value = "com.example.app1\ncom.example.app2"
mock_hopper.return_value = HopperOutput(
found=True, output="com.example.app1", reason="Found it"
)
# Pass use_fallback=False
await find_package(mock_context, "App 1", use_fallback=False)
mock_hopper.assert_called_once_with(
ctx=mock_context,
request=("I'm looking for the package name of the following app: 'App 1'"),
data="com.example.app1\ncom.example.app2",
use_fallback=False,
)
+208
View File
@@ -0,0 +1,208 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.launch_app import (
LaunchApp,
LaunchAppArgs,
LaunchAppTool,
get_launch_app_tool,
launch_app,
launch_app_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.package_cache = {}
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
driver.launch_app = AsyncMock(return_value=True)
return driver
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def test_launch_app_tool_subclass_and_registry():
"""Verify LaunchAppTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(LaunchAppTool, ArtemisTool)
assert issubclass(LaunchApp, ArtemisTool)
assert isinstance(launch_app, ArtemisTool)
assert isinstance(launch_app, LaunchAppTool)
assert launch_app.name == "launch_app"
assert launch_app.category == "action"
assert launch_app.args_schema == LaunchAppArgs
# Registry lookup
reg_tool = ToolRegistry.get("launch_app")
assert reg_tool is not None
assert isinstance(reg_tool, LaunchAppTool)
# GenAI FunctionDeclaration export
declaration = launch_app.to_genai_declaration()
assert declaration.name == "launch_app"
assert "app_name" in declaration.parameters.properties
# Wrapper check
assert launch_app_wrapper is not None
assert launch_app_wrapper.tool_fn_getter == get_launch_app_tool
@pytest.mark.asyncio
async def test_launch_app_direct_execution_with_driver(mock_driver):
"""Verify direct execution with BaseDeviceDriver."""
result = await launch_app.execute(driver=mock_driver, app_name="com.android.settings")
mock_driver.launch_app.assert_called_once_with("com.android.settings")
assert "App 'com.android.settings' launched successfully." in result
@pytest.mark.asyncio
async def test_launch_app_direct_execution_with_ctx(mock_ctx):
"""Verify direct execution with ArtemisContext."""
with (
patch(
"artemis.tools.mobile.launch_app.find_package",
new_callable=AsyncMock,
return_value="com.example.myapp",
) as mock_find,
patch(
"artemis.tools.mobile.launch_app.launch_app_with_retries",
new_callable=AsyncMock,
return_value=(True, None),
) as mock_launch,
):
result = await launch_app.execute(ctx=mock_ctx, app_name="My App")
mock_find.assert_called_once_with(ctx=mock_ctx, app_name="My App")
mock_launch.assert_called_once_with(ctx=mock_ctx, app_package="com.example.myapp")
assert "App 'My App' launched successfully." in result
@pytest.mark.asyncio
async def test_launch_app_package_not_found(mock_ctx):
"""Verify outcome when package is not found."""
with patch(
"artemis.tools.mobile.launch_app.find_package",
new_callable=AsyncMock,
return_value=None,
):
result = await launch_app.execute(ctx=mock_ctx, app_name="Nonexistent App")
assert "Failed to launch app 'Nonexistent App': Package not found." in result
@pytest.mark.asyncio
async def test_launch_app_execution_failure(mock_ctx):
"""Verify outcome when launch_app_with_retries fails."""
with (
patch(
"artemis.tools.mobile.launch_app.find_package",
new_callable=AsyncMock,
return_value="com.example.failapp",
),
patch(
"artemis.tools.mobile.launch_app.launch_app_with_retries",
new_callable=AsyncMock,
return_value=(False, "App crashed on startup"),
),
):
result = await launch_app.execute(ctx=mock_ctx, app_name="Crash App")
assert "Failed to launch app 'Crash App': App crashed on startup" in result
@pytest.mark.asyncio
async def test_launch_app_with_state_command(mock_ctx, mock_state):
"""Verify LaunchAppTool returns Command when state is provided."""
with (
patch(
"artemis.tools.mobile.launch_app.find_package",
new_callable=AsyncMock,
return_value="com.example.stateapp",
),
patch(
"artemis.tools.mobile.launch_app.launch_app_with_retries",
new_callable=AsyncMock,
return_value=(True, None),
),
):
cmd = await launch_app.execute(
ctx=mock_ctx,
app_name="State App",
tool_call_id="call_la_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_la_1"
assert messages[0].status == "success"
assert "App 'State App' launched successfully." in messages[0].content
@pytest.mark.asyncio
async def test_launch_app_callable_execution(mock_driver):
"""Verify invoking launch_app directly as a callable."""
result = await launch_app(driver=mock_driver, app_name="com.android.chrome")
assert "App 'com.android.chrome' launched successfully." in result
@pytest.mark.asyncio
async def test_launch_app_missing_name(mock_ctx):
"""Verify error handling when app_name is empty."""
result = await launch_app.execute(ctx=mock_ctx, app_name="")
assert "Error during launch app: app_name parameter is required." in result
@pytest.mark.asyncio
async def test_get_launch_app_tool_langchain_ainvoke(mock_ctx):
"""Verify get_launch_app_tool exports a LangChain tool that works with ainvoke."""
la_tool = get_launch_app_tool(mock_ctx)
assert la_tool.name == "launch_app"
with (
patch(
"artemis.tools.mobile.launch_app.find_package",
new_callable=AsyncMock,
return_value="com.example.ainvoke",
),
patch(
"artemis.tools.mobile.launch_app.launch_app_with_retries",
new_callable=AsyncMock,
return_value=(True, None),
),
):
result = await la_tool.ainvoke({"app_name": "Invoke App"})
assert "App 'Invoke App' launched successfully." in result
@@ -0,0 +1,109 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
ListNotesArgs,
ListNotesPure,
ListNotesPureTool,
get_list_notes_tool_pure,
list_notes_pure,
)
from artemis.utils.notes import save_note_content
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_list_notes_pure_tool_subclass_and_registry():
"""Verify ListNotesPureTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(ListNotesPureTool, ArtemisTool)
assert issubclass(ListNotesPure, ArtemisTool)
assert isinstance(list_notes_pure, ArtemisTool)
assert isinstance(list_notes_pure, ListNotesPureTool)
assert list_notes_pure.name == "list_notes_pure"
assert list_notes_pure.category == "memory"
assert list_notes_pure.args_schema == ListNotesArgs
# Registry lookup
reg_tool = ToolRegistry.get("list_notes_pure")
assert reg_tool is not None
assert isinstance(reg_tool, ListNotesPureTool)
# GenAI FunctionDeclaration export
declaration = list_notes_pure.to_genai_declaration()
assert declaration.name == "list_notes_pure"
@pytest.mark.asyncio
async def test_list_notes_pure_direct_execution_empty(mock_ctx):
"""Verify listing notes when no notes are stored."""
result = await list_notes_pure.execute(ctx=mock_ctx)
assert "No notes saved yet." in result
@pytest.mark.asyncio
async def test_list_notes_pure_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of ListNotesPureTool.execute lists stored notes."""
save_note_content(tmp_path, "note_1", "line 1\nline 2")
save_note_content(tmp_path, "note_2", "single line")
result = await list_notes_pure.execute(ctx=mock_ctx)
assert "Here are all the notes:" in result
assert "- note_1 (2 lines)" in result
assert "- note_2 (1 lines)" in result
@pytest.mark.asyncio
async def test_list_notes_pure_callable_execution(mock_ctx, tmp_path):
"""Verify invoking list_notes_pure directly as a callable."""
save_note_content(tmp_path, "callable_note", "line A\nline B")
result = await list_notes_pure(ctx=mock_ctx)
assert "Here are all the notes:" in result
assert "- callable_note (2 lines)" in result
@pytest.mark.asyncio
async def test_list_notes_pure_execution_failure(mock_ctx):
"""Verify error handling when list_notes_pure fails."""
with patch(
"artemis.tools.scratchpad.list_notes_info",
side_effect=RuntimeError("Disk failure"),
):
result = await list_notes_pure.execute(ctx=mock_ctx)
assert "Failed to list notes: Disk failure" in result
@pytest.mark.asyncio
async def test_get_list_notes_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_list_notes_tool_pure exports a LangChain tool named 'list_notes'."""
save_note_content(tmp_path, "pure_export_note", "content")
pure_tool = get_list_notes_tool_pure(mock_ctx)
assert pure_tool.name == "list_notes"
result = await pure_tool.ainvoke({})
assert "Here are all the notes:" in result
assert "- pure_export_note (1 lines)" in result
+158
View File
@@ -0,0 +1,158 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
ListNotes,
ListNotesArgs,
ListNotesTool,
get_list_notes_tool,
get_list_notes_tool_pure,
list_notes,
list_notes_wrapper,
)
from artemis.utils.notes import save_note_content
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_list_notes_tool_subclass_and_registry():
"""Verify ListNotesTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(ListNotesTool, ArtemisTool)
assert issubclass(ListNotes, ArtemisTool)
assert isinstance(list_notes, ArtemisTool)
assert isinstance(list_notes, ListNotesTool)
assert list_notes.name == "list_notes"
assert list_notes.category == "memory"
assert list_notes.args_schema == ListNotesArgs
# Registry lookup
reg_tool = ToolRegistry.get("list_notes")
assert reg_tool is not None
assert isinstance(reg_tool, ListNotesTool)
# GenAI FunctionDeclaration export
declaration = list_notes.to_genai_declaration()
assert declaration.name == "list_notes"
# Wrapper check
assert list_notes_wrapper is not None
assert list_notes_wrapper.tool_fn_getter == get_list_notes_tool
@pytest.mark.asyncio
async def test_list_notes_direct_execution_empty(mock_ctx):
"""Verify listing notes when no notes are stored."""
result = await list_notes.execute(ctx=mock_ctx)
assert "No notes saved yet." in result
@pytest.mark.asyncio
async def test_list_notes_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of ListNotesTool.execute lists stored notes."""
save_note_content(tmp_path, "note_alpha", "line 1\nline 2")
save_note_content(tmp_path, "note_beta", "single line")
result = await list_notes.execute(ctx=mock_ctx)
assert "Here are all the notes:" in result
assert "- note_alpha (2 lines)" in result
assert "- note_beta (1 lines)" in result
@pytest.mark.asyncio
async def test_list_notes_callable_execution(mock_ctx, tmp_path):
"""Verify invoking list_notes directly as a callable."""
save_note_content(tmp_path, "note_gamma", "line A\nline B\nline C")
result = await list_notes(ctx=mock_ctx)
assert "Here are all the notes:" in result
assert "- note_gamma (3 lines)" in result
@pytest.mark.asyncio
async def test_list_notes_with_state_command(mock_ctx, tmp_path):
"""Verify ListNotesTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
save_note_content(tmp_path, "state_note", "content")
mock_state = MagicMock(spec=State)
mock_state.current_agent = "operator"
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
result = await list_notes.execute(
ctx=mock_ctx,
state=mock_state,
tool_call_id="call_list_999",
)
assert isinstance(result, Command)
assert VALIDATOR_MESSAGES_KEY in result.update
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
assert len(tool_messages) == 1
assert tool_messages[0].tool_call_id == "call_list_999"
assert "Here are all the notes:" in tool_messages[0].content
assert "- state_note (1 lines)" in tool_messages[0].content
assert tool_messages[0].status == "success"
mock_state.asanitize_update.assert_called_once()
@pytest.mark.asyncio
async def test_list_notes_execution_failure(mock_ctx):
"""Verify error handling when list_notes fails."""
with patch(
"artemis.tools.scratchpad.list_notes_info",
side_effect=RuntimeError("Disk failure"),
):
result = await list_notes.execute(ctx=mock_ctx)
assert "Failed to list notes: Disk failure" in result
@pytest.mark.asyncio
async def test_get_list_notes_tool_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_list_notes_tool exports a LangChain tool that works with ainvoke."""
save_note_content(tmp_path, "lc_note", "hello")
lc_tool = get_list_notes_tool(mock_ctx)
assert lc_tool.name == "list_notes"
result = await lc_tool.ainvoke({})
assert "Here are all the notes:" in result
assert "- lc_note (1 lines)" in result
@pytest.mark.asyncio
async def test_get_list_notes_tool_pure(mock_ctx, tmp_path):
"""Verify get_list_notes_tool_pure returns a pure LangChain tool."""
save_note_content(tmp_path, "pure_note", "world")
pure_tool = get_list_notes_tool_pure(mock_ctx)
assert pure_tool.name == "list_notes"
result = await pure_tool.ainvoke({})
assert "Here are all the notes:" in result
assert "- pure_note (1 lines)" in result
+115
View File
@@ -0,0 +1,115 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.log_tool import (
AnalyzeLogs,
AnalyzeLogsArgs,
AnalyzeLogsTool,
LogTool,
analyze_logs,
analyze_logs_wrapper,
get_analyze_logs_tool,
)
import pytest
@pytest.mark.asyncio
async def test_analyze_logs_tool_execution():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_state = MagicMock(spec=State)
expected_output = "Found 2 errors in system log: NullPointerException at line 42."
mock_analyst = MagicMock()
mock_analyst.run = AsyncMock(return_value=expected_output)
with patch(
"artemis.tools.log_tool.LogAnalyzerNode",
return_value=mock_analyst,
):
tool = get_analyze_logs_tool(mock_ctx)
result = await tool.ainvoke(
{
"specific_query": "Check for crash exceptions",
"state": mock_state,
}
)
mock_analyst.run.assert_called_once_with("Check for crash exceptions", mock_state)
assert result == expected_output
@pytest.mark.asyncio
async def test_analyze_logs_tool_direct_call():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_state = MagicMock(spec=State)
mock_analyst = MagicMock()
mock_analyst.run = AsyncMock(return_value="Log scan complete.")
with patch(
"artemis.tools.log_tool.LogAnalyzerNode",
return_value=mock_analyst,
):
result = await analyze_logs(
ctx=mock_ctx,
specific_query="Find ANR",
state=mock_state,
)
mock_analyst.run.assert_called_once_with("Find ANR", mock_state)
assert result == "Log scan complete."
@pytest.mark.asyncio
async def test_analyze_logs_tool_no_context():
result = await analyze_logs(
ctx=None,
specific_query="Find crash",
)
assert result == "Error: ArtemisContext is required for LogAnalyzer."
def test_analyze_logs_tool_subclass_and_registry():
"""Verify AnalyzeLogsTool is an ArtemisTool subclass and registered in ToolRegistry."""
assert issubclass(AnalyzeLogsTool, ArtemisTool)
assert issubclass(AnalyzeLogs, ArtemisTool)
assert issubclass(LogTool, ArtemisTool)
assert isinstance(analyze_logs, ArtemisTool)
assert isinstance(analyze_logs, AnalyzeLogsTool)
assert analyze_logs.name == "analyze_logs"
assert analyze_logs.category == "custom"
assert analyze_logs.args_schema == AnalyzeLogsArgs
# Registry lookup
reg_tool = ToolRegistry.get("analyze_logs")
assert reg_tool is not None
assert isinstance(reg_tool, AnalyzeLogsTool)
# GenAI FunctionDeclaration export
declaration = analyze_logs.to_genai_declaration()
assert declaration.name == "analyze_logs"
assert "specific_query" in declaration.parameters.properties
# Wrapper check
assert analyze_logs_wrapper is not None
assert analyze_logs_wrapper.tool_fn_getter == get_analyze_logs_tool
+116
View File
@@ -0,0 +1,116 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from datetime import datetime
from unittest.mock import Mock, patch
from artemis.context import ArtemisContext
from artemis.tools.mobile.log_utils import fetch_and_filter_logs
import pytest
@pytest.fixture
def mock_context():
ctx = Mock(spec=ArtemisContext)
ctx.data_engine = Mock()
ctx.data_engine.session_start_time = 1700000000.0 # Example timestamp
return ctx
@pytest.fixture
def mock_device():
return Mock()
@patch("artemis.tools.mobile.log_utils.get_adb_device")
def test_fetch_and_filter_logs_basic(mock_get_adb_device, mock_context, mock_device):
mock_get_adb_device.return_value = mock_device
mock_device.shell.return_value = "line 1\nline 2\n"
result = fetch_and_filter_logs(mock_context, lines=10)
mock_device.shell.assert_called_once_with("logcat -v threadtime -t 10")
assert result == "line 1\nline 2\n"
@patch("artemis.tools.mobile.log_utils.get_adb_device")
@patch("artemis.tools.mobile.log_utils.resolve_time")
def test_fetch_and_filter_logs_since_time_success(
mock_resolve_time, mock_get_adb_device, mock_context, mock_device
):
mock_get_adb_device.return_value = mock_device
since_dt = datetime(datetime.now().year, 5, 3, 11, 11, 11, 111000)
mock_resolve_time.side_effect = [
("05-03 11:11:11.111", since_dt), # for since_time
(None, None), # for until_time
]
mock_device.shell.return_value = "05-03 11:11:12.000 line 1\n05-03 11:11:13.000 line 2\n"
result = fetch_and_filter_logs(mock_context, since_time="05-03 11:11:11.111")
mock_device.shell.assert_called_once_with('logcat -v threadtime -t "05-03 11:11:11.111"')
assert "line 1" in result
assert "line 2" in result
@patch("artemis.tools.mobile.log_utils.get_adb_device")
@patch("artemis.tools.mobile.log_utils.resolve_time")
def test_fetch_and_filter_logs_since_time_fallback(
mock_resolve_time, mock_get_adb_device, mock_context, mock_device
):
mock_get_adb_device.return_value = mock_device
current_year = datetime.now().year
since_dt = datetime(current_year, 5, 4, 12, 0, 0)
mock_resolve_time.side_effect = [
("05-04 12:00:00.000", since_dt), # for since_time
(None, None), # for until_time
]
# Direct fetch fails
mock_device.shell.side_effect = [
Exception("logcat failed"), # first call to direct fetch
"05-04 11:59:00.000 line old\n05-04 12:00:01.000 line new\n", # second call (fallback)
]
result = fetch_and_filter_logs(mock_context, lines=100, since_time="05-04 12:00:00.000")
assert mock_device.shell.call_count == 2
mock_device.shell.assert_any_call('logcat -v threadtime -t "05-04 12:00:00.000"')
mock_device.shell.assert_any_call("logcat -v threadtime -t 100")
# Should filter out "line old" because it's before since_dt
assert "line old" not in result
assert "line new" in result
@patch("artemis.tools.mobile.log_utils.get_adb_device")
@patch("artemis.tools.mobile.log_utils.resolve_time")
def test_fetch_and_filter_logs_until_time(
mock_resolve_time, mock_get_adb_device, mock_context, mock_device
):
mock_get_adb_device.return_value = mock_device
current_year = datetime.now().year
until_dt = datetime(current_year, 5, 4, 12, 0, 0)
mock_resolve_time.side_effect = [
(None, None), # for since_time
("05-04 12:00:00.000", until_dt), # for until_time
]
mock_device.shell.return_value = "05-04 11:59:00.000 line old\n05-04 12:00:01.000 line new\n"
result = fetch_and_filter_logs(mock_context, lines=100, until_time="05-04 12:00:00.000")
mock_device.shell.assert_called_once_with("logcat -v threadtime -t 100")
assert "line old" in result
assert "line new" not in result
+162
View File
@@ -0,0 +1,162 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.exec_tools import (
LongPress,
LongPressArgs,
LongPressTool,
get_long_press_tool,
long_press,
long_press_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
driver.screen_size = (1080, 2400)
driver.long_press = AsyncMock(return_value=True)
return driver
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def test_long_press_tool_subclass_and_registry():
"""Verify LongPressTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(LongPressTool, ArtemisTool)
assert issubclass(LongPress, ArtemisTool)
assert isinstance(long_press, ArtemisTool)
assert isinstance(long_press, LongPressTool)
assert long_press.name == "long_press"
assert long_press.category == "action"
assert long_press.args_schema == LongPressArgs
# Registry lookup
reg_tool = ToolRegistry.get("long_press")
assert reg_tool is not None
assert isinstance(reg_tool, LongPressTool)
# GenAI FunctionDeclaration export
declaration = long_press.to_genai_declaration()
assert declaration.name == "long_press"
assert "target" in declaration.parameters.properties
assert "duration" in declaration.parameters.properties
# Wrapper check
assert long_press_wrapper is not None
assert long_press_wrapper.tool_fn_getter == get_long_press_tool
@pytest.mark.asyncio
async def test_long_press_direct_execution_with_driver(mock_driver):
"""Verify direct execution of LongPressTool with BaseDeviceDriver."""
result = await long_press.execute(driver=mock_driver, target=[500, 500], duration=1500)
mock_driver.long_press.assert_called_once_with(540, 1200, duration_ms=1500)
assert "Long pressed at [500, 500] (normalized) successfully." in result
@pytest.mark.asyncio
async def test_long_press_direct_execution_with_ctx(mock_ctx):
"""Verify direct execution of LongPressTool with ArtemisContext."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_cls.return_value = mock_controller_inst
result = await long_press.execute(ctx=mock_ctx, target=[250, 750], duration=2000)
mock_controller_inst.tap_at.assert_called_once_with(
270, 1800, long_press=True, long_press_duration=2000
)
assert "Long pressed at [250, 750] (normalized) successfully." in result
@pytest.mark.asyncio
async def test_long_press_with_state_command(mock_ctx, mock_state):
"""Verify LongPressTool returns Command when state is provided."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_cls.return_value = mock_controller_inst
cmd = await long_press.execute(
ctx=mock_ctx,
target=[100, 200],
duration=1200,
tool_call_id="call_lp_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_lp_1"
assert messages[0].status == "success"
assert "Long pressed at [100, 200]" in messages[0].content
@pytest.mark.asyncio
async def test_long_press_callable_execution(mock_driver):
"""Verify invoking long_press directly as a callable."""
result = await long_press(driver=mock_driver, target=[100, 100])
assert "Long pressed at [100, 100] (normalized) successfully." in result
@pytest.mark.asyncio
async def test_long_press_invalid_target(mock_ctx):
"""Verify error handling on invalid target coordinates."""
result = await long_press.execute(ctx=mock_ctx, target=[])
assert "Error during long press: Invalid target coordinates" in result
@pytest.mark.asyncio
async def test_get_long_press_tool_langchain_ainvoke(mock_ctx):
"""Verify get_long_press_tool exports a LangChain tool that works with ainvoke."""
lp_tool = get_long_press_tool(mock_ctx)
assert lp_tool.name == "long_press"
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.tap_at = AsyncMock(return_value=MagicMock(error=None))
mock_controller_cls.return_value = mock_controller_inst
result = await lp_tool.ainvoke({"target": [500, 500], "duration": 1500})
assert "Long pressed at [500, 500] (normalized) successfully." in result
@@ -0,0 +1,186 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for ObjectDetectionTool and OperatorObjectDetectionTool."""
import json
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.object_detection_tool import (
ObjectDetection,
ObjectDetectionArgs,
ObjectDetectionTool,
ObjectDetectorTool,
OperatorObjectDetection,
OperatorObjectDetectionArgs,
OperatorObjectDetectionTool,
OperatorObjectDetectorTool,
get_object_detector_tool,
get_operator_object_detector_tool,
object_detection,
object_detection_wrapper,
operator_object_detection,
operator_object_detection_wrapper,
)
import pytest
def test_object_detection_tool_subclass_and_registry():
"""Verify ObjectDetectionTool and OperatorObjectDetectionTool subclass ArtemisTool and register."""
assert issubclass(ObjectDetectionTool, ArtemisTool)
assert issubclass(ObjectDetection, ArtemisTool)
assert issubclass(ObjectDetectorTool, ArtemisTool)
assert issubclass(OperatorObjectDetectionTool, ArtemisTool)
assert issubclass(OperatorObjectDetectionTool, ObjectDetectionTool)
assert issubclass(OperatorObjectDetection, ArtemisTool)
assert issubclass(OperatorObjectDetectorTool, ArtemisTool)
assert isinstance(object_detection, ArtemisTool)
assert isinstance(object_detection, ObjectDetectionTool)
assert isinstance(operator_object_detection, ArtemisTool)
assert isinstance(operator_object_detection, OperatorObjectDetectionTool)
# Properties
assert object_detection.name == "object_detection"
assert object_detection.category == "perception"
assert object_detection.args_schema == ObjectDetectionArgs
assert operator_object_detection.name == "object_detection"
assert operator_object_detection.category == "perception"
assert operator_object_detection.args_schema == OperatorObjectDetectionArgs
# Registry lookup
reg_tool = ToolRegistry.get("object_detection")
assert reg_tool is not None
assert isinstance(reg_tool, ObjectDetectionTool)
# GenAI FunctionDeclaration export
declaration = object_detection.to_genai_declaration()
assert declaration.name == "object_detection"
assert "image_path" in declaration.parameters.properties
assert "queries" in declaration.parameters.properties
op_declaration = operator_object_detection.to_genai_declaration()
assert op_declaration.name == "object_detection"
assert "queries" in op_declaration.parameters.properties
# Wrappers
assert object_detection_wrapper.tool_fn_getter == get_object_detector_tool
assert operator_object_detection_wrapper.tool_fn_getter == get_operator_object_detector_tool
@pytest.mark.asyncio
async def test_object_detection_direct_execution_success():
"""Verify direct async call to object_detection with image_path."""
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
expected_result = {
"detected": [{"label": "button", "point": [100, 200]}],
"failed": [],
}
with patch(
"artemis.tools.object_detection_tool._run_object_detection",
AsyncMock(return_value=expected_result),
) as mock_run:
result = await object_detection(
ctx=mock_ctx,
image_path="/path/to/screenshot.jpg",
queries=["button"],
)
mock_run.assert_called_once()
assert json.loads(result) == expected_result
@pytest.mark.asyncio
async def test_operator_object_detection_direct_execution_with_state():
"""Verify operator_object_detection reads screenshot from state."""
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/path/to/state_screen.jpg"
mock_state.asanitize_update = None # Non-langgraph direct call
expected_result = {
"detected": [{"label": "search bar", "point": [500, 100]}],
"failed": [],
}
with patch(
"artemis.tools.object_detection_tool._run_object_detection",
AsyncMock(return_value=expected_result),
) as mock_run:
result = await operator_object_detection(
ctx=mock_ctx,
state=mock_state,
queries=["search bar"],
)
mock_run.assert_called_once()
# Verify it used the latest_screenshot from state
call_args = mock_run.call_args[0]
assert call_args[1] == "/path/to/state_screen.jpg"
assert json.loads(result) == expected_result
@pytest.mark.asyncio
async def test_object_detection_no_image_error():
"""Verify error when no image path is given and no state screenshot is available."""
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
result = await object_detection(
ctx=mock_ctx,
image_path=None,
queries=["icon"],
)
assert "Object Detection failed" in result
assert "No image path provided" in result
@pytest.mark.asyncio
async def test_object_detection_exception_handling():
"""Verify graceful handling when _run_object_detection raises an exception."""
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = None
with patch(
"artemis.tools.object_detection_tool._run_object_detection",
AsyncMock(side_effect=RuntimeError("Model inference timeout")),
):
result = await object_detection(
ctx=mock_ctx,
image_path="/path/to/screen.jpg",
queries=["icon"],
)
assert "Object Detection failed: Model inference timeout" in result
def test_get_langchain_tools():
"""Verify LangChain BaseTool exports."""
mock_ctx = MagicMock(spec=ArtemisContext)
tool = get_object_detector_tool(mock_ctx)
op_tool = get_operator_object_detector_tool(mock_ctx)
assert tool.name == "object_detection"
assert tool.args_schema == ObjectDetectionArgs
assert op_tool.name == "object_detection"
assert op_tool.args_schema == OperatorObjectDetectionArgs
@@ -0,0 +1,70 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.object_detection_tool import get_operator_object_detector_tool
import pytest
@pytest.mark.asyncio
async def test_object_detection_trace():
mock_ctx = MagicMock(spec=ArtemisContext)
mock_ctx.data_engine = MagicMock()
mock_state = MagicMock(spec=State)
mock_state.latest_screenshot = "/path/to/screenshot.jpg"
mock_state.asanitize_update = AsyncMock(return_value={"status": "success"})
# We need to mock the decorator in object_detection_tool.py if we want to avoid it calling real get_llm
# Or we can just mock get_llm.
# In the tool definition: @trace(type="tool", ctx=ctx)
# This calls record_trace immediately on entry/exit.
mock_tool = get_operator_object_detector_tool(mock_ctx)
# Mock the internal function called by the tool
with patch(
"artemis.tools.object_detection_tool._run_object_detection",
AsyncMock(return_value="{}"),
) as mock_run:
# We need to handle the fact that ainvoke might fail if LangChain internals are not fully mocked.
# But let's try it.
await mock_tool.ainvoke(
{
"type": "tool_call",
"name": "object_detection",
"id": "call_123",
"args": {"queries": ["test_label"], "state": mock_state},
}
)
# Verify _run_object_detection was called
mock_run.assert_called_once()
# Verify record_trace was called for the span
assert mock_ctx.data_engine.record_trace.call_count >= 1
calls = mock_ctx.data_engine.record_trace.call_args_list
span_called = False
for call in calls:
kwargs = call.kwargs
if kwargs.get("type") == "span" and kwargs.get("name") == "run_object_detection":
span_called = True
break
assert span_called, "TraceSpan 'run_object_detection' was not called"
+238
View File
@@ -0,0 +1,238 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.ocr import (
OCRRecognition,
OcrArgs,
OcrRecognition,
OcrRecognitionTool,
OcrTool,
get_ocr_tool,
ocr_recognition,
ocr_recognition_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
driver.screen_size = (1080, 2400)
screen_data = MagicMock()
screen_data.screenshot_base64 = base64.b64encode(b"dummy_image_data").decode("utf-8")
screen_data.screenshot_bytes = b"dummy_image_data"
driver.get_screen_data = AsyncMock(return_value=screen_data)
return driver
@pytest.fixture
def mock_state(tmp_path):
screenshot_file = tmp_path / "test_screenshot.png"
screenshot_file.write_bytes(b"dummy_image_content")
state = MagicMock(spec=State)
state.latest_screenshot = str(screenshot_file)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def test_ocr_tool_subclass_and_registry():
"""Verify OcrRecognitionTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(OcrRecognitionTool, ArtemisTool)
assert issubclass(OcrRecognition, ArtemisTool)
assert issubclass(OCRRecognition, ArtemisTool)
assert issubclass(OcrTool, ArtemisTool)
assert isinstance(ocr_recognition, ArtemisTool)
assert isinstance(ocr_recognition, OcrRecognitionTool)
assert ocr_recognition.name == "ocr_recognition"
assert ocr_recognition.category == "explorer"
assert ocr_recognition.args_schema == OcrArgs
# Registry lookup
reg_tool = ToolRegistry.get("ocr_recognition")
assert reg_tool is not None
assert isinstance(reg_tool, OcrRecognitionTool)
# GenAI FunctionDeclaration export
declaration = ocr_recognition.to_genai_declaration()
assert declaration.name == "ocr_recognition"
# Wrapper check
assert ocr_recognition_wrapper is not None
assert ocr_recognition_wrapper.tool_fn_getter == get_ocr_tool
class SimpleState:
def __init__(self, screenshot_path: str):
self.latest_screenshot = screenshot_path
@pytest.mark.asyncio
async def test_ocr_direct_execution_with_state(mock_ctx, tmp_path):
"""Verify direct execution with state and screenshot file."""
screenshot_file = tmp_path / "test_screenshot.png"
screenshot_file.write_bytes(b"dummy_image_content")
simple_state = SimpleState(str(screenshot_file))
mock_results = [
{
"text": "Submit",
"position": [
{"x": 100, "y": 200},
{"x": 200, "y": 200},
{"x": 200, "y": 300},
{"x": 100, "y": 300},
],
}
]
with patch(
"artemis.tools.mobile.ocr.perform_ocr",
new_callable=AsyncMock,
return_value=mock_results,
):
result = await ocr_recognition.execute(ctx=mock_ctx, state=simple_state)
assert "OCR Recognition successful." in result
assert "Submit" in result
@pytest.mark.asyncio
async def test_ocr_direct_execution_with_driver(mock_ctx, mock_driver):
"""Verify direct execution with driver."""
mock_results = [
{
"text": "Settings",
"position": [
{"x": 500, "y": 500},
{"x": 600, "y": 600},
],
}
]
with patch(
"artemis.tools.mobile.ocr.perform_ocr",
new_callable=AsyncMock,
return_value=mock_results,
):
result = await ocr_recognition.execute(ctx=mock_ctx, driver=mock_driver)
assert "OCR Recognition successful." in result
assert "Settings" in result
@pytest.mark.asyncio
async def test_ocr_no_text_detected(mock_ctx, tmp_path):
"""Verify outcome when no text is detected."""
screenshot_file = tmp_path / "test_empty_screenshot.png"
screenshot_file.write_bytes(b"dummy_empty_content")
simple_state = SimpleState(str(screenshot_file))
with patch(
"artemis.tools.mobile.ocr.perform_ocr",
new_callable=AsyncMock,
return_value=[],
):
result = await ocr_recognition.execute(ctx=mock_ctx, state=simple_state)
assert "No text detected on the screen." in result
@pytest.mark.asyncio
async def test_ocr_with_state_command(mock_ctx, mock_state):
"""Verify OcrRecognitionTool returns Command when state is provided."""
mock_results = [{"text": "Hello", "position": [{"x": 100, "y": 100}]}]
with patch(
"artemis.tools.mobile.ocr.perform_ocr",
new_callable=AsyncMock,
return_value=mock_results,
):
cmd = await ocr_recognition.execute(
ctx=mock_ctx,
tool_call_id="call_ocr_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_ocr_1"
assert messages[0].status == "success"
assert "Hello" in messages[0].content
@pytest.mark.asyncio
async def test_ocr_failure_missing_screenshot(mock_ctx):
"""Verify error handling when screenshot path is missing."""
empty_state = MagicMock(spec=State)
empty_state.latest_screenshot = None
empty_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
cmd = await ocr_recognition.execute(
ctx=mock_ctx,
tool_call_id="call_ocr_err",
state=empty_state,
)
assert isinstance(cmd, Command)
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert messages[0].status == "error"
assert "OCR Recognition failed" in messages[0].content
@pytest.mark.asyncio
async def test_ocr_callable_execution(mock_ctx, mock_driver):
"""Verify invoking ocr_recognition directly as a callable."""
with patch(
"artemis.tools.mobile.ocr.perform_ocr",
new_callable=AsyncMock,
return_value=[{"text": "Callable", "position": []}],
):
result = await ocr_recognition(ctx=mock_ctx, driver=mock_driver)
assert "OCR Recognition successful." in result
assert "Callable" in result
@pytest.mark.asyncio
async def test_get_ocr_tool_langchain_ainvoke(mock_ctx, mock_driver):
"""Verify get_ocr_tool exports a LangChain tool that works with ainvoke."""
mock_ctx._active_driver = mock_driver
ocr_tool = get_ocr_tool(mock_ctx)
assert ocr_tool.name == "ocr_recognition"
with patch(
"artemis.tools.mobile.ocr.perform_ocr",
new_callable=AsyncMock,
return_value=[{"text": "LangChain", "position": []}],
):
result = await ocr_tool.ainvoke({})
assert "OCR Recognition successful." in result
assert "LangChain" in result
+174
View File
@@ -0,0 +1,174 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.exec_tools import (
PressKey,
PressKeyArgs,
PressKeyTool,
get_press_key_tool,
press_key,
press_key_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.device = MagicMock()
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
driver.press_key = AsyncMock(return_value=True)
return driver
@pytest.fixture
def mock_state():
state = MagicMock(spec=State)
async def _mock_asanitize_update(ctx, update, agent):
return update
state.asanitize_update = AsyncMock(side_effect=_mock_asanitize_update)
return state
def test_press_key_tool_subclass_and_registry():
"""Verify PressKeyTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(PressKeyTool, ArtemisTool)
assert issubclass(PressKey, ArtemisTool)
assert isinstance(press_key, ArtemisTool)
assert isinstance(press_key, PressKeyTool)
assert press_key.name == "press_key"
assert press_key.category == "action"
assert press_key.args_schema == PressKeyArgs
# Registry lookup
reg_tool = ToolRegistry.get("press_key")
assert reg_tool is not None
assert isinstance(reg_tool, PressKeyTool)
# GenAI FunctionDeclaration export
declaration = press_key.to_genai_declaration()
assert declaration.name == "press_key"
assert "key" in declaration.parameters.properties
# Wrapper check
assert press_key_wrapper is not None
assert press_key_wrapper.tool_fn_getter == get_press_key_tool
@pytest.mark.asyncio
async def test_press_key_direct_execution_with_driver(mock_driver):
"""Verify direct execution with BaseDeviceDriver."""
result = await press_key.execute(driver=mock_driver, key="BACK")
mock_driver.press_key.assert_called_once_with("BACK")
assert "Pressed key 'BACK' successfully." in result
@pytest.mark.asyncio
async def test_press_key_direct_execution_with_ctx(mock_ctx):
"""Verify direct execution with ArtemisContext for various keys."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.press_enter = AsyncMock(return_value=True)
mock_controller_inst.go_back = AsyncMock(return_value=True)
mock_controller_inst.go_home = AsyncMock(return_value=True)
mock_controller_inst.press_key = AsyncMock(return_value=True)
mock_controller_cls.return_value = mock_controller_inst
# Test ENTER
res_enter = await press_key.execute(ctx=mock_ctx, key="ENTER")
mock_controller_inst.press_enter.assert_called_once()
assert "Pressed key 'ENTER' successfully." in res_enter
# Test BACK
res_back = await press_key.execute(ctx=mock_ctx, key="BACK")
mock_controller_inst.go_back.assert_called_once()
assert "Pressed key 'BACK' successfully." in res_back
# Test HOME
res_home = await press_key.execute(ctx=mock_ctx, key="HOME")
mock_controller_inst.go_home.assert_called_once()
assert "Pressed key 'HOME' successfully." in res_home
# Test APP_SWITCH
res_app_switch = await press_key.execute(ctx=mock_ctx, key="APP_SWITCH")
mock_controller_inst.press_key.assert_called_with("KEYCODE_APP_SWITCH")
assert "Pressed key 'APP_SWITCH' successfully." in res_app_switch
@pytest.mark.asyncio
async def test_press_key_with_state_command(mock_ctx, mock_state):
"""Verify PressKeyTool returns Command when state is provided."""
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.go_home = AsyncMock(return_value=True)
mock_controller_cls.return_value = mock_controller_inst
cmd = await press_key.execute(
ctx=mock_ctx,
key="HOME",
tool_call_id="call_pk_1",
state=mock_state,
)
assert isinstance(cmd, Command)
assert VALIDATOR_MESSAGES_KEY in cmd.update
messages = cmd.update[VALIDATOR_MESSAGES_KEY]
assert len(messages) == 1
assert messages[0].tool_call_id == "call_pk_1"
assert messages[0].status == "success"
assert "Pressed key 'HOME' successfully." in messages[0].content
@pytest.mark.asyncio
async def test_press_key_callable_execution(mock_driver):
"""Verify invoking press_key directly as a callable."""
result = await press_key(driver=mock_driver, key="ENTER")
assert "Pressed key 'ENTER' successfully." in result
@pytest.mark.asyncio
async def test_press_key_missing_key(mock_ctx):
"""Verify error handling when key is empty or missing."""
result = await press_key.execute(ctx=mock_ctx, key="")
assert "Error during press key: Key parameter is required." in result
@pytest.mark.asyncio
async def test_get_press_key_tool_langchain_ainvoke(mock_ctx):
"""Verify get_press_key_tool exports a LangChain tool that works with ainvoke."""
pk_tool = get_press_key_tool(mock_ctx)
assert pk_tool.name == "press_key"
with patch("artemis.tools.mobile.exec_tools.UnifiedMobileController") as mock_controller_cls:
mock_controller_inst = MagicMock()
mock_controller_inst.go_back = AsyncMock(return_value=True)
mock_controller_cls.return_value = mock_controller_inst
result = await pk_tool.ainvoke({"key": "BACK"})
assert "Pressed key 'BACK' successfully." in result
@@ -0,0 +1,145 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock
from artemis.context import ArtemisContext
from artemis.drivers.base import BaseDeviceDriver
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.read_hierarchy import (
GetUIHierarchy,
GetUiHierarchy,
GetUiHierarchyArgs,
GetUiHierarchyTool,
get_ui_hierarchy,
get_ui_hierarchy_tool,
ui_hierarchy_wrapper,
)
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
ctx.ui_adb_client = MagicMock()
ctx.ui_adb_client.get_hierarchy.return_value = "<hierarchy><node text='Home'/></hierarchy>"
return ctx
@pytest.fixture
def mock_driver():
driver = MagicMock(spec=BaseDeviceDriver)
screen_data = MagicMock()
screen_data.ui_hierarchy_xml = "<hierarchy><node text='Settings'/></hierarchy>"
driver.get_screen_data = AsyncMock(return_value=screen_data)
return driver
def test_get_ui_hierarchy_tool_subclass_and_registry():
"""Verify GetUiHierarchyTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(GetUiHierarchyTool, ArtemisTool)
assert issubclass(GetUiHierarchy, ArtemisTool)
assert issubclass(GetUIHierarchy, ArtemisTool)
assert isinstance(get_ui_hierarchy, ArtemisTool)
assert isinstance(get_ui_hierarchy, GetUiHierarchyTool)
assert get_ui_hierarchy.name == "get_ui_hierarchy"
assert get_ui_hierarchy.category == "diagnostic"
assert get_ui_hierarchy.args_schema == GetUiHierarchyArgs
# Registry lookup
reg_tool = ToolRegistry.get("get_ui_hierarchy")
assert reg_tool is not None
assert isinstance(reg_tool, GetUiHierarchyTool)
# GenAI FunctionDeclaration export
declaration = get_ui_hierarchy.to_genai_declaration()
assert declaration.name == "get_ui_hierarchy"
# Wrapper check
assert ui_hierarchy_wrapper is not None
assert ui_hierarchy_wrapper.tool_fn_getter == get_ui_hierarchy_tool
@pytest.mark.asyncio
async def test_get_ui_hierarchy_direct_execution_with_ctx(mock_ctx):
"""Verify direct execution with ArtemisContext."""
result = await get_ui_hierarchy.execute(ctx=mock_ctx)
mock_ctx.ui_adb_client.get_hierarchy.assert_called_once()
assert "<hierarchy><node text='Home'/></hierarchy>" in result
@pytest.mark.asyncio
async def test_get_ui_hierarchy_direct_execution_with_driver_screen_data(mock_driver):
"""Verify direct execution with driver's get_screen_data."""
result = await get_ui_hierarchy.execute(driver=mock_driver)
mock_driver.get_screen_data.assert_called_once()
assert "<hierarchy><node text='Settings'/></hierarchy>" in result
@pytest.mark.asyncio
async def test_get_ui_hierarchy_direct_execution_with_driver_get_hierarchy():
"""Verify direct execution with driver's get_ui_hierarchy."""
driver = MagicMock(spec=BaseDeviceDriver)
del driver.get_screen_data
driver.get_ui_hierarchy = AsyncMock(
return_value="<hierarchy><node text='DirectDriver'/></hierarchy>"
)
result = await get_ui_hierarchy.execute(driver=driver)
driver.get_ui_hierarchy.assert_called_once()
assert "<hierarchy><node text='DirectDriver'/></hierarchy>" in result
@pytest.mark.asyncio
async def test_get_ui_hierarchy_with_state(mock_ctx):
"""Verify GetUiHierarchyTool returns string when state is provided."""
state = MagicMock()
result = await get_ui_hierarchy.execute(
ctx=mock_ctx,
tool_call_id="call_hier_1",
state=state,
)
assert "<hierarchy><node text='Home'/></hierarchy>" in result
@pytest.mark.asyncio
async def test_get_ui_hierarchy_no_client_or_driver():
"""Verify error handling when no client or driver is provided."""
result = await get_ui_hierarchy.execute()
assert "Error retrieving UI hierarchy: No UI Automator client or driver provided." in result
@pytest.mark.asyncio
async def test_get_ui_hierarchy_exception_handling(mock_ctx):
"""Verify error handling when ui_adb_client raises an exception."""
mock_ctx.ui_adb_client.get_hierarchy.side_effect = RuntimeError("ADB server died")
result = await get_ui_hierarchy.execute(ctx=mock_ctx)
assert "Error retrieving UI hierarchy: ADB server died" in result
@pytest.mark.asyncio
async def test_get_ui_hierarchy_callable_execution(mock_ctx):
"""Verify invoking get_ui_hierarchy directly as a callable."""
result = await get_ui_hierarchy(ctx=mock_ctx)
assert "<node text='Home'/>" in result
@pytest.mark.asyncio
async def test_get_ui_hierarchy_tool_langchain_ainvoke(mock_ctx):
"""Verify get_ui_hierarchy_tool exports a LangChain tool that works with ainvoke."""
tool = get_ui_hierarchy_tool(mock_ctx)
assert tool.name == "get_ui_hierarchy"
result = await tool.ainvoke({})
assert "<node text='Home'/>" in result
+140
View File
@@ -0,0 +1,140 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.read_logs import (
ReadLogs,
ReadLogsArgs,
ReadLogsTool,
ReadLogsToolAlias,
get_read_logs_tool,
read_logs,
)
import pytest
@pytest.fixture
def mock_ctx():
ctx = MagicMock(spec=ArtemisContext)
return ctx
def test_read_logs_tool_subclass_and_registry():
"""Verify ReadLogsTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(ReadLogsTool, ArtemisTool)
assert issubclass(ReadLogs, ArtemisTool)
assert issubclass(ReadLogsToolAlias, ArtemisTool)
assert isinstance(read_logs, ArtemisTool)
assert isinstance(read_logs, ReadLogsTool)
assert read_logs.name == "read_logs"
assert read_logs.category == "diagnostic"
assert read_logs.args_schema == ReadLogsArgs
# Registry lookup
reg_tool = ToolRegistry.get("read_logs")
assert reg_tool is not None
assert isinstance(reg_tool, ReadLogsTool)
# GenAI FunctionDeclaration export
declaration = read_logs.to_genai_declaration()
assert declaration.name == "read_logs"
assert "lines" in declaration.parameters.properties
assert "since_time" in declaration.parameters.properties
assert "until_time" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_read_logs_direct_execution(mock_ctx):
"""Verify direct execution of ReadLogsTool."""
mock_log_output = "08-12 10:00:00.000 1000 1000 I Test: Hello Logs\n"
with patch(
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
return_value=mock_log_output,
) as mock_fetch:
result = await read_logs.execute(
ctx=mock_ctx,
lines=100,
since_time="08-12 09:59:00",
until_time="08-12 10:01:00",
)
mock_fetch.assert_called_once_with(
ctx=mock_ctx,
lines=100,
since_time="08-12 09:59:00",
until_time="08-12 10:01:00",
)
assert result == mock_log_output
@pytest.mark.asyncio
async def test_read_logs_default_lines(mock_ctx):
"""Verify read_logs defaults lines to 200 when None."""
with patch(
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
return_value="logs",
) as mock_fetch:
result = await read_logs.execute(ctx=mock_ctx, lines=None)
mock_fetch.assert_called_once_with(
ctx=mock_ctx,
lines=200,
since_time=None,
until_time=None,
)
assert result == "logs"
@pytest.mark.asyncio
async def test_read_logs_exception_handling(mock_ctx):
"""Verify error handling when fetching logs raises an exception."""
with patch(
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
side_effect=RuntimeError("Device disconnected"),
):
result = await read_logs.execute(ctx=mock_ctx)
assert "Failed to read logs: Device disconnected" in result
@pytest.mark.asyncio
async def test_read_logs_callable_execution(mock_ctx):
"""Verify invoking read_logs directly as a callable."""
with patch(
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
return_value="sample log stream",
):
result = await read_logs(ctx=mock_ctx, lines=50)
assert result == "sample log stream"
@pytest.mark.asyncio
async def test_get_read_logs_tool_langchain_ainvoke(mock_ctx):
"""Verify get_read_logs_tool exports a LangChain tool that works with ainvoke."""
tool = get_read_logs_tool(mock_ctx)
assert tool.name == "read_logs"
with patch(
"artemis.tools.mobile.read_logs.fetch_and_filter_logs",
return_value="langchain log content",
) as mock_fetch:
result = await tool.ainvoke({"lines": 300})
mock_fetch.assert_called_once_with(
ctx=mock_ctx,
lines=300,
since_time=None,
until_time=None,
)
assert result == "langchain log content"
@@ -0,0 +1,127 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
ReadNoteArgs,
ReadNotePure,
ReadNotePureTool,
get_read_note_tool_pure,
read_note_pure,
)
from artemis.utils.notes import save_note_content
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_read_note_pure_tool_subclass_and_registry():
"""Verify ReadNotePureTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(ReadNotePureTool, ArtemisTool)
assert issubclass(ReadNotePure, ArtemisTool)
assert isinstance(read_note_pure, ArtemisTool)
assert isinstance(read_note_pure, ReadNotePureTool)
assert read_note_pure.name == "read_note_pure"
assert read_note_pure.category == "memory"
assert read_note_pure.args_schema == ReadNoteArgs
# Registry lookup
reg_tool = ToolRegistry.get("read_note_pure")
assert reg_tool is not None
assert isinstance(reg_tool, ReadNotePureTool)
# GenAI FunctionDeclaration export
declaration = read_note_pure.to_genai_declaration()
assert declaration.name == "read_note_pure"
assert "key" in declaration.parameters.properties
assert "start_line" in declaration.parameters.properties
assert "end_line" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_read_note_pure_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of ReadNotePureTool.execute reads saved note."""
save_note_content(tmp_path, "sample_pure", "line 1\nline 2\nline 3")
result = await read_note_pure.execute(ctx=mock_ctx, key="sample_pure")
assert "Successfully read note 'sample_pure'." in result
assert "line 1\nline 2\nline 3" in result
@pytest.mark.asyncio
async def test_read_note_pure_slice(mock_ctx, tmp_path):
"""Verify reading a line slice with start_line and end_line via ReadNotePureTool."""
save_note_content(tmp_path, "slice_pure", "line 1\nline 2\nline 3\nline 4")
result = await read_note_pure.execute(
ctx=mock_ctx,
key="slice_pure",
start_line=2,
end_line=3,
)
assert "Successfully read note 'slice_pure' (lines 2 to 3)." in result
assert "line 1" not in result
assert "line 2\nline 3" in result
assert "line 4" not in result
@pytest.mark.asyncio
async def test_read_note_pure_callable_execution(mock_ctx, tmp_path):
"""Verify invoking read_note_pure directly as a callable."""
save_note_content(tmp_path, "callable_pure", "hello from pure callable")
result = await read_note_pure(ctx=mock_ctx, key="callable_pure")
assert "Successfully read note 'callable_pure'." in result
assert "hello from pure callable" in result
@pytest.mark.asyncio
async def test_read_note_pure_not_found(mock_ctx):
"""Verify error handling when note is missing."""
result = await read_note_pure.execute(ctx=mock_ctx, key="nonexistent_pure")
assert "Note 'nonexistent_pure' not found" in result
@pytest.mark.asyncio
async def test_read_note_pure_other_error(mock_ctx):
"""Verify error handling when read_note_content raises an exception."""
with patch(
"artemis.tools.scratchpad.read_note_content",
side_effect=OSError("Disk error"),
):
result = await read_note_pure.execute(ctx=mock_ctx, key="err_note")
assert "Failed to read note 'err_note': Disk error" in result
@pytest.mark.asyncio
async def test_get_read_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_read_note_tool_pure exports a LangChain tool named 'read_note'."""
save_note_content(tmp_path, "pure_export", "exported pure note")
pure_tool = get_read_note_tool_pure(mock_ctx)
assert pure_tool.name == "read_note"
result = await pure_tool.ainvoke({"key": "pure_export"})
assert "Successfully read note 'pure_export'." in result
assert "exported pure note" in result
+177
View File
@@ -0,0 +1,177 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
ReadNote,
ReadNoteArgs,
ReadNoteTool,
get_read_note_tool,
get_read_note_tool_pure,
read_note,
read_note_wrapper,
)
from artemis.utils.notes import save_note_content
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_read_note_tool_subclass_and_registry():
"""Verify ReadNoteTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(ReadNoteTool, ArtemisTool)
assert issubclass(ReadNote, ArtemisTool)
assert isinstance(read_note, ArtemisTool)
assert isinstance(read_note, ReadNoteTool)
assert read_note.name == "read_note"
assert read_note.category == "memory"
assert read_note.args_schema == ReadNoteArgs
# Registry lookup
reg_tool = ToolRegistry.get("read_note")
assert reg_tool is not None
assert isinstance(reg_tool, ReadNoteTool)
# GenAI FunctionDeclaration export
declaration = read_note.to_genai_declaration()
assert declaration.name == "read_note"
assert "key" in declaration.parameters.properties
assert "start_line" in declaration.parameters.properties
assert "end_line" in declaration.parameters.properties
# Wrapper check
assert read_note_wrapper is not None
assert read_note_wrapper.tool_fn_getter == get_read_note_tool
@pytest.mark.asyncio
async def test_read_note_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of ReadNoteTool.execute reads saved note."""
save_note_content(tmp_path, "sample", "line 1\nline 2\nline 3\nline 4\nline 5")
result = await read_note.execute(ctx=mock_ctx, key="sample")
assert "Successfully read note 'sample'." in result
assert "line 1\nline 2\nline 3\nline 4\nline 5" in result
@pytest.mark.asyncio
async def test_read_note_slice(mock_ctx, tmp_path):
"""Verify reading a line slice with start_line and end_line."""
save_note_content(tmp_path, "sample", "line 1\nline 2\nline 3\nline 4\nline 5")
result = await read_note.execute(
ctx=mock_ctx,
key="sample",
start_line=2,
end_line=4,
)
assert "Successfully read note 'sample' (lines 2 to 4)." in result
assert "line 1" not in result
assert "line 2\nline 3\nline 4" in result
assert "line 5" not in result
@pytest.mark.asyncio
async def test_read_note_callable_execution(mock_ctx, tmp_path):
"""Verify invoking read_note directly as a callable."""
save_note_content(tmp_path, "callable_sample", "hello from callable")
result = await read_note(ctx=mock_ctx, key="callable_sample")
assert "Successfully read note 'callable_sample'." in result
assert "hello from callable" in result
@pytest.mark.asyncio
async def test_read_note_with_state_command(mock_ctx, tmp_path):
"""Verify ReadNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
save_note_content(tmp_path, "state_sample", "state note content")
mock_state = MagicMock(spec=State)
mock_state.current_agent = "operator"
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
result = await read_note.execute(
ctx=mock_ctx,
key="state_sample",
state=mock_state,
tool_call_id="call_read_999",
)
assert isinstance(result, Command)
assert VALIDATOR_MESSAGES_KEY in result.update
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
assert len(tool_messages) == 1
assert tool_messages[0].tool_call_id == "call_read_999"
assert "Successfully read note 'state_sample'." in tool_messages[0].content
assert "state note content" in tool_messages[0].content
assert tool_messages[0].status == "success"
mock_state.asanitize_update.assert_called_once()
@pytest.mark.asyncio
async def test_read_note_not_found(mock_ctx):
"""Verify error handling when a note is not found."""
result = await read_note.execute(ctx=mock_ctx, key="nonexistent")
assert "Note 'nonexistent' not found" in result
@pytest.mark.asyncio
async def test_read_note_other_error(mock_ctx):
"""Verify error handling for unexpected exceptions."""
with patch(
"artemis.tools.scratchpad.read_note_content",
side_effect=ValueError("Corrupted data"),
):
result = await read_note.execute(ctx=mock_ctx, key="bad_note")
assert "Failed to read note 'bad_note': Corrupted data" in result
@pytest.mark.asyncio
async def test_get_read_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_read_note_tool exports a LangChain tool that works with ainvoke."""
save_note_content(tmp_path, "lc_sample", "langchain content")
lc_tool = get_read_note_tool(mock_ctx)
assert lc_tool.name == "read_note"
result = await lc_tool.ainvoke({"key": "lc_sample"})
assert "Successfully read note 'lc_sample'." in result
assert "langchain content" in result
@pytest.mark.asyncio
async def test_get_read_note_tool_pure(mock_ctx, tmp_path):
"""Verify get_read_note_tool_pure returns a pure LangChain tool."""
save_note_content(tmp_path, "pure_sample", "pure content")
pure_tool = get_read_note_tool_pure(mock_ctx)
assert pure_tool.name == "read_note"
result = await pure_tool.ainvoke({"key": "pure_sample"})
assert "Successfully read note 'pure_sample'." in result
assert "pure content" in result
@@ -0,0 +1,122 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest.mock import MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
SaveNoteArgs,
SaveNotePure,
SaveNotePureTool,
get_save_note_tool_pure,
save_note_pure,
)
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_save_note_pure_tool_subclass_and_registry():
"""Verify SaveNotePureTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(SaveNotePureTool, ArtemisTool)
assert issubclass(SaveNotePure, ArtemisTool)
assert isinstance(save_note_pure, ArtemisTool)
assert isinstance(save_note_pure, SaveNotePureTool)
assert save_note_pure.name == "save_note_pure"
assert save_note_pure.category == "memory"
assert save_note_pure.args_schema == SaveNoteArgs
# Registry lookup
reg_tool = ToolRegistry.get("save_note_pure")
assert reg_tool is not None
assert isinstance(reg_tool, SaveNotePureTool)
# GenAI FunctionDeclaration export
declaration = save_note_pure.to_genai_declaration()
assert declaration.name == "save_note_pure"
assert "key" in declaration.parameters.properties
assert "content" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_save_note_pure_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of SaveNotePureTool.execute creates note file."""
result = await save_note_pure.execute(
ctx=mock_ctx,
key="test_pure_save",
content="Pure saved content",
)
assert result == "Successfully saved note to test_pure_save.md."
note_path = Path(tmp_path) / "notes" / "test_pure_save.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Pure saved content"
@pytest.mark.asyncio
async def test_save_note_pure_callable_execution(mock_ctx, tmp_path):
"""Verify invoking save_note_pure directly as a callable."""
result = await save_note_pure(
ctx=mock_ctx,
key="callable_pure_save",
content="Callable pure content",
)
assert result == "Successfully saved note to callable_pure_save.md."
note_path = Path(tmp_path) / "notes" / "callable_pure_save.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Callable pure content"
@pytest.mark.asyncio
async def test_save_note_pure_execution_failure(mock_ctx):
"""Verify error handling when save_note_content fails."""
with patch(
"artemis.tools.scratchpad.save_note_content",
side_effect=PermissionError("Permission denied"),
):
result = await save_note_pure.execute(
ctx=mock_ctx,
key="fail_save",
content="Some content",
)
assert "Failed to save note fail_save.md: Permission denied" in result
@pytest.mark.asyncio
async def test_get_save_note_tool_pure_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_save_note_tool_pure exports a LangChain tool named 'save_note'."""
pure_tool = get_save_note_tool_pure(mock_ctx)
assert pure_tool.name == "save_note"
result = await pure_tool.ainvoke(
{
"key": "lc_pure_save",
"content": "Saved via LangChain pure tool",
}
)
assert result == "Successfully saved note to lc_pure_save.md."
note_path = Path(tmp_path) / "notes" / "lc_pure_save.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Saved via LangChain pure tool"
+180
View File
@@ -0,0 +1,180 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
from artemis.constants import VALIDATOR_MESSAGES_KEY
from artemis.context import ArtemisContext
from artemis.graph.state import State
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.scratchpad import (
SaveNote,
SaveNoteArgs,
SaveNoteTool,
get_save_note_tool,
get_save_note_tool_pure,
save_note,
save_note_wrapper,
)
from langgraph.types import Command
import pytest
@pytest.fixture
def mock_ctx(tmp_path):
ctx = MagicMock(spec=ArtemisContext)
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = str(tmp_path)
return ctx
def test_save_note_tool_subclass_and_registry():
"""Verify SaveNoteTool is a subclass of ArtemisTool and properly registered in ToolRegistry."""
assert issubclass(SaveNoteTool, ArtemisTool)
assert issubclass(SaveNote, ArtemisTool)
assert isinstance(save_note, ArtemisTool)
assert isinstance(save_note, SaveNoteTool)
assert save_note.name == "save_note"
assert save_note.category == "memory"
assert save_note.args_schema == SaveNoteArgs
# Registry lookup
reg_tool = ToolRegistry.get("save_note")
assert reg_tool is not None
assert isinstance(reg_tool, SaveNoteTool)
# GenAI FunctionDeclaration export
declaration = save_note.to_genai_declaration()
assert declaration.name == "save_note"
assert "key" in declaration.parameters.properties
assert "content" in declaration.parameters.properties
# Wrapper check
assert save_note_wrapper is not None
assert save_note_wrapper.tool_fn_getter == get_save_note_tool
@pytest.mark.asyncio
async def test_save_note_direct_execution_success(mock_ctx, tmp_path):
"""Verify direct execution of SaveNoteTool.execute creates note file."""
result = await save_note.execute(
ctx=mock_ctx,
key="test_note",
content="Hello Artemis memory",
)
assert result == "Successfully saved note to test_note.md."
note_path = Path(tmp_path) / "notes" / "test_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Hello Artemis memory"
@pytest.mark.asyncio
async def test_save_note_callable_execution(mock_ctx, tmp_path):
"""Verify invoking save_note directly as a callable."""
result = await save_note(
ctx=mock_ctx,
key="callable_note",
content="Callable invocation test",
)
assert result == "Successfully saved note to callable_note.md."
note_path = Path(tmp_path) / "notes" / "callable_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Callable invocation test"
@pytest.mark.asyncio
async def test_save_note_with_state_command(mock_ctx, tmp_path):
"""Verify SaveNoteTool.execute with state returns Command updating VALIDATOR_MESSAGES_KEY."""
mock_state = MagicMock(spec=State)
mock_state.current_agent = "operator"
mock_state.asanitize_update = AsyncMock(side_effect=lambda ctx, update, agent: update)
result = await save_note.execute(
ctx=mock_ctx,
key="state_note",
content="Note with state",
state=mock_state,
tool_call_id="call_save_999",
)
assert isinstance(result, Command)
assert VALIDATOR_MESSAGES_KEY in result.update
tool_messages = result.update[VALIDATOR_MESSAGES_KEY]
assert len(tool_messages) == 1
assert tool_messages[0].tool_call_id == "call_save_999"
assert tool_messages[0].content == "Successfully saved note to state_note.md."
assert tool_messages[0].status == "success"
mock_state.asanitize_update.assert_called_once()
note_path = Path(tmp_path) / "notes" / "state_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Note with state"
@pytest.mark.asyncio
async def test_save_note_execution_failure(mock_ctx):
"""Verify error handling when saving note fails."""
with patch(
"artemis.tools.scratchpad.save_note_content",
side_effect=PermissionError("Permission denied"),
):
result = await save_note.execute(
ctx=mock_ctx,
key="fail_note",
content="Fail content",
)
assert "Failed to save note fail_note.md: Permission denied" in result
@pytest.mark.asyncio
async def test_get_save_note_tool_langchain_ainvoke(mock_ctx, tmp_path):
"""Verify get_save_note_tool exports a LangChain tool that works with ainvoke."""
lc_tool = get_save_note_tool(mock_ctx)
assert lc_tool.name == "save_note"
result = await lc_tool.ainvoke(
{
"key": "lc_note",
"content": "Saved via LangChain BaseTool",
}
)
assert result == "Successfully saved note to lc_note.md."
note_path = Path(tmp_path) / "notes" / "lc_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Saved via LangChain BaseTool"
@pytest.mark.asyncio
async def test_get_save_note_tool_pure(mock_ctx, tmp_path):
"""Verify get_save_note_tool_pure returns a pure LangChain tool."""
pure_tool = get_save_note_tool_pure(mock_ctx)
assert pure_tool.name == "save_note"
result = await pure_tool.ainvoke(
{
"key": "pure_note",
"content": "Saved via pure tool",
}
)
assert result == "Successfully saved note to pure_note.md."
note_path = Path(tmp_path) / "notes" / "pure_note.md"
assert note_path.exists()
assert note_path.read_text(encoding="utf-8") == "Saved via pure tool"
+262
View File
@@ -0,0 +1,262 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock, patch
from artemis.context import ArtemisContext
from artemis.tools.base import ArtemisTool, ToolRegistry
from artemis.tools.mobile.search_logs import (
SearchLogs,
SearchLogsArgs,
SearchLogsTool,
SearchLogsToolAlias,
get_search_logs_tool,
search_and_merge_logs,
search_logs,
)
import pytest
def test_search_and_merge_logs_no_matches():
logs = "line 0: info\nline 1: info\nline 2: info"
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
assert result == "No matches found for 'ERROR'."
def test_search_and_merge_logs_single_match():
logs = "\n".join([f"line {i}" if i != 5 else "ERROR: failure" for i in range(10)])
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
expected = (
"--- Match at line(s) 5 ---\n"
" 3: line 3\n"
" 4: line 4\n"
" 5: [MATCH] ERROR: failure\n"
" 6: line 6\n"
" 7: line 7\n"
"--------------------"
)
assert result == expected
def test_search_and_merge_logs_overlapping_matches():
logs = "\n".join([f"line {i}" if i not in [4, 6] else "ERROR: failure" for i in range(10)])
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
# Match at 4 (range 2 to 6 inclusive)
# Match at 6 (range 4 to 8 inclusive)
# Merged range: 2 to 8 inclusive
expected = (
"--- Match at line(s) 4, 6 ---\n"
" 2: line 2\n"
" 3: line 3\n"
" 4: [MATCH] ERROR: failure\n"
" 5: line 5\n"
" 6: [MATCH] ERROR: failure\n"
" 7: line 7\n"
" 8: line 8\n"
"--------------------"
)
assert result == expected
def test_search_and_merge_logs_adjacent_matches():
# Match at 3 (range 1 to 5 inclusive)
# Match at 6 (range 4 to 8 inclusive)
# Overlap is at lines 4-5. They are adjacent/overlapping and should merge.
logs = "\n".join([f"line {i}" if i not in [3, 6] else "ERROR: failure" for i in range(10)])
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
expected = (
"--- Match at line(s) 3, 6 ---\n"
" 1: line 1\n"
" 2: line 2\n"
" 3: [MATCH] ERROR: failure\n"
" 4: line 4\n"
" 5: line 5\n"
" 6: [MATCH] ERROR: failure\n"
" 7: line 7\n"
" 8: line 8\n"
"--------------------"
)
assert result == expected
def test_search_and_merge_logs_multiple_separate_blocks():
# Match at 2 (range 0 to 4 inclusive)
# Match at 8 (range 6 to 9 inclusive since len is 10)
# Non-overlapping since end of first is 4, start of second is 6.
logs = "\n".join([f"line {i}" if i not in [2, 8] else "ERROR: failure" for i in range(10)])
result = search_and_merge_logs(logs, "ERROR", context_lines=2)
expected = (
"--- Match at line(s) 2 ---\n"
" 0: line 0\n"
" 1: line 1\n"
" 2: [MATCH] ERROR: failure\n"
" 3: line 3\n"
" 4: line 4\n"
"--------------------\n"
"--- Match at line(s) 8 ---\n"
" 6: line 6\n"
" 7: line 7\n"
" 8: [MATCH] ERROR: failure\n"
" 9: line 9\n"
"--------------------"
)
assert result == expected
def test_search_and_merge_logs_regex():
logs = "line 0\nERR_123: failed\nline 2\nERR_456: critical"
result = search_and_merge_logs(logs, r"ERR_\d+", context_lines=1, is_regex=True)
expected = (
"--- Match at line(s) 1, 3 ---\n"
" 0: line 0\n"
" 1: [MATCH] ERR_123: failed\n"
" 2: line 2\n"
" 3: [MATCH] ERR_456: critical\n"
"--------------------"
)
assert result == expected
def test_search_and_merge_logs_invalid_regex():
logs = "line 0"
result = search_and_merge_logs(logs, "[invalid", is_regex=True)
assert result.startswith("Error: Invalid regex:")
# ---------------------------------------------------------------------------
# SearchLogsTool Unit Tests
# ---------------------------------------------------------------------------
@pytest.fixture
def mock_ctx():
return MagicMock(spec=ArtemisContext)
def test_search_logs_tool_subclass_and_registry():
"""Verify SearchLogsTool is a subclass of ArtemisTool and properly registered."""
assert issubclass(SearchLogsTool, ArtemisTool)
assert issubclass(SearchLogs, ArtemisTool)
assert issubclass(SearchLogsToolAlias, ArtemisTool)
assert isinstance(search_logs, ArtemisTool)
assert isinstance(search_logs, SearchLogsTool)
assert search_logs.name == "search_logs"
assert search_logs.category == "diagnostic"
assert search_logs.args_schema == SearchLogsArgs
# Registry lookup
reg_tool = ToolRegistry.get("search_logs")
assert reg_tool is not None
assert isinstance(reg_tool, SearchLogsTool)
# GenAI FunctionDeclaration export
declaration = search_logs.to_genai_declaration()
assert declaration.name == "search_logs"
assert "keyword" in declaration.parameters.properties
assert "is_regex" in declaration.parameters.properties
assert "lines" in declaration.parameters.properties
assert "since_time" in declaration.parameters.properties
assert "until_time" in declaration.parameters.properties
assert "context_lines" in declaration.parameters.properties
@pytest.mark.asyncio
async def test_search_logs_direct_execution(mock_ctx):
"""Verify direct execution of SearchLogsTool."""
mock_log_output = "line 1: FATAL: crash occurred\nline 2: normal"
with patch(
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
return_value=mock_log_output,
) as mock_fetch:
result = await search_logs.execute(
ctx=mock_ctx,
keyword="FATAL",
is_regex=False,
lines=5000,
since_time="08-12 10:00:00",
until_time="08-12 10:05:00",
context_lines=1,
)
mock_fetch.assert_called_once_with(
ctx=mock_ctx,
lines=5000,
since_time="08-12 10:00:00",
until_time="08-12 10:05:00",
)
assert "[MATCH] line 1: FATAL: crash occurred" in result
@pytest.mark.asyncio
async def test_search_logs_default_args(mock_ctx):
"""Verify search_logs defaults lines to 10000 and context_lines to 0."""
with patch(
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
return_value="info line\nerror line",
) as mock_fetch:
result = await search_logs.execute(
ctx=mock_ctx, keyword="error", lines=None, context_lines=None
)
mock_fetch.assert_called_once_with(
ctx=mock_ctx,
lines=10000,
since_time=None,
until_time=None,
)
assert "[MATCH] error line" in result
@pytest.mark.asyncio
async def test_search_logs_exception_handling(mock_ctx):
"""Verify error handling when searching logs raises an exception."""
with patch(
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
side_effect=RuntimeError("Device unreachable"),
):
result = await search_logs.execute(ctx=mock_ctx, keyword="test")
assert "Error searching logs: Device unreachable" in result
@pytest.mark.asyncio
async def test_search_logs_callable_execution(mock_ctx):
"""Verify invoking search_logs directly as a callable."""
with patch(
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
return_value="NullPointerException in activity",
):
result = await search_logs(ctx=mock_ctx, keyword="NullPointer", context_lines=0)
assert "[MATCH] NullPointerException in activity" in result
@pytest.mark.asyncio
async def test_get_search_logs_tool_langchain_ainvoke(mock_ctx):
"""Verify get_search_logs_tool exports a LangChain tool that works with ainvoke."""
tool = get_search_logs_tool(mock_ctx)
assert tool.name == "search_logs"
with patch(
"artemis.tools.mobile.search_logs.fetch_and_filter_logs",
return_value="critical error detected",
) as mock_fetch:
result = await tool.ainvoke({"keyword": "critical", "lines": 2000})
mock_fetch.assert_called_once_with(
ctx=mock_ctx,
lines=2000,
since_time=None,
until_time=None,
)
assert "[MATCH] critical error detected" in result
Loaded 100 of 115 files, more files were not shown because too many files have changed in this diff. Show more